Upload 1580 files
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- runtime/Lib/site_packages/models/__init__.py +13 -0
- runtime/Lib/site_packages/models/audio/__init__.py +3 -0
- runtime/Lib/site_packages/models/audio/aec/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/aec/layers/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/aec/layers/activations.py +62 -0
- runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py +80 -0
- runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py +180 -0
- runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py +52 -0
- runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py +484 -0
- runtime/Lib/site_packages/models/audio/aec/network/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/aec/network/loss.py +396 -0
- runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py +250 -0
- runtime/Lib/site_packages/models/audio/aec/network/se_net.py +487 -0
- runtime/Lib/site_packages/models/audio/ans/__init__.py +23 -0
- runtime/Lib/site_packages/models/audio/ans/complex_nn.py +206 -0
- runtime/Lib/site_packages/models/audio/ans/conv_stft.py +113 -0
- runtime/Lib/site_packages/models/audio/ans/denoise_net.py +73 -0
- runtime/Lib/site_packages/models/audio/ans/frcrn.py +280 -0
- runtime/Lib/site_packages/models/audio/ans/layers/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/ans/layers/activations.py +62 -0
- runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py +86 -0
- runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py +31 -0
- runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py +156 -0
- runtime/Lib/site_packages/models/audio/ans/se_module_complex.py +27 -0
- runtime/Lib/site_packages/models/audio/ans/unet.py +276 -0
- runtime/Lib/site_packages/models/audio/asr/__init__.py +23 -0
- runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py +51 -0
- runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py +38 -0
- runtime/Lib/site_packages/models/audio/itn/__init__.py +22 -0
- runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py +44 -0
- runtime/Lib/site_packages/models/audio/kws/__init__.py +26 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py +497 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py +238 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py +233 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/model.py +98 -0
- runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py +123 -0
- runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py +32 -0
- runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py +99 -0
- runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py +521 -0
- runtime/Lib/site_packages/models/audio/kws/nearfield/model.py +178 -0
- runtime/Lib/site_packages/models/audio/punc/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py +43 -0
- runtime/Lib/site_packages/models/audio/separation/__init__.py +24 -0
- runtime/Lib/site_packages/models/audio/separation/layer_norm.py +68 -0
- runtime/Lib/site_packages/models/audio/separation/m2/__init__.py +0 -0
- runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py +278 -0
- runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py +144 -0
- runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py +125 -0
runtime/Lib/site_packages/models/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.error import (AUDIO_IMPORT_ERROR,
|
| 5 |
+
TENSORFLOW_IMPORT_WARNING)
|
| 6 |
+
from modelscope.utils.import_utils import is_torch_available
|
| 7 |
+
from . import audio, cv, multi_modal, nlp
|
| 8 |
+
from .base import Head, Model
|
| 9 |
+
from .builder import BACKBONES, HEADS, MODELS, build_model
|
| 10 |
+
|
| 11 |
+
if is_torch_available():
|
| 12 |
+
from .base.base_torch_model import TorchModel
|
| 13 |
+
from .base.base_torch_head import TorchHead
|
runtime/Lib/site_packages/models/audio/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
from . import ans, asr, itn, kws, separation, sv, tts
|
runtime/Lib/site_packages/models/audio/aec/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/aec/layers/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/aec/layers/activations.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
|
| 5 |
+
from .layer_base import LayerBase
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class RectifiedLinear(LayerBase):
|
| 9 |
+
|
| 10 |
+
def __init__(self, input_dim, output_dim):
|
| 11 |
+
super(RectifiedLinear, self).__init__()
|
| 12 |
+
self.dim = input_dim
|
| 13 |
+
self.relu = nn.ReLU()
|
| 14 |
+
|
| 15 |
+
def forward(self, input):
|
| 16 |
+
return self.relu(input)
|
| 17 |
+
|
| 18 |
+
def to_kaldi_nnet(self):
|
| 19 |
+
re_str = ''
|
| 20 |
+
re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
|
| 21 |
+
return re_str
|
| 22 |
+
|
| 23 |
+
def load_kaldi_nnet(self, instr):
|
| 24 |
+
return instr
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class LogSoftmax(LayerBase):
|
| 28 |
+
|
| 29 |
+
def __init__(self, input_dim, output_dim):
|
| 30 |
+
super(LogSoftmax, self).__init__()
|
| 31 |
+
self.dim = input_dim
|
| 32 |
+
self.ls = nn.LogSoftmax()
|
| 33 |
+
|
| 34 |
+
def forward(self, input):
|
| 35 |
+
return self.ls(input)
|
| 36 |
+
|
| 37 |
+
def to_kaldi_nnet(self):
|
| 38 |
+
re_str = ''
|
| 39 |
+
re_str += '<Softmax> %d %d\n' % (self.dim, self.dim)
|
| 40 |
+
return re_str
|
| 41 |
+
|
| 42 |
+
def load_kaldi_nnet(self, instr):
|
| 43 |
+
return instr
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class Sigmoid(LayerBase):
|
| 47 |
+
|
| 48 |
+
def __init__(self, input_dim, output_dim):
|
| 49 |
+
super(Sigmoid, self).__init__()
|
| 50 |
+
self.dim = input_dim
|
| 51 |
+
self.sig = nn.Sigmoid()
|
| 52 |
+
|
| 53 |
+
def forward(self, input):
|
| 54 |
+
return self.sig(input)
|
| 55 |
+
|
| 56 |
+
def to_kaldi_nnet(self):
|
| 57 |
+
re_str = ''
|
| 58 |
+
re_str += '<Sigmoid> %d %d\n' % (self.dim, self.dim)
|
| 59 |
+
return re_str
|
| 60 |
+
|
| 61 |
+
def load_kaldi_nnet(self, instr):
|
| 62 |
+
return instr
|
runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch as th
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
|
| 7 |
+
from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
|
| 8 |
+
to_kaldi_matrix)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class AffineTransform(LayerBase):
|
| 12 |
+
|
| 13 |
+
def __init__(self, input_dim, output_dim):
|
| 14 |
+
super(AffineTransform, self).__init__()
|
| 15 |
+
self.input_dim = input_dim
|
| 16 |
+
self.output_dim = output_dim
|
| 17 |
+
self.linear = nn.Linear(input_dim, output_dim)
|
| 18 |
+
|
| 19 |
+
def forward(self, input):
|
| 20 |
+
return self.linear(input)
|
| 21 |
+
|
| 22 |
+
def to_kaldi_nnet(self):
|
| 23 |
+
re_str = ''
|
| 24 |
+
re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
|
| 25 |
+
self.input_dim)
|
| 26 |
+
re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
|
| 27 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 28 |
+
x = linear_weights.squeeze().numpy()
|
| 29 |
+
re_str += to_kaldi_matrix(x)
|
| 30 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 31 |
+
x = linear_bias.squeeze().numpy()
|
| 32 |
+
re_str += to_kaldi_matrix(x)
|
| 33 |
+
return re_str
|
| 34 |
+
|
| 35 |
+
def to_raw_nnet(self, fid):
|
| 36 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 37 |
+
x = linear_weights.squeeze().numpy()
|
| 38 |
+
x.tofile(fid)
|
| 39 |
+
|
| 40 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 41 |
+
x = linear_bias.squeeze().numpy()
|
| 42 |
+
x.tofile(fid)
|
| 43 |
+
|
| 44 |
+
def load_kaldi_nnet(self, instr):
|
| 45 |
+
output = expect_token_number(
|
| 46 |
+
instr,
|
| 47 |
+
'<LearnRateCoef>',
|
| 48 |
+
)
|
| 49 |
+
if output is None:
|
| 50 |
+
raise Exception('AffineTransform format error for <LearnRateCoef>')
|
| 51 |
+
instr, lr = output
|
| 52 |
+
|
| 53 |
+
output = expect_token_number(instr, '<BiasLearnRateCoef>')
|
| 54 |
+
if output is None:
|
| 55 |
+
raise Exception(
|
| 56 |
+
'AffineTransform format error for <BiasLearnRateCoef>')
|
| 57 |
+
instr, lr = output
|
| 58 |
+
|
| 59 |
+
output = expect_token_number(instr, '<MaxNorm>')
|
| 60 |
+
if output is None:
|
| 61 |
+
raise Exception('AffineTransform format error for <MaxNorm>')
|
| 62 |
+
instr, lr = output
|
| 63 |
+
|
| 64 |
+
output = expect_kaldi_matrix(instr)
|
| 65 |
+
if output is None:
|
| 66 |
+
raise Exception('AffineTransform format error for parsing matrix')
|
| 67 |
+
instr, mat = output
|
| 68 |
+
|
| 69 |
+
print(mat.shape)
|
| 70 |
+
self.linear.weight = th.nn.Parameter(
|
| 71 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 72 |
+
|
| 73 |
+
output = expect_kaldi_matrix(instr)
|
| 74 |
+
if output is None:
|
| 75 |
+
raise Exception('AffineTransform format error for parsing matrix')
|
| 76 |
+
instr, mat = output
|
| 77 |
+
mat = np.squeeze(mat)
|
| 78 |
+
self.linear.bias = th.nn.Parameter(
|
| 79 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 80 |
+
return instr
|
runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch as th
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
|
| 8 |
+
from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
|
| 9 |
+
to_kaldi_matrix)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class DeepFsmn(LayerBase):
|
| 13 |
+
|
| 14 |
+
def __init__(self,
|
| 15 |
+
input_dim,
|
| 16 |
+
output_dim,
|
| 17 |
+
lorder=None,
|
| 18 |
+
rorder=None,
|
| 19 |
+
hidden_size=None,
|
| 20 |
+
layer_norm=False,
|
| 21 |
+
dropout=0):
|
| 22 |
+
super(DeepFsmn, self).__init__()
|
| 23 |
+
|
| 24 |
+
self.input_dim = input_dim
|
| 25 |
+
self.output_dim = output_dim
|
| 26 |
+
|
| 27 |
+
if lorder is None:
|
| 28 |
+
return
|
| 29 |
+
|
| 30 |
+
self.lorder = lorder
|
| 31 |
+
self.rorder = rorder
|
| 32 |
+
self.hidden_size = hidden_size
|
| 33 |
+
self.layer_norm = layer_norm
|
| 34 |
+
|
| 35 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 36 |
+
self.norm = nn.LayerNorm(hidden_size)
|
| 37 |
+
self.drop1 = nn.Dropout(p=dropout)
|
| 38 |
+
self.drop2 = nn.Dropout(p=dropout)
|
| 39 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 40 |
+
|
| 41 |
+
self.conv1 = nn.Conv2d(
|
| 42 |
+
output_dim,
|
| 43 |
+
output_dim, [lorder, 1], [1, 1],
|
| 44 |
+
groups=output_dim,
|
| 45 |
+
bias=False)
|
| 46 |
+
self.conv2 = nn.Conv2d(
|
| 47 |
+
output_dim,
|
| 48 |
+
output_dim, [rorder, 1], [1, 1],
|
| 49 |
+
groups=output_dim,
|
| 50 |
+
bias=False)
|
| 51 |
+
|
| 52 |
+
def forward(self, input):
|
| 53 |
+
|
| 54 |
+
f1 = F.relu(self.linear(input))
|
| 55 |
+
|
| 56 |
+
f1 = self.drop1(f1)
|
| 57 |
+
if self.layer_norm:
|
| 58 |
+
f1 = self.norm(f1)
|
| 59 |
+
|
| 60 |
+
p1 = self.project(f1)
|
| 61 |
+
|
| 62 |
+
x = th.unsqueeze(p1, 1)
|
| 63 |
+
|
| 64 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 65 |
+
|
| 66 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
|
| 67 |
+
yr = F.pad(x_per, [0, 0, 0, self.rorder])
|
| 68 |
+
yr = yr[:, :, 1:, :]
|
| 69 |
+
|
| 70 |
+
out = x_per + self.conv1(y) + self.conv2(yr)
|
| 71 |
+
out = self.drop2(out)
|
| 72 |
+
|
| 73 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 74 |
+
|
| 75 |
+
return input + out1.squeeze()
|
| 76 |
+
|
| 77 |
+
def to_kaldi_nnet(self):
|
| 78 |
+
re_str = ''
|
| 79 |
+
re_str += '<UniDeepFsmn> %d %d\n'\
|
| 80 |
+
% (self.output_dim, self.input_dim)
|
| 81 |
+
re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n'\
|
| 82 |
+
% (1, self.hidden_size, self.lorder, 1)
|
| 83 |
+
lfiters = self.state_dict()['conv1.weight']
|
| 84 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 85 |
+
re_str += to_kaldi_matrix(x)
|
| 86 |
+
proj_weights = self.state_dict()['project.weight']
|
| 87 |
+
x = proj_weights.squeeze().numpy()
|
| 88 |
+
re_str += to_kaldi_matrix(x)
|
| 89 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 90 |
+
x = linear_weights.squeeze().numpy()
|
| 91 |
+
re_str += to_kaldi_matrix(x)
|
| 92 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 93 |
+
x = linear_bias.squeeze().numpy()
|
| 94 |
+
re_str += to_kaldi_matrix(x)
|
| 95 |
+
return re_str
|
| 96 |
+
|
| 97 |
+
def load_kaldi_nnet(self, instr):
|
| 98 |
+
output = expect_token_number(
|
| 99 |
+
instr,
|
| 100 |
+
'<LearnRateCoef>',
|
| 101 |
+
)
|
| 102 |
+
if output is None:
|
| 103 |
+
raise Exception('UniDeepFsmn format error for <LearnRateCoef>')
|
| 104 |
+
instr, lr = output
|
| 105 |
+
|
| 106 |
+
output = expect_token_number(
|
| 107 |
+
instr,
|
| 108 |
+
'<HidSize>',
|
| 109 |
+
)
|
| 110 |
+
if output is None:
|
| 111 |
+
raise Exception('UniDeepFsmn format error for <HidSize>')
|
| 112 |
+
instr, hiddensize = output
|
| 113 |
+
self.hidden_size = int(hiddensize)
|
| 114 |
+
|
| 115 |
+
output = expect_token_number(
|
| 116 |
+
instr,
|
| 117 |
+
'<LOrder>',
|
| 118 |
+
)
|
| 119 |
+
if output is None:
|
| 120 |
+
raise Exception('UniDeepFsmn format error for <LOrder>')
|
| 121 |
+
instr, lorder = output
|
| 122 |
+
self.lorder = int(lorder)
|
| 123 |
+
|
| 124 |
+
output = expect_token_number(
|
| 125 |
+
instr,
|
| 126 |
+
'<LStride>',
|
| 127 |
+
)
|
| 128 |
+
if output is None:
|
| 129 |
+
raise Exception('UniDeepFsmn format error for <LStride>')
|
| 130 |
+
instr, lstride = output
|
| 131 |
+
self.lstride = lstride
|
| 132 |
+
|
| 133 |
+
output = expect_token_number(
|
| 134 |
+
instr,
|
| 135 |
+
'<MaxNorm>',
|
| 136 |
+
)
|
| 137 |
+
if output is None:
|
| 138 |
+
raise Exception('UniDeepFsmn format error for <MaxNorm>')
|
| 139 |
+
|
| 140 |
+
output = expect_kaldi_matrix(instr)
|
| 141 |
+
if output is None:
|
| 142 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 143 |
+
instr, mat = output
|
| 144 |
+
mat1 = np.fliplr(mat.T).copy()
|
| 145 |
+
self.conv1 = nn.Conv2d(
|
| 146 |
+
self.output_dim,
|
| 147 |
+
self.output_dim, [self.lorder, 1], [1, 1],
|
| 148 |
+
groups=self.output_dim,
|
| 149 |
+
bias=False)
|
| 150 |
+
mat_th = th.from_numpy(mat1).type(th.FloatTensor)
|
| 151 |
+
mat_th = mat_th.unsqueeze(1)
|
| 152 |
+
mat_th = mat_th.unsqueeze(3)
|
| 153 |
+
self.conv1.weight = th.nn.Parameter(mat_th)
|
| 154 |
+
|
| 155 |
+
output = expect_kaldi_matrix(instr)
|
| 156 |
+
if output is None:
|
| 157 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 158 |
+
instr, mat = output
|
| 159 |
+
|
| 160 |
+
self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
|
| 161 |
+
self.linear = nn.Linear(self.input_dim, self.hidden_size)
|
| 162 |
+
|
| 163 |
+
self.project.weight = th.nn.Parameter(
|
| 164 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 165 |
+
|
| 166 |
+
output = expect_kaldi_matrix(instr)
|
| 167 |
+
if output is None:
|
| 168 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 169 |
+
instr, mat = output
|
| 170 |
+
self.linear.weight = th.nn.Parameter(
|
| 171 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 172 |
+
|
| 173 |
+
output = expect_kaldi_matrix(instr)
|
| 174 |
+
if output is None:
|
| 175 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 176 |
+
instr, mat = output
|
| 177 |
+
self.linear.bias = th.nn.Parameter(
|
| 178 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 179 |
+
|
| 180 |
+
return instr
|
runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import abc
|
| 4 |
+
import re
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def expect_token_number(instr, token):
|
| 11 |
+
first_token = re.match(r'^\s*' + token, instr)
|
| 12 |
+
if first_token is None:
|
| 13 |
+
return None
|
| 14 |
+
instr = instr[first_token.end():]
|
| 15 |
+
lr = re.match(r'^\s*(-?\d+\.?\d*e?-?\d*?)', instr)
|
| 16 |
+
if lr is None:
|
| 17 |
+
return None
|
| 18 |
+
return instr[lr.end():], lr.groups()[0]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def expect_kaldi_matrix(instr):
|
| 22 |
+
pos2 = instr.find('[', 0)
|
| 23 |
+
pos3 = instr.find(']', pos2)
|
| 24 |
+
mat = []
|
| 25 |
+
for stt in instr[pos2 + 1:pos3].split('\n'):
|
| 26 |
+
tmp_mat = np.fromstring(stt, dtype=np.float32, sep=' ')
|
| 27 |
+
if tmp_mat.size > 0:
|
| 28 |
+
mat.append(tmp_mat)
|
| 29 |
+
return instr[pos3 + 1:], np.array(mat)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def to_kaldi_matrix(np_mat):
|
| 33 |
+
"""
|
| 34 |
+
function that transform as str numpy mat to standard kaldi str matrix
|
| 35 |
+
:param np_mat: numpy mat
|
| 36 |
+
:return: str
|
| 37 |
+
"""
|
| 38 |
+
np.set_printoptions(threshold=np.inf, linewidth=np.nan, suppress=True)
|
| 39 |
+
out_str = str(np_mat)
|
| 40 |
+
out_str = out_str.replace('[', '')
|
| 41 |
+
out_str = out_str.replace(']', '')
|
| 42 |
+
return '[ %s ]\n' % out_str
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class LayerBase(nn.Module, metaclass=abc.ABCMeta):
|
| 46 |
+
|
| 47 |
+
def __init__(self):
|
| 48 |
+
super(LayerBase, self).__init__()
|
| 49 |
+
|
| 50 |
+
@abc.abstractmethod
|
| 51 |
+
def to_kaldi_nnet(self):
|
| 52 |
+
pass
|
runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py
ADDED
|
@@ -0,0 +1,484 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch as th
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
|
| 8 |
+
from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
|
| 9 |
+
to_kaldi_matrix)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class SepConv(nn.Module):
|
| 13 |
+
|
| 14 |
+
def __init__(self,
|
| 15 |
+
in_channels,
|
| 16 |
+
filters,
|
| 17 |
+
out_channels,
|
| 18 |
+
kernel_size=(5, 2),
|
| 19 |
+
dilation=(1, 1)):
|
| 20 |
+
""" :param kernel_size (time, frequency)
|
| 21 |
+
|
| 22 |
+
"""
|
| 23 |
+
super(SepConv, self).__init__()
|
| 24 |
+
# depthwise + pointwise
|
| 25 |
+
self.dconv = nn.Conv2d(
|
| 26 |
+
in_channels,
|
| 27 |
+
in_channels * filters,
|
| 28 |
+
kernel_size,
|
| 29 |
+
dilation=dilation,
|
| 30 |
+
groups=in_channels)
|
| 31 |
+
self.pconv = nn.Conv2d(
|
| 32 |
+
in_channels * filters, out_channels, kernel_size=1)
|
| 33 |
+
self.padding = dilation[0] * (kernel_size[0] - 1)
|
| 34 |
+
|
| 35 |
+
def forward(self, input):
|
| 36 |
+
''' input: [B, C, T, F]
|
| 37 |
+
'''
|
| 38 |
+
x = F.pad(input, [0, 0, self.padding, 0])
|
| 39 |
+
x = self.dconv(x)
|
| 40 |
+
x = self.pconv(x)
|
| 41 |
+
return x
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class Conv2d(nn.Module):
|
| 45 |
+
|
| 46 |
+
def __init__(self,
|
| 47 |
+
input_dim,
|
| 48 |
+
output_dim,
|
| 49 |
+
lorder=20,
|
| 50 |
+
rorder=0,
|
| 51 |
+
groups=1,
|
| 52 |
+
bias=False,
|
| 53 |
+
skip_connect=True):
|
| 54 |
+
super(Conv2d, self).__init__()
|
| 55 |
+
self.lorder = lorder
|
| 56 |
+
self.conv = nn.Conv2d(
|
| 57 |
+
input_dim, output_dim, [lorder, 1], groups=groups, bias=bias)
|
| 58 |
+
self.rorder = rorder
|
| 59 |
+
if self.rorder:
|
| 60 |
+
self.conv2 = nn.Conv2d(
|
| 61 |
+
input_dim, output_dim, [rorder, 1], groups=groups, bias=bias)
|
| 62 |
+
self.skip_connect = skip_connect
|
| 63 |
+
|
| 64 |
+
def forward(self, input):
|
| 65 |
+
# [B, 1, T, F]
|
| 66 |
+
x = th.unsqueeze(input, 1)
|
| 67 |
+
# [B, F, T, 1]
|
| 68 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 69 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
|
| 70 |
+
out = self.conv(y)
|
| 71 |
+
if self.rorder:
|
| 72 |
+
yr = F.pad(x_per, [0, 0, 0, self.rorder])
|
| 73 |
+
yr = yr[:, :, 1:, :]
|
| 74 |
+
out += self.conv2(yr)
|
| 75 |
+
out = out.permute(0, 3, 2, 1).squeeze(1)
|
| 76 |
+
if self.skip_connect:
|
| 77 |
+
out = out + input
|
| 78 |
+
return out
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
class SelfAttLayer(nn.Module):
|
| 82 |
+
|
| 83 |
+
def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
|
| 84 |
+
super(SelfAttLayer, self).__init__()
|
| 85 |
+
|
| 86 |
+
self.input_dim = input_dim
|
| 87 |
+
self.output_dim = output_dim
|
| 88 |
+
|
| 89 |
+
if lorder is None:
|
| 90 |
+
return
|
| 91 |
+
|
| 92 |
+
self.lorder = lorder
|
| 93 |
+
self.hidden_size = hidden_size
|
| 94 |
+
|
| 95 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 96 |
+
|
| 97 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 98 |
+
|
| 99 |
+
self.att = nn.Linear(input_dim, lorder, bias=False)
|
| 100 |
+
|
| 101 |
+
def forward(self, input):
|
| 102 |
+
|
| 103 |
+
f1 = F.relu(self.linear(input))
|
| 104 |
+
|
| 105 |
+
p1 = self.project(f1)
|
| 106 |
+
|
| 107 |
+
x = th.unsqueeze(p1, 1)
|
| 108 |
+
|
| 109 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 110 |
+
|
| 111 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
|
| 112 |
+
|
| 113 |
+
# z [B, F, T, lorder]
|
| 114 |
+
z = x_per
|
| 115 |
+
for i in range(1, self.lorder):
|
| 116 |
+
z = th.cat([z, y[:, :, self.lorder - 1 - i:-i, :]], axis=-1)
|
| 117 |
+
|
| 118 |
+
# [B, T, lorder]
|
| 119 |
+
att = F.softmax(self.att(input), dim=-1)
|
| 120 |
+
att = th.unsqueeze(att, 1)
|
| 121 |
+
z = th.sum(z * att, axis=-1)
|
| 122 |
+
|
| 123 |
+
out1 = z.permute(0, 2, 1)
|
| 124 |
+
|
| 125 |
+
return input + out1
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class TFFsmn(nn.Module):
|
| 129 |
+
|
| 130 |
+
def __init__(self,
|
| 131 |
+
input_dim,
|
| 132 |
+
output_dim,
|
| 133 |
+
lorder=None,
|
| 134 |
+
hidden_size=None,
|
| 135 |
+
dilation=1,
|
| 136 |
+
layer_norm=False,
|
| 137 |
+
dropout=0,
|
| 138 |
+
skip_connect=True):
|
| 139 |
+
super(TFFsmn, self).__init__()
|
| 140 |
+
|
| 141 |
+
self.skip_connect = skip_connect
|
| 142 |
+
|
| 143 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 144 |
+
self.norm = nn.Identity()
|
| 145 |
+
if layer_norm:
|
| 146 |
+
self.norm = nn.LayerNorm(input_dim)
|
| 147 |
+
self.act = nn.ReLU()
|
| 148 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 149 |
+
|
| 150 |
+
self.conv1 = nn.Conv2d(
|
| 151 |
+
output_dim,
|
| 152 |
+
output_dim, [lorder, 1],
|
| 153 |
+
dilation=[dilation, 1],
|
| 154 |
+
groups=output_dim,
|
| 155 |
+
bias=False)
|
| 156 |
+
self.padding_left = dilation * (lorder - 1)
|
| 157 |
+
dorder = 5
|
| 158 |
+
self.conv2 = nn.Conv2d(1, 1, [dorder, 1], bias=False)
|
| 159 |
+
self.padding_freq = dorder - 1
|
| 160 |
+
|
| 161 |
+
def forward(self, input):
|
| 162 |
+
return self.compute1(input)
|
| 163 |
+
|
| 164 |
+
def compute1(self, input):
|
| 165 |
+
''' linear-dconv-relu(norm)-linear-dconv
|
| 166 |
+
'''
|
| 167 |
+
x = self.linear(input)
|
| 168 |
+
# [B, 1, F, T]
|
| 169 |
+
x = th.unsqueeze(x, 1).permute(0, 1, 3, 2)
|
| 170 |
+
z = F.pad(x, [0, 0, self.padding_freq, 0])
|
| 171 |
+
z = self.conv2(z) + x
|
| 172 |
+
x = z.permute(0, 3, 2, 1).squeeze(-1)
|
| 173 |
+
x = self.act(x)
|
| 174 |
+
x = self.norm(x)
|
| 175 |
+
x = self.project(x)
|
| 176 |
+
x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
|
| 177 |
+
# [B, F, T+lorder-1, 1]
|
| 178 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 179 |
+
out = self.conv1(y)
|
| 180 |
+
if self.skip_connect:
|
| 181 |
+
out = out + x
|
| 182 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 183 |
+
|
| 184 |
+
return input + out
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
class CNNFsmn(nn.Module):
|
| 188 |
+
''' use cnn to reduce parameters
|
| 189 |
+
'''
|
| 190 |
+
|
| 191 |
+
def __init__(self,
|
| 192 |
+
input_dim,
|
| 193 |
+
output_dim,
|
| 194 |
+
lorder=None,
|
| 195 |
+
hidden_size=None,
|
| 196 |
+
dilation=1,
|
| 197 |
+
layer_norm=False,
|
| 198 |
+
dropout=0,
|
| 199 |
+
skip_connect=True):
|
| 200 |
+
super(CNNFsmn, self).__init__()
|
| 201 |
+
|
| 202 |
+
self.input_dim = input_dim
|
| 203 |
+
self.output_dim = output_dim
|
| 204 |
+
self.skip_connect = skip_connect
|
| 205 |
+
|
| 206 |
+
if lorder is None:
|
| 207 |
+
return
|
| 208 |
+
|
| 209 |
+
self.lorder = lorder
|
| 210 |
+
self.hidden_size = hidden_size
|
| 211 |
+
|
| 212 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 213 |
+
self.act = nn.ReLU()
|
| 214 |
+
kernel_size = (3, 8)
|
| 215 |
+
stride = (1, 4)
|
| 216 |
+
self.conv = nn.Sequential(
|
| 217 |
+
nn.ConstantPad2d((stride[1], 0, kernel_size[0] - 1, 0), 0),
|
| 218 |
+
nn.Conv2d(1, stride[1], kernel_size=kernel_size, stride=stride))
|
| 219 |
+
|
| 220 |
+
self.dconv = nn.Conv2d(
|
| 221 |
+
output_dim,
|
| 222 |
+
output_dim, [lorder, 1],
|
| 223 |
+
dilation=[dilation, 1],
|
| 224 |
+
groups=output_dim,
|
| 225 |
+
bias=False)
|
| 226 |
+
self.padding_left = dilation * (lorder - 1)
|
| 227 |
+
|
| 228 |
+
def forward(self, input):
|
| 229 |
+
return self.compute2(input)
|
| 230 |
+
|
| 231 |
+
def compute1(self, input):
|
| 232 |
+
''' linear-relu(norm)-conv2d-relu?-dconv
|
| 233 |
+
'''
|
| 234 |
+
# [B, T, F]
|
| 235 |
+
x = self.linear(input)
|
| 236 |
+
x = self.act(x)
|
| 237 |
+
x = th.unsqueeze(x, 1)
|
| 238 |
+
x = self.conv(x)
|
| 239 |
+
# [B, C, T, F] -> [B, 1, T, F]
|
| 240 |
+
b, c, t, f = x.shape
|
| 241 |
+
x = x.view([b, 1, t, -1])
|
| 242 |
+
x = x.permute(0, 3, 2, 1)
|
| 243 |
+
# [B, F, T+lorder-1, 1]
|
| 244 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 245 |
+
out = self.dconv(y)
|
| 246 |
+
if self.skip_connect:
|
| 247 |
+
out = out + x
|
| 248 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 249 |
+
return input + out
|
| 250 |
+
|
| 251 |
+
def compute2(self, input):
|
| 252 |
+
''' conv2d-relu-linear-relu?-dconv
|
| 253 |
+
'''
|
| 254 |
+
x = th.unsqueeze(input, 1)
|
| 255 |
+
x = self.conv(x)
|
| 256 |
+
x = self.act(x)
|
| 257 |
+
# [B, C, T, F] -> [B, T, F]
|
| 258 |
+
b, c, t, f = x.shape
|
| 259 |
+
x = x.view([b, t, -1])
|
| 260 |
+
x = self.linear(x)
|
| 261 |
+
x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
|
| 262 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 263 |
+
out = self.dconv(y)
|
| 264 |
+
if self.skip_connect:
|
| 265 |
+
out = out + x
|
| 266 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 267 |
+
return input + out
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
class UniDeepFsmn(LayerBase):
|
| 271 |
+
|
| 272 |
+
def __init__(self,
|
| 273 |
+
input_dim,
|
| 274 |
+
output_dim,
|
| 275 |
+
lorder=None,
|
| 276 |
+
hidden_size=None,
|
| 277 |
+
dilation=1,
|
| 278 |
+
layer_norm=False,
|
| 279 |
+
dropout=0,
|
| 280 |
+
skip_connect=True):
|
| 281 |
+
super(UniDeepFsmn, self).__init__()
|
| 282 |
+
|
| 283 |
+
self.input_dim = input_dim
|
| 284 |
+
self.output_dim = output_dim
|
| 285 |
+
self.skip_connect = skip_connect
|
| 286 |
+
|
| 287 |
+
if lorder is None:
|
| 288 |
+
return
|
| 289 |
+
|
| 290 |
+
self.lorder = lorder
|
| 291 |
+
self.hidden_size = hidden_size
|
| 292 |
+
|
| 293 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 294 |
+
self.norm = nn.Identity()
|
| 295 |
+
if layer_norm:
|
| 296 |
+
self.norm = nn.LayerNorm(input_dim)
|
| 297 |
+
self.act = nn.ReLU()
|
| 298 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 299 |
+
|
| 300 |
+
self.conv1 = nn.Conv2d(
|
| 301 |
+
output_dim,
|
| 302 |
+
output_dim, [lorder, 1],
|
| 303 |
+
dilation=[dilation, 1],
|
| 304 |
+
groups=output_dim,
|
| 305 |
+
bias=False)
|
| 306 |
+
self.padding_left = dilation * (lorder - 1)
|
| 307 |
+
|
| 308 |
+
def forward(self, input):
|
| 309 |
+
return self.compute1(input)
|
| 310 |
+
|
| 311 |
+
def compute1(self, input):
|
| 312 |
+
''' linear-relu(norm)-linear-dconv
|
| 313 |
+
'''
|
| 314 |
+
# [B, T, F]
|
| 315 |
+
x = self.linear(input)
|
| 316 |
+
x = self.act(x)
|
| 317 |
+
x = self.norm(x)
|
| 318 |
+
x = self.project(x)
|
| 319 |
+
x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
|
| 320 |
+
# [B, F, T+lorder-1, 1]
|
| 321 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 322 |
+
out = self.conv1(y)
|
| 323 |
+
if self.skip_connect:
|
| 324 |
+
out = out + x
|
| 325 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 326 |
+
|
| 327 |
+
return input + out
|
| 328 |
+
|
| 329 |
+
def compute2(self, input):
|
| 330 |
+
''' linear-dconv-linear-relu(norm)
|
| 331 |
+
'''
|
| 332 |
+
x = self.project(input)
|
| 333 |
+
x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
|
| 334 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 335 |
+
out = self.conv1(y)
|
| 336 |
+
if self.skip_connect:
|
| 337 |
+
out = out + x
|
| 338 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 339 |
+
x = self.linear(out)
|
| 340 |
+
x = self.act(x)
|
| 341 |
+
x = self.norm(x)
|
| 342 |
+
|
| 343 |
+
return input + x
|
| 344 |
+
|
| 345 |
+
def compute3(self, input):
|
| 346 |
+
''' dconv-linear-relu(norm)-linear
|
| 347 |
+
'''
|
| 348 |
+
x = th.unsqueeze(input, 1).permute(0, 3, 2, 1)
|
| 349 |
+
y = F.pad(x, [0, 0, self.padding_left, 0])
|
| 350 |
+
out = self.conv1(y)
|
| 351 |
+
if self.skip_connect:
|
| 352 |
+
out = out + x
|
| 353 |
+
out = out.permute(0, 3, 2, 1).squeeze()
|
| 354 |
+
x = self.linear(out)
|
| 355 |
+
x = self.act(x)
|
| 356 |
+
x = self.norm(x)
|
| 357 |
+
x = self.project(x)
|
| 358 |
+
|
| 359 |
+
return input + x
|
| 360 |
+
|
| 361 |
+
def to_kaldi_nnet(self):
|
| 362 |
+
re_str = ''
|
| 363 |
+
re_str += '<UniDeepFsmn> %d %d\n' \
|
| 364 |
+
% (self.output_dim, self.input_dim)
|
| 365 |
+
re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n' \
|
| 366 |
+
% (1, self.hidden_size, self.lorder, 1)
|
| 367 |
+
lfiters = self.state_dict()['conv1.weight']
|
| 368 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 369 |
+
re_str += to_kaldi_matrix(x)
|
| 370 |
+
proj_weights = self.state_dict()['project.weight']
|
| 371 |
+
x = proj_weights.squeeze().numpy()
|
| 372 |
+
re_str += to_kaldi_matrix(x)
|
| 373 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 374 |
+
x = linear_weights.squeeze().numpy()
|
| 375 |
+
re_str += to_kaldi_matrix(x)
|
| 376 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 377 |
+
x = linear_bias.squeeze().numpy()
|
| 378 |
+
re_str += to_kaldi_matrix(x)
|
| 379 |
+
return re_str
|
| 380 |
+
|
| 381 |
+
def to_raw_nnet(self, fid):
|
| 382 |
+
lfiters = self.state_dict()['conv1.weight']
|
| 383 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 384 |
+
x.tofile(fid)
|
| 385 |
+
|
| 386 |
+
proj_weights = self.state_dict()['project.weight']
|
| 387 |
+
x = proj_weights.squeeze().numpy()
|
| 388 |
+
x.tofile(fid)
|
| 389 |
+
|
| 390 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 391 |
+
x = linear_weights.squeeze().numpy()
|
| 392 |
+
x.tofile(fid)
|
| 393 |
+
|
| 394 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 395 |
+
x = linear_bias.squeeze().numpy()
|
| 396 |
+
x.tofile(fid)
|
| 397 |
+
|
| 398 |
+
def load_kaldi_nnet(self, instr):
|
| 399 |
+
output = expect_token_number(
|
| 400 |
+
instr,
|
| 401 |
+
'<LearnRateCoef>',
|
| 402 |
+
)
|
| 403 |
+
if output is None:
|
| 404 |
+
raise Exception('UniDeepFsmn format error for <LearnRateCoef>')
|
| 405 |
+
instr, lr = output
|
| 406 |
+
|
| 407 |
+
output = expect_token_number(
|
| 408 |
+
instr,
|
| 409 |
+
'<HidSize>',
|
| 410 |
+
)
|
| 411 |
+
if output is None:
|
| 412 |
+
raise Exception('UniDeepFsmn format error for <HidSize>')
|
| 413 |
+
instr, hiddensize = output
|
| 414 |
+
self.hidden_size = int(hiddensize)
|
| 415 |
+
|
| 416 |
+
output = expect_token_number(
|
| 417 |
+
instr,
|
| 418 |
+
'<LOrder>',
|
| 419 |
+
)
|
| 420 |
+
if output is None:
|
| 421 |
+
raise Exception('UniDeepFsmn format error for <LOrder>')
|
| 422 |
+
instr, lorder = output
|
| 423 |
+
self.lorder = int(lorder)
|
| 424 |
+
|
| 425 |
+
output = expect_token_number(
|
| 426 |
+
instr,
|
| 427 |
+
'<LStride>',
|
| 428 |
+
)
|
| 429 |
+
if output is None:
|
| 430 |
+
raise Exception('UniDeepFsmn format error for <LStride>')
|
| 431 |
+
instr, lstride = output
|
| 432 |
+
self.lstride = lstride
|
| 433 |
+
|
| 434 |
+
output = expect_token_number(
|
| 435 |
+
instr,
|
| 436 |
+
'<MaxNorm>',
|
| 437 |
+
)
|
| 438 |
+
if output is None:
|
| 439 |
+
raise Exception('UniDeepFsmn format error for <MaxNorm>')
|
| 440 |
+
|
| 441 |
+
output = expect_kaldi_matrix(instr)
|
| 442 |
+
if output is None:
|
| 443 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 444 |
+
instr, mat = output
|
| 445 |
+
mat1 = np.fliplr(mat.T).copy()
|
| 446 |
+
|
| 447 |
+
self.conv1 = nn.Conv2d(
|
| 448 |
+
self.output_dim,
|
| 449 |
+
self.output_dim, [self.lorder, 1], [1, 1],
|
| 450 |
+
groups=self.output_dim,
|
| 451 |
+
bias=False)
|
| 452 |
+
|
| 453 |
+
mat_th = th.from_numpy(mat1).type(th.FloatTensor)
|
| 454 |
+
mat_th = mat_th.unsqueeze(1)
|
| 455 |
+
mat_th = mat_th.unsqueeze(3)
|
| 456 |
+
self.conv1.weight = th.nn.Parameter(mat_th)
|
| 457 |
+
|
| 458 |
+
output = expect_kaldi_matrix(instr)
|
| 459 |
+
if output is None:
|
| 460 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 461 |
+
instr, mat = output
|
| 462 |
+
|
| 463 |
+
self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
|
| 464 |
+
self.linear = nn.Linear(self.input_dim, self.hidden_size)
|
| 465 |
+
|
| 466 |
+
self.project.weight = th.nn.Parameter(
|
| 467 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 468 |
+
|
| 469 |
+
output = expect_kaldi_matrix(instr)
|
| 470 |
+
if output is None:
|
| 471 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 472 |
+
instr, mat = output
|
| 473 |
+
self.linear.weight = th.nn.Parameter(
|
| 474 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 475 |
+
|
| 476 |
+
output = expect_kaldi_matrix(instr)
|
| 477 |
+
if output is None:
|
| 478 |
+
raise Exception('UniDeepFsmn format error for parsing matrix')
|
| 479 |
+
instr, mat = output
|
| 480 |
+
mat = np.squeeze(mat)
|
| 481 |
+
self.linear.bias = th.nn.Parameter(
|
| 482 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 483 |
+
|
| 484 |
+
return instr
|
runtime/Lib/site_packages/models/audio/aec/network/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/aec/network/loss.py
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
from .modulation_loss import (GaborSTRFConv, MelScale,
|
| 7 |
+
ModulationDomainLossModule)
|
| 8 |
+
|
| 9 |
+
EPS = 1e-8
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def compute_mask(mixed_spec, clean_spec, mask_type='psmiam', clip=1):
|
| 13 |
+
'''
|
| 14 |
+
stft: (batch, ..., 2) or complex(batch, ...)
|
| 15 |
+
y = x + n
|
| 16 |
+
'''
|
| 17 |
+
if torch.is_complex(mixed_spec):
|
| 18 |
+
yr, yi = mixed_spec.real, mixed_spec.imag
|
| 19 |
+
else:
|
| 20 |
+
yr, yi = mixed_spec[..., 0], mixed_spec[..., 1]
|
| 21 |
+
if torch.is_complex(clean_spec):
|
| 22 |
+
xr, xi = clean_spec.real, clean_spec.imag
|
| 23 |
+
else:
|
| 24 |
+
xr, xi = clean_spec[..., 0], clean_spec[..., 1]
|
| 25 |
+
|
| 26 |
+
if mask_type == 'iam':
|
| 27 |
+
ymag = torch.sqrt(yr**2 + yi**2)
|
| 28 |
+
xmag = torch.sqrt(xr**2 + xi**2)
|
| 29 |
+
iam = xmag / (ymag + EPS)
|
| 30 |
+
return torch.clamp(iam, 0, 1)
|
| 31 |
+
|
| 32 |
+
elif mask_type == 'psm':
|
| 33 |
+
ypow = yr**2 + yi**2
|
| 34 |
+
psm = (xr * yr + xi * yi) / (ypow + EPS)
|
| 35 |
+
return torch.clamp(psm, 0, 1)
|
| 36 |
+
|
| 37 |
+
elif mask_type == 'psmiam':
|
| 38 |
+
ypow = yr**2 + yi**2
|
| 39 |
+
psm = (xr * yr + xi * yi) / (ypow + EPS)
|
| 40 |
+
ymag = torch.sqrt(yr**2 + yi**2)
|
| 41 |
+
xmag = torch.sqrt(xr**2 + xi**2)
|
| 42 |
+
iam = xmag / (ymag + EPS)
|
| 43 |
+
psmiam = psm * iam
|
| 44 |
+
return torch.clamp(psmiam, 0, 1)
|
| 45 |
+
|
| 46 |
+
elif mask_type == 'crm':
|
| 47 |
+
ypow = yr**2 + yi**2
|
| 48 |
+
mr = (xr * yr + xi * yi) / (ypow + EPS)
|
| 49 |
+
mi = (xi * yr - xr * yi) / (ypow + EPS)
|
| 50 |
+
mr = torch.clamp(mr, -clip, clip)
|
| 51 |
+
mi = torch.clamp(mi, -clip, clip)
|
| 52 |
+
return mr, mi
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def energy_vad(spec,
|
| 56 |
+
thdhigh=320 * 600 * 600 * 2,
|
| 57 |
+
thdlow=320 * 300 * 300 * 2,
|
| 58 |
+
int16=True):
|
| 59 |
+
'''
|
| 60 |
+
energy based vad should be accurate enough
|
| 61 |
+
spec: (batch, bins, frames, 2)
|
| 62 |
+
returns (batch, frames)
|
| 63 |
+
'''
|
| 64 |
+
energy = torch.sum(spec[..., 0]**2 + spec[..., 1]**2, dim=1)
|
| 65 |
+
vad = energy > thdhigh
|
| 66 |
+
idx = torch.logical_and(vad == 0, energy > thdlow)
|
| 67 |
+
vad[idx] = 0.5
|
| 68 |
+
return vad
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def modulation_loss_init(n_fft):
|
| 72 |
+
gabor_strf_parameters = torch.load(
|
| 73 |
+
'./network/gabor_strf_parameters.pt')['state_dict']
|
| 74 |
+
gabor_modulation_kernels = GaborSTRFConv(supn=30, supk=30, nkern=60)
|
| 75 |
+
gabor_modulation_kernels.load_state_dict(gabor_strf_parameters)
|
| 76 |
+
|
| 77 |
+
modulation_loss_module = ModulationDomainLossModule(
|
| 78 |
+
gabor_modulation_kernels.eval())
|
| 79 |
+
for param in modulation_loss_module.parameters():
|
| 80 |
+
param.requires_grad = False
|
| 81 |
+
|
| 82 |
+
stft2mel = MelScale(
|
| 83 |
+
n_mels=80, sample_rate=16000, n_stft=n_fft // 2 + 1).cuda()
|
| 84 |
+
|
| 85 |
+
return modulation_loss_module, stft2mel
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def mask_loss_function(
|
| 89 |
+
loss_func='psm_loss',
|
| 90 |
+
loss_type='mse', # ['mse', 'mae', 'comb']
|
| 91 |
+
mask_type='psmiam',
|
| 92 |
+
use_mod_loss=False,
|
| 93 |
+
use_wav2vec_loss=False,
|
| 94 |
+
n_fft=640,
|
| 95 |
+
hop_length=320,
|
| 96 |
+
EPS=1e-8,
|
| 97 |
+
weight=None):
|
| 98 |
+
if weight is not None:
|
| 99 |
+
print(f'Use loss weight: {weight}')
|
| 100 |
+
winlen = n_fft
|
| 101 |
+
window = torch.hamming_window(winlen, periodic=False)
|
| 102 |
+
|
| 103 |
+
def stft(x, return_complex=False):
|
| 104 |
+
# returns [batch, bins, frames, 2]
|
| 105 |
+
return torch.stft(
|
| 106 |
+
x,
|
| 107 |
+
n_fft,
|
| 108 |
+
hop_length,
|
| 109 |
+
winlen,
|
| 110 |
+
window=window.to(x.device),
|
| 111 |
+
center=False,
|
| 112 |
+
return_complex=return_complex)
|
| 113 |
+
|
| 114 |
+
def istft(x, slen):
|
| 115 |
+
return torch.istft(
|
| 116 |
+
x,
|
| 117 |
+
n_fft,
|
| 118 |
+
hop_length,
|
| 119 |
+
winlen,
|
| 120 |
+
window=window.to(x.device),
|
| 121 |
+
center=False,
|
| 122 |
+
length=slen)
|
| 123 |
+
|
| 124 |
+
def mask_loss(targets, masks, nframes):
|
| 125 |
+
''' [Batch, Time, Frequency]
|
| 126 |
+
'''
|
| 127 |
+
with torch.no_grad():
|
| 128 |
+
mask_for_loss = torch.ones_like(targets)
|
| 129 |
+
for idx, num in enumerate(nframes):
|
| 130 |
+
mask_for_loss[idx, num:, :] = 0
|
| 131 |
+
masks = masks * mask_for_loss
|
| 132 |
+
targets = targets * mask_for_loss
|
| 133 |
+
|
| 134 |
+
if weight is None:
|
| 135 |
+
alpha = 1
|
| 136 |
+
else: # for aec ST
|
| 137 |
+
alpha = weight - targets
|
| 138 |
+
|
| 139 |
+
if loss_type == 'mse':
|
| 140 |
+
loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2))
|
| 141 |
+
elif loss_type == 'mae':
|
| 142 |
+
loss = torch.sum(alpha * torch.abs(targets - masks))
|
| 143 |
+
else: # mse(mask), mae(mask) approx 1:2
|
| 144 |
+
loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2)
|
| 145 |
+
+ 0.1 * alpha * torch.abs(targets - masks))
|
| 146 |
+
loss /= torch.sum(nframes)
|
| 147 |
+
return loss
|
| 148 |
+
|
| 149 |
+
def spectrum_loss(targets, spec, nframes):
|
| 150 |
+
''' [Batch, Time, Frequency, 2]
|
| 151 |
+
'''
|
| 152 |
+
with torch.no_grad():
|
| 153 |
+
mask_for_loss = torch.ones_like(targets[..., 0])
|
| 154 |
+
for idx, num in enumerate(nframes):
|
| 155 |
+
mask_for_loss[idx, num:, :] = 0
|
| 156 |
+
xr = spec[..., 0] * mask_for_loss
|
| 157 |
+
xi = spec[..., 1] * mask_for_loss
|
| 158 |
+
yr = targets[..., 0] * mask_for_loss
|
| 159 |
+
yi = targets[..., 1] * mask_for_loss
|
| 160 |
+
xmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2) * mask_for_loss
|
| 161 |
+
ymag = torch.sqrt(targets[..., 0]**2
|
| 162 |
+
+ targets[..., 1]**2) * mask_for_loss
|
| 163 |
+
|
| 164 |
+
loss1 = torch.sum(torch.pow(xr - yr, 2) + torch.pow(xi - yi, 2))
|
| 165 |
+
loss2 = torch.sum(torch.pow(xmag - ymag, 2))
|
| 166 |
+
|
| 167 |
+
loss = (loss1 + loss2) / torch.sum(nframes)
|
| 168 |
+
return loss
|
| 169 |
+
|
| 170 |
+
def sa_loss_dlen(mixed, clean, masks, nframes):
|
| 171 |
+
yspec = stft(mixed).permute([0, 2, 1, 3]) / 32768
|
| 172 |
+
xspec = stft(clean).permute([0, 2, 1, 3]) / 32768
|
| 173 |
+
with torch.no_grad():
|
| 174 |
+
mask_for_loss = torch.ones_like(xspec[..., 0])
|
| 175 |
+
for idx, num in enumerate(nframes):
|
| 176 |
+
mask_for_loss[idx, num:, :] = 0
|
| 177 |
+
emag = ((yspec[..., 0]**2 + yspec[..., 1]**2)**0.15) * (masks**0.3)
|
| 178 |
+
xmag = (xspec[..., 0]**2 + xspec[..., 1]**2)**0.15
|
| 179 |
+
emag = emag * mask_for_loss
|
| 180 |
+
xmag = xmag * mask_for_loss
|
| 181 |
+
|
| 182 |
+
loss = torch.sum(torch.pow(emag - xmag, 2)) / torch.sum(nframes)
|
| 183 |
+
return loss
|
| 184 |
+
|
| 185 |
+
def psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask=None):
|
| 186 |
+
mixed_spec = stft(mixed)
|
| 187 |
+
clean_spec = stft(clean)
|
| 188 |
+
targets = compute_mask(mixed_spec, clean_spec, mask_type)
|
| 189 |
+
# [B, T, F]
|
| 190 |
+
targets = targets.permute(0, 2, 1)
|
| 191 |
+
|
| 192 |
+
loss = mask_loss(targets, masks, nframes)
|
| 193 |
+
|
| 194 |
+
if subtask is not None:
|
| 195 |
+
vadtargets = energy_vad(clean_spec)
|
| 196 |
+
with torch.no_grad():
|
| 197 |
+
mask_for_loss = torch.ones_like(targets[:, :, 0])
|
| 198 |
+
for idx, num in enumerate(nframes):
|
| 199 |
+
mask_for_loss[idx, num:] = 0
|
| 200 |
+
subtask = subtask[:, :, 0] * mask_for_loss
|
| 201 |
+
vadtargets = vadtargets * mask_for_loss
|
| 202 |
+
|
| 203 |
+
loss_vad = F.binary_cross_entropy(subtask, vadtargets)
|
| 204 |
+
return loss + loss_vad
|
| 205 |
+
return loss
|
| 206 |
+
|
| 207 |
+
def modulation_loss(mixed, clean, masks, nframes, subtask=None):
|
| 208 |
+
mixed_spec = stft(mixed, True)
|
| 209 |
+
clean_spec = stft(clean, True)
|
| 210 |
+
enhanced_mag = torch.abs(mixed_spec)
|
| 211 |
+
clean_mag = torch.abs(clean_spec)
|
| 212 |
+
with torch.no_grad():
|
| 213 |
+
mask_for_loss = torch.ones_like(clean_mag)
|
| 214 |
+
for idx, num in enumerate(nframes):
|
| 215 |
+
mask_for_loss[idx, :, num:] = 0
|
| 216 |
+
clean_mag = clean_mag * mask_for_loss
|
| 217 |
+
enhanced_mag = enhanced_mag * mask_for_loss * masks.permute([0, 2, 1])
|
| 218 |
+
|
| 219 |
+
# Covert to log-mel representation
|
| 220 |
+
# (B,T,#mel_channels)
|
| 221 |
+
clean_log_mel = torch.log(
|
| 222 |
+
torch.transpose(stft2mel(clean_mag**2), 2, 1) + 1e-8)
|
| 223 |
+
enhanced_log_mel = torch.log(
|
| 224 |
+
torch.transpose(stft2mel(enhanced_mag**2), 2, 1) + 1e-8)
|
| 225 |
+
|
| 226 |
+
alpha = compute_mask(mixed_spec, clean_spec, mask_type)
|
| 227 |
+
alpha = alpha.permute(0, 2, 1)
|
| 228 |
+
loss = 0.05 * modulation_loss_module(enhanced_log_mel, clean_log_mel,
|
| 229 |
+
alpha)
|
| 230 |
+
loss2 = psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask)
|
| 231 |
+
# print(loss.item(), loss2.item()) #approx 1:4
|
| 232 |
+
loss = loss + loss2
|
| 233 |
+
return loss
|
| 234 |
+
|
| 235 |
+
def wav2vec_loss(mixed, clean, masks, nframes, subtask=None):
|
| 236 |
+
mixed /= 32768
|
| 237 |
+
clean /= 32768
|
| 238 |
+
mixed_spec = stft(mixed)
|
| 239 |
+
with torch.no_grad():
|
| 240 |
+
mask_for_loss = torch.ones_like(masks)
|
| 241 |
+
for idx, num in enumerate(nframes):
|
| 242 |
+
mask_for_loss[idx, num:, :] = 0
|
| 243 |
+
masks_est = masks * mask_for_loss
|
| 244 |
+
|
| 245 |
+
estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
|
| 246 |
+
est_clean = istft(estimate, clean.shape[1])
|
| 247 |
+
loss = wav2vec_loss_module(est_clean, clean)
|
| 248 |
+
return loss
|
| 249 |
+
|
| 250 |
+
def sisdr_loss_dlen(mixed,
|
| 251 |
+
clean,
|
| 252 |
+
masks,
|
| 253 |
+
nframes,
|
| 254 |
+
subtask=None,
|
| 255 |
+
zero_mean=True):
|
| 256 |
+
mixed_spec = stft(mixed)
|
| 257 |
+
with torch.no_grad():
|
| 258 |
+
mask_for_loss = torch.ones_like(masks)
|
| 259 |
+
for idx, num in enumerate(nframes):
|
| 260 |
+
mask_for_loss[idx, num:, :] = 0
|
| 261 |
+
masks_est = masks * mask_for_loss
|
| 262 |
+
|
| 263 |
+
estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
|
| 264 |
+
est_clean = istft(estimate, clean.shape[1])
|
| 265 |
+
flen = min(clean.shape[1], est_clean.shape[1])
|
| 266 |
+
clean = clean[:, :flen]
|
| 267 |
+
est_clean = est_clean[:, :flen]
|
| 268 |
+
|
| 269 |
+
# follow asteroid/losses/sdr.py
|
| 270 |
+
if zero_mean:
|
| 271 |
+
clean = clean - torch.mean(clean, dim=1, keepdim=True)
|
| 272 |
+
est_clean = est_clean - torch.mean(est_clean, dim=1, keepdim=True)
|
| 273 |
+
|
| 274 |
+
dot = torch.sum(est_clean * clean, dim=1, keepdim=True)
|
| 275 |
+
s_clean_energy = torch.sum(clean**2, dim=1, keepdim=True) + EPS
|
| 276 |
+
scaled_clean = dot * clean / s_clean_energy
|
| 277 |
+
e_noise = est_clean - scaled_clean
|
| 278 |
+
|
| 279 |
+
# [batch]
|
| 280 |
+
sisdr = torch.sum(
|
| 281 |
+
scaled_clean**2, dim=1) / (
|
| 282 |
+
torch.sum(e_noise**2, dim=1) + EPS)
|
| 283 |
+
sisdr = -10 * torch.log10(sisdr + EPS)
|
| 284 |
+
loss = sisdr.mean()
|
| 285 |
+
return loss
|
| 286 |
+
|
| 287 |
+
def sisdr_freq_loss_dlen(mixed, clean, masks, nframes, subtask=None):
|
| 288 |
+
mixed_spec = stft(mixed)
|
| 289 |
+
clean_spec = stft(clean)
|
| 290 |
+
with torch.no_grad():
|
| 291 |
+
mask_for_loss = torch.ones_like(masks)
|
| 292 |
+
for idx, num in enumerate(nframes):
|
| 293 |
+
mask_for_loss[idx, num:, :] = 0
|
| 294 |
+
masks_est = masks * mask_for_loss
|
| 295 |
+
|
| 296 |
+
estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
|
| 297 |
+
|
| 298 |
+
dot_real = estimate[..., 0] * clean_spec[..., 0] + \
|
| 299 |
+
estimate[..., 1] * clean_spec[..., 1]
|
| 300 |
+
dot_imag = estimate[..., 0] * clean_spec[..., 1] - \
|
| 301 |
+
estimate[..., 1] * clean_spec[..., 0]
|
| 302 |
+
dot = torch.cat([dot_real.unsqueeze(3), dot_imag.unsqueeze(3)], dim=-1)
|
| 303 |
+
s_clean_energy = clean_spec[..., 0] ** 2 + \
|
| 304 |
+
clean_spec[..., 1] ** 2 + EPS
|
| 305 |
+
scaled_clean = dot * clean_spec / s_clean_energy.unsqueeze(3)
|
| 306 |
+
e_noise = estimate - scaled_clean
|
| 307 |
+
|
| 308 |
+
# [batch]
|
| 309 |
+
scaled_clean_energy = torch.sum(
|
| 310 |
+
scaled_clean[..., 0]**2 + scaled_clean[..., 1]**2, dim=1)
|
| 311 |
+
e_noise_energy = torch.sum(
|
| 312 |
+
e_noise[..., 0]**2 + e_noise[..., 1]**2, dim=1)
|
| 313 |
+
sisdr = torch.sum(
|
| 314 |
+
scaled_clean_energy, dim=1) / (
|
| 315 |
+
torch.sum(e_noise_energy, dim=1) + EPS)
|
| 316 |
+
sisdr = -10 * torch.log10(sisdr + EPS)
|
| 317 |
+
loss = sisdr.mean()
|
| 318 |
+
return loss
|
| 319 |
+
|
| 320 |
+
def crm_loss_dlen(mixed, clean, masks, nframes, subtask=None):
|
| 321 |
+
mixed_spec = stft(mixed).permute([0, 2, 1, 3])
|
| 322 |
+
clean_spec = stft(clean).permute([0, 2, 1, 3])
|
| 323 |
+
mixed_spec = mixed_spec / 32768
|
| 324 |
+
clean_spec = clean_spec / 32768
|
| 325 |
+
tgt_mr, tgt_mi = compute_mask(mixed_spec, clean_spec, mask_type='crm')
|
| 326 |
+
|
| 327 |
+
D = int(masks.shape[2] / 2)
|
| 328 |
+
with torch.no_grad():
|
| 329 |
+
mask_for_loss = torch.ones_like(clean_spec[..., 0])
|
| 330 |
+
for idx, num in enumerate(nframes):
|
| 331 |
+
mask_for_loss[idx, num:, :] = 0
|
| 332 |
+
mr = masks[..., :D] * mask_for_loss
|
| 333 |
+
mi = masks[..., D:] * mask_for_loss
|
| 334 |
+
tgt_mr = tgt_mr * mask_for_loss
|
| 335 |
+
tgt_mi = tgt_mi * mask_for_loss
|
| 336 |
+
|
| 337 |
+
if weight is None:
|
| 338 |
+
alpha = 1
|
| 339 |
+
else:
|
| 340 |
+
alpha = weight - tgt_mr
|
| 341 |
+
# signal approximation
|
| 342 |
+
yr = mixed_spec[..., 0]
|
| 343 |
+
yi = mixed_spec[..., 1]
|
| 344 |
+
loss1 = torch.sum(alpha * torch.pow((mr * yr - mi * yi) - clean_spec[..., 0], 2)) \
|
| 345 |
+
+ torch.sum(alpha * torch.pow((mr * yi + mi * yr) - clean_spec[..., 1], 2))
|
| 346 |
+
# mask approximation
|
| 347 |
+
loss2 = torch.sum(alpha * torch.pow(mr - tgt_mr, 2)) \
|
| 348 |
+
+ torch.sum(alpha * torch.pow(mi - tgt_mi, 2))
|
| 349 |
+
loss = 0.5 * (loss1 + loss2) / torch.sum(nframes)
|
| 350 |
+
return loss
|
| 351 |
+
|
| 352 |
+
def crm_miso_loss_dlen(mixed, clean, masks, nframes):
|
| 353 |
+
return crm_loss_dlen(mixed[..., 0], clean[..., 0], masks, nframes)
|
| 354 |
+
|
| 355 |
+
def mimo_loss_dlen(mixed, clean, masks, nframes):
|
| 356 |
+
chs = mixed.shape[-1]
|
| 357 |
+
D = masks.shape[2] // chs
|
| 358 |
+
loss = psm_vad_loss_dlen(mixed[..., 0], clean[..., 0], masks[..., :D],
|
| 359 |
+
nframes)
|
| 360 |
+
for ch in range(1, chs):
|
| 361 |
+
loss1 = psm_vad_loss_dlen(mixed[..., ch], clean[..., ch],
|
| 362 |
+
masks[..., ch * D:ch * D + D], nframes)
|
| 363 |
+
loss = loss + loss1
|
| 364 |
+
return loss / chs
|
| 365 |
+
|
| 366 |
+
def spec_loss_dlen(mixed, clean, spec, nframes):
|
| 367 |
+
clean_spec = stft(clean).permute([0, 2, 1, 3])
|
| 368 |
+
clean_spec = clean_spec / 32768
|
| 369 |
+
|
| 370 |
+
D = spec.shape[2] // 2
|
| 371 |
+
spec_est = torch.cat([spec[..., :D, None], spec[..., D:, None]],
|
| 372 |
+
dim=-1)
|
| 373 |
+
loss = spectrum_loss(clean_spec, spec_est, nframes)
|
| 374 |
+
return loss
|
| 375 |
+
|
| 376 |
+
if loss_func == 'psm_vad_loss_dlen':
|
| 377 |
+
return psm_vad_loss_dlen
|
| 378 |
+
elif loss_func == 'sisdr_loss_dlen':
|
| 379 |
+
return sisdr_loss_dlen
|
| 380 |
+
elif loss_func == 'sisdr_freq_loss_dlen':
|
| 381 |
+
return sisdr_freq_loss_dlen
|
| 382 |
+
elif loss_func == 'crm_loss_dlen':
|
| 383 |
+
return crm_loss_dlen
|
| 384 |
+
elif loss_func == 'modulation_loss':
|
| 385 |
+
return modulation_loss
|
| 386 |
+
elif loss_func == 'wav2vec_loss':
|
| 387 |
+
return wav2vec_loss
|
| 388 |
+
elif loss_func == 'mimo_loss_dlen':
|
| 389 |
+
return mimo_loss_dlen
|
| 390 |
+
elif loss_func == 'spec_loss_dlen':
|
| 391 |
+
return spec_loss_dlen
|
| 392 |
+
elif loss_func == 'sa_loss_dlen':
|
| 393 |
+
return sa_loss_dlen
|
| 394 |
+
else:
|
| 395 |
+
print('error loss func')
|
| 396 |
+
return None
|
runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from torchaudio.transforms import MelScale
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class ModulationDomainLossModule(torch.nn.Module):
|
| 12 |
+
"""Modulation-domain loss function developed in [1] for supervised speech enhancement
|
| 13 |
+
|
| 14 |
+
In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram
|
| 15 |
+
as the input spectrogram representation.
|
| 16 |
+
Specific parameter details are in the paper and in the example below
|
| 17 |
+
|
| 18 |
+
Parameters
|
| 19 |
+
----------
|
| 20 |
+
modulation_kernels: nn.Module
|
| 21 |
+
Differentiable module that transforms a spectrogram representation to the modulation domain
|
| 22 |
+
|
| 23 |
+
modulation_domain = modulation_kernels(input_tf_representation)
|
| 24 |
+
Input Spectrogram representation (B, T, F) ---> |(M) modulation_kernels|--->Modulation Domain(B, M, T', F')
|
| 25 |
+
|
| 26 |
+
norm: boolean
|
| 27 |
+
Normalizes the modulation domain representation to be 0 mean across time
|
| 28 |
+
|
| 29 |
+
[1] T. Vuong, Y. Xia, and R. M. Stern, “A modulation-domain lossfor neural-network-based real-time
|
| 30 |
+
speech enhancement”
|
| 31 |
+
Accepted ICASSP 2021, https://arxiv.org/abs/2102.07330
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def __init__(self, modulation_kernels, norm=True):
|
| 37 |
+
super(ModulationDomainLossModule, self).__init__()
|
| 38 |
+
|
| 39 |
+
self.modulation_kernels = modulation_kernels
|
| 40 |
+
self.mse = nn.MSELoss(reduce=False)
|
| 41 |
+
self.norm = norm
|
| 42 |
+
|
| 43 |
+
def forward(self, enhanced_spect, clean_spect, weight=None):
|
| 44 |
+
"""Calculate modulation-domain loss
|
| 45 |
+
Args:
|
| 46 |
+
enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels).
|
| 47 |
+
clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels).
|
| 48 |
+
Returns:
|
| 49 |
+
Tensor: Modulation-domain loss value.
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
clean_mod = self.modulation_kernels(clean_spect)
|
| 53 |
+
enhanced_mod = self.modulation_kernels(enhanced_spect)
|
| 54 |
+
|
| 55 |
+
if self.norm:
|
| 56 |
+
mean_clean_mod = torch.mean(clean_mod, dim=2)
|
| 57 |
+
mean_enhanced_mod = torch.mean(enhanced_mod, dim=2)
|
| 58 |
+
|
| 59 |
+
clean_mod = clean_mod - mean_clean_mod.unsqueeze(2)
|
| 60 |
+
enhanced_mod = enhanced_mod - mean_enhanced_mod.unsqueeze(2)
|
| 61 |
+
|
| 62 |
+
if weight is None:
|
| 63 |
+
alpha = 1
|
| 64 |
+
else: # TF-mask weight
|
| 65 |
+
alpha = 1 + torch.sum(weight, dim=-1, keepdim=True).unsqueeze(1)
|
| 66 |
+
mod_mse_loss = self.mse(enhanced_mod, clean_mod) * alpha
|
| 67 |
+
mod_mse_loss = torch.mean(
|
| 68 |
+
torch.sum(mod_mse_loss, dim=(1, 2, 3))
|
| 69 |
+
/ torch.sum(clean_mod**2, dim=(1, 2, 3)))
|
| 70 |
+
|
| 71 |
+
return mod_mse_loss
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class ModulationDomainNCCLossModule(torch.nn.Module):
|
| 75 |
+
"""Modulation-domain loss function developed in [1] for supervised speech enhancement
|
| 76 |
+
|
| 77 |
+
# Speech Intelligibility Prediction Using Spectro-Temporal Modulation Analysis - based off of this
|
| 78 |
+
|
| 79 |
+
In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram
|
| 80 |
+
as the input spectrogram representation.
|
| 81 |
+
Specific parameter details are in the paper and in the example below
|
| 82 |
+
|
| 83 |
+
Parameters
|
| 84 |
+
----------
|
| 85 |
+
modulation_kernels: nn.Module
|
| 86 |
+
Differentiable module that transforms a spectrogram representation to the modulation domain
|
| 87 |
+
|
| 88 |
+
modulation_domain = modulation_kernels(input_tf_representation)
|
| 89 |
+
Input Spectrogram representation(B, T, F) --- (M) modulation_kernels---> Modulation Domain(B, M, T', F')
|
| 90 |
+
|
| 91 |
+
[1]
|
| 92 |
+
|
| 93 |
+
"""
|
| 94 |
+
|
| 95 |
+
def __init__(self, modulation_kernels):
|
| 96 |
+
super(ModulationDomainNCCLossModule, self).__init__()
|
| 97 |
+
|
| 98 |
+
self.modulation_kernels = modulation_kernels
|
| 99 |
+
self.mse = nn.MSELoss(reduce=False)
|
| 100 |
+
|
| 101 |
+
def forward(self, enhanced_spect, clean_spect):
|
| 102 |
+
"""Calculate modulation-domain loss
|
| 103 |
+
Args:
|
| 104 |
+
enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels).
|
| 105 |
+
clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels).
|
| 106 |
+
Returns:
|
| 107 |
+
Tensor: Modulation-domain loss value.
|
| 108 |
+
"""
|
| 109 |
+
|
| 110 |
+
clean_mod = self.modulation_kernels(clean_spect)
|
| 111 |
+
enhanced_mod = self.modulation_kernels(enhanced_spect)
|
| 112 |
+
mean_clean_mod = torch.mean(clean_mod, dim=2)
|
| 113 |
+
mean_enhanced_mod = torch.mean(enhanced_mod, dim=2)
|
| 114 |
+
|
| 115 |
+
normalized_clean = clean_mod - mean_clean_mod.unsqueeze(2)
|
| 116 |
+
normalized_enhanced = enhanced_mod - mean_enhanced_mod.unsqueeze(2)
|
| 117 |
+
|
| 118 |
+
inner_product = torch.sum(
|
| 119 |
+
normalized_clean * normalized_enhanced, dim=2)
|
| 120 |
+
normalized_denom = (torch.sum(
|
| 121 |
+
normalized_clean * normalized_clean, dim=2))**.5 * (torch.sum(
|
| 122 |
+
normalized_enhanced * normalized_enhanced, dim=2))**.5
|
| 123 |
+
|
| 124 |
+
ncc = inner_product / normalized_denom
|
| 125 |
+
mod_mse_loss = torch.mean((ncc - 1.0)**2)
|
| 126 |
+
|
| 127 |
+
return mod_mse_loss
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class GaborSTRFConv(nn.Module):
|
| 131 |
+
"""Gabor-STRF-based cross-correlation kernel."""
|
| 132 |
+
|
| 133 |
+
def __init__(self,
|
| 134 |
+
supn,
|
| 135 |
+
supk,
|
| 136 |
+
nkern,
|
| 137 |
+
rates=None,
|
| 138 |
+
scales=None,
|
| 139 |
+
norm_strf=True,
|
| 140 |
+
real_only=False):
|
| 141 |
+
"""Instantiate a Gabor-based STRF convolution layer.
|
| 142 |
+
Parameters
|
| 143 |
+
----------
|
| 144 |
+
supn: int
|
| 145 |
+
Time support in number of frames. Also the window length.
|
| 146 |
+
supk: int
|
| 147 |
+
Frequency support in number of channels. Also the window length.
|
| 148 |
+
nkern: int
|
| 149 |
+
Number of kernels, each with a learnable rate and scale.
|
| 150 |
+
rates: list of float, None
|
| 151 |
+
Initial values for temporal modulation.
|
| 152 |
+
scales: list of float, None
|
| 153 |
+
Initial values for spectral modulation.
|
| 154 |
+
norm_strf: Boolean
|
| 155 |
+
Normalize STRF kernels to be unit length
|
| 156 |
+
real_only: Boolean
|
| 157 |
+
If True, nkern REAL gabor-STRF kernels
|
| 158 |
+
If False, nkern//2 REAL and nkern//2 IMAGINARY gabor-STRF kernels
|
| 159 |
+
"""
|
| 160 |
+
super(GaborSTRFConv, self).__init__()
|
| 161 |
+
self.numN = supn
|
| 162 |
+
self.numK = supk
|
| 163 |
+
self.numKern = nkern
|
| 164 |
+
self.real_only = real_only
|
| 165 |
+
self.norm_strf = norm_strf
|
| 166 |
+
|
| 167 |
+
if not real_only:
|
| 168 |
+
nkern = nkern // 2
|
| 169 |
+
|
| 170 |
+
if supk % 2 == 0: # force odd number
|
| 171 |
+
supk += 1
|
| 172 |
+
self.supk = torch.arange(supk, dtype=torch.float32)
|
| 173 |
+
if supn % 2 == 0: # force odd number
|
| 174 |
+
supn += 1
|
| 175 |
+
self.supn = torch.arange(supn, dtype=self.supk.dtype)
|
| 176 |
+
self.padding = (supn // 2, supk // 2)
|
| 177 |
+
# Set up learnable parameters
|
| 178 |
+
# for param in (rates, scales):
|
| 179 |
+
# assert (not param) or len(param) == nkern
|
| 180 |
+
if not rates:
|
| 181 |
+
|
| 182 |
+
rates = torch.rand(nkern) * math.pi / 2.0
|
| 183 |
+
|
| 184 |
+
if not scales:
|
| 185 |
+
|
| 186 |
+
scales = (torch.rand(nkern) * 2.0 - 1.0) * math.pi / 2.0
|
| 187 |
+
|
| 188 |
+
self.rates_ = nn.Parameter(torch.Tensor(rates))
|
| 189 |
+
self.scales_ = nn.Parameter(torch.Tensor(scales))
|
| 190 |
+
|
| 191 |
+
def strfs(self):
|
| 192 |
+
"""Make STRFs using the current parameters."""
|
| 193 |
+
|
| 194 |
+
if self.supn.device != self.rates_.device: # for first run
|
| 195 |
+
self.supn = self.supn.to(self.rates_.device)
|
| 196 |
+
self.supk = self.supk.to(self.rates_.device)
|
| 197 |
+
n0, k0 = self.padding
|
| 198 |
+
|
| 199 |
+
nwind = .5 - .5 * \
|
| 200 |
+
torch.cos(2 * math.pi * (self.supn + 1) / (len(self.supn) + 1))
|
| 201 |
+
kwind = .5 - .5 * \
|
| 202 |
+
torch.cos(2 * math.pi * (self.supk + 1) / (len(self.supk) + 1))
|
| 203 |
+
|
| 204 |
+
new_wind = torch.matmul((nwind).unsqueeze(-1), (kwind).unsqueeze(0))
|
| 205 |
+
|
| 206 |
+
n_n_0 = self.supn - n0
|
| 207 |
+
k_k_0 = self.supk - k0
|
| 208 |
+
n_mult = torch.matmul(
|
| 209 |
+
n_n_0.unsqueeze(1),
|
| 210 |
+
torch.ones((1, len(self.supk))).type(torch.FloatTensor).to(
|
| 211 |
+
self.rates_.device))
|
| 212 |
+
k_mult = torch.matmul(
|
| 213 |
+
torch.ones((len(self.supn),
|
| 214 |
+
1)).type(torch.FloatTensor).to(self.rates_.device),
|
| 215 |
+
k_k_0.unsqueeze(0))
|
| 216 |
+
|
| 217 |
+
inside = self.rates_.unsqueeze(1).unsqueeze(
|
| 218 |
+
1) * n_mult + self.scales_.unsqueeze(1).unsqueeze(1) * k_mult
|
| 219 |
+
real_strf = torch.cos(inside) * new_wind.unsqueeze(0)
|
| 220 |
+
|
| 221 |
+
if self.real_only:
|
| 222 |
+
final_strf = real_strf
|
| 223 |
+
|
| 224 |
+
else:
|
| 225 |
+
imag_strf = torch.sin(inside) * new_wind.unsqueeze(0)
|
| 226 |
+
final_strf = torch.cat([real_strf, imag_strf], dim=0)
|
| 227 |
+
|
| 228 |
+
if self.norm_strf:
|
| 229 |
+
final_strf = final_strf / (torch.sum(
|
| 230 |
+
final_strf**2, dim=(1, 2)).unsqueeze(1).unsqueeze(2))**.5
|
| 231 |
+
|
| 232 |
+
return final_strf
|
| 233 |
+
|
| 234 |
+
def forward(self, sigspec):
|
| 235 |
+
"""Forward pass a batch of (real) spectra [Batch x Time x Frequency]."""
|
| 236 |
+
if len(sigspec.shape) == 2: # expand batch dimension if single eg
|
| 237 |
+
sigspec = sigspec.unsqueeze(0)
|
| 238 |
+
strfs = self.strfs().unsqueeze(1).type_as(sigspec)
|
| 239 |
+
out = F.conv2d(sigspec.unsqueeze(1), strfs, padding=self.padding)
|
| 240 |
+
return out
|
| 241 |
+
|
| 242 |
+
def __repr__(self):
|
| 243 |
+
"""Gabor filter"""
|
| 244 |
+
report = """
|
| 245 |
+
+++++ Gabor Filter Kernels [{}], supn[{}], supk[{}] real only [{}] norm strf [{}] +++++
|
| 246 |
+
|
| 247 |
+
""".format(self.numKern, self.numN, self.numK, self.real_only,
|
| 248 |
+
self.norm_strf)
|
| 249 |
+
|
| 250 |
+
return report
|
runtime/Lib/site_packages/models/audio/aec/network/se_net.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
from modelscope.models.audio.aec.layers.activations import (RectifiedLinear,
|
| 8 |
+
Sigmoid)
|
| 9 |
+
from modelscope.models.audio.aec.layers.affine_transform import AffineTransform
|
| 10 |
+
from modelscope.models.audio.aec.layers.deep_fsmn import DeepFsmn
|
| 11 |
+
from modelscope.models.audio.aec.layers.uni_deep_fsmn import (Conv2d,
|
| 12 |
+
UniDeepFsmn)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class MaskNet(nn.Module):
|
| 16 |
+
|
| 17 |
+
def __init__(self,
|
| 18 |
+
indim,
|
| 19 |
+
outdim,
|
| 20 |
+
layers=9,
|
| 21 |
+
hidden_dim=128,
|
| 22 |
+
hidden_dim2=None,
|
| 23 |
+
lorder=20,
|
| 24 |
+
rorder=0,
|
| 25 |
+
dilation=1,
|
| 26 |
+
layer_norm=False,
|
| 27 |
+
dropout=0,
|
| 28 |
+
crm=False,
|
| 29 |
+
vad=False,
|
| 30 |
+
linearout=False):
|
| 31 |
+
super(MaskNet, self).__init__()
|
| 32 |
+
|
| 33 |
+
self.linear1 = AffineTransform(indim, hidden_dim)
|
| 34 |
+
self.relu = RectifiedLinear(hidden_dim, hidden_dim)
|
| 35 |
+
if hidden_dim2 is None:
|
| 36 |
+
hidden_dim2 = hidden_dim
|
| 37 |
+
|
| 38 |
+
if rorder == 0:
|
| 39 |
+
repeats = [
|
| 40 |
+
UniDeepFsmn(
|
| 41 |
+
hidden_dim,
|
| 42 |
+
hidden_dim,
|
| 43 |
+
lorder,
|
| 44 |
+
hidden_dim2,
|
| 45 |
+
dilation=dilation,
|
| 46 |
+
layer_norm=layer_norm,
|
| 47 |
+
dropout=dropout) for i in range(layers)
|
| 48 |
+
]
|
| 49 |
+
else:
|
| 50 |
+
repeats = [
|
| 51 |
+
DeepFsmn(
|
| 52 |
+
hidden_dim,
|
| 53 |
+
hidden_dim,
|
| 54 |
+
lorder,
|
| 55 |
+
rorder,
|
| 56 |
+
hidden_dim2,
|
| 57 |
+
layer_norm=layer_norm,
|
| 58 |
+
dropout=dropout) for i in range(layers)
|
| 59 |
+
]
|
| 60 |
+
self.deepfsmn = nn.Sequential(*repeats)
|
| 61 |
+
|
| 62 |
+
self.linear2 = AffineTransform(hidden_dim, outdim)
|
| 63 |
+
|
| 64 |
+
self.crm = crm
|
| 65 |
+
if self.crm:
|
| 66 |
+
self.sig = nn.Tanh()
|
| 67 |
+
else:
|
| 68 |
+
self.sig = Sigmoid(outdim, outdim)
|
| 69 |
+
|
| 70 |
+
self.vad = vad
|
| 71 |
+
if self.vad:
|
| 72 |
+
self.linear3 = AffineTransform(hidden_dim, 1)
|
| 73 |
+
|
| 74 |
+
self.layers = layers
|
| 75 |
+
self.linearout = linearout
|
| 76 |
+
if self.linearout and self.vad:
|
| 77 |
+
print('Warning: not supported nnet')
|
| 78 |
+
|
| 79 |
+
def forward(self, feat, ctl=None):
|
| 80 |
+
x1 = self.linear1(feat)
|
| 81 |
+
x2 = self.relu(x1)
|
| 82 |
+
if ctl is not None:
|
| 83 |
+
ctl = min(ctl, self.layers - 1)
|
| 84 |
+
for i in range(ctl):
|
| 85 |
+
x2 = self.deepfsmn[i](x2)
|
| 86 |
+
mask = self.sig(self.linear2(x2))
|
| 87 |
+
if self.vad:
|
| 88 |
+
vad = torch.sigmoid(self.linear3(x2))
|
| 89 |
+
return mask, vad
|
| 90 |
+
else:
|
| 91 |
+
return mask
|
| 92 |
+
x3 = self.deepfsmn(x2)
|
| 93 |
+
if self.linearout:
|
| 94 |
+
return self.linear2(x3)
|
| 95 |
+
mask = self.sig(self.linear2(x3))
|
| 96 |
+
if self.vad:
|
| 97 |
+
vad = torch.sigmoid(self.linear3(x3))
|
| 98 |
+
return mask, vad
|
| 99 |
+
else:
|
| 100 |
+
return mask
|
| 101 |
+
|
| 102 |
+
def to_kaldi_nnet(self):
|
| 103 |
+
re_str = ''
|
| 104 |
+
re_str += '<Nnet>\n'
|
| 105 |
+
re_str += self.linear1.to_kaldi_nnet()
|
| 106 |
+
re_str += self.relu.to_kaldi_nnet()
|
| 107 |
+
for dfsmn in self.deepfsmn:
|
| 108 |
+
re_str += dfsmn.to_kaldi_nnet()
|
| 109 |
+
re_str += self.linear2.to_kaldi_nnet()
|
| 110 |
+
re_str += self.sig.to_kaldi_nnet()
|
| 111 |
+
re_str += '</Nnet>\n'
|
| 112 |
+
|
| 113 |
+
return re_str
|
| 114 |
+
|
| 115 |
+
def to_raw_nnet(self, fid):
|
| 116 |
+
self.linear1.to_raw_nnet(fid)
|
| 117 |
+
for dfsmn in self.deepfsmn:
|
| 118 |
+
dfsmn.to_raw_nnet(fid)
|
| 119 |
+
self.linear2.to_raw_nnet(fid)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class StageNet(nn.Module):
|
| 123 |
+
|
| 124 |
+
def __init__(self,
|
| 125 |
+
indim,
|
| 126 |
+
outdim,
|
| 127 |
+
layers=9,
|
| 128 |
+
layers2=6,
|
| 129 |
+
hidden_dim=128,
|
| 130 |
+
lorder=20,
|
| 131 |
+
rorder=0,
|
| 132 |
+
layer_norm=False,
|
| 133 |
+
dropout=0,
|
| 134 |
+
crm=False,
|
| 135 |
+
vad=False,
|
| 136 |
+
linearout=False):
|
| 137 |
+
super(StageNet, self).__init__()
|
| 138 |
+
|
| 139 |
+
self.stage1 = nn.ModuleList()
|
| 140 |
+
self.stage2 = nn.ModuleList()
|
| 141 |
+
layer = nn.Sequential(nn.Linear(indim, hidden_dim), nn.ReLU())
|
| 142 |
+
self.stage1.append(layer)
|
| 143 |
+
for i in range(layers):
|
| 144 |
+
layer = UniDeepFsmn(
|
| 145 |
+
hidden_dim,
|
| 146 |
+
hidden_dim,
|
| 147 |
+
lorder,
|
| 148 |
+
hidden_dim,
|
| 149 |
+
layer_norm=layer_norm,
|
| 150 |
+
dropout=dropout)
|
| 151 |
+
self.stage1.append(layer)
|
| 152 |
+
layer = nn.Sequential(nn.Linear(hidden_dim, 321), nn.Sigmoid())
|
| 153 |
+
self.stage1.append(layer)
|
| 154 |
+
# stage2
|
| 155 |
+
layer = nn.Sequential(nn.Linear(321 + indim, hidden_dim), nn.ReLU())
|
| 156 |
+
self.stage2.append(layer)
|
| 157 |
+
for i in range(layers2):
|
| 158 |
+
layer = UniDeepFsmn(
|
| 159 |
+
hidden_dim,
|
| 160 |
+
hidden_dim,
|
| 161 |
+
lorder,
|
| 162 |
+
hidden_dim,
|
| 163 |
+
layer_norm=layer_norm,
|
| 164 |
+
dropout=dropout)
|
| 165 |
+
self.stage2.append(layer)
|
| 166 |
+
layer = nn.Sequential(
|
| 167 |
+
nn.Linear(hidden_dim, outdim),
|
| 168 |
+
nn.Sigmoid() if not crm else nn.Tanh())
|
| 169 |
+
self.stage2.append(layer)
|
| 170 |
+
self.crm = crm
|
| 171 |
+
self.vad = vad
|
| 172 |
+
self.linearout = linearout
|
| 173 |
+
self.window = torch.hamming_window(640, periodic=False).cuda()
|
| 174 |
+
self.freezed = False
|
| 175 |
+
|
| 176 |
+
def freeze(self):
|
| 177 |
+
if not self.freezed:
|
| 178 |
+
for param in self.stage1.parameters():
|
| 179 |
+
param.requires_grad = False
|
| 180 |
+
self.freezed = True
|
| 181 |
+
print('freezed stage1')
|
| 182 |
+
|
| 183 |
+
def forward(self, feat, mixture, ctl=None):
|
| 184 |
+
if ctl == 'off':
|
| 185 |
+
x = feat
|
| 186 |
+
for i in range(len(self.stage1)):
|
| 187 |
+
x = self.stage1[i](x)
|
| 188 |
+
return x
|
| 189 |
+
else:
|
| 190 |
+
self.freeze()
|
| 191 |
+
x = feat
|
| 192 |
+
for i in range(len(self.stage1)):
|
| 193 |
+
x = self.stage1[i](x)
|
| 194 |
+
|
| 195 |
+
spec = torch.stft(
|
| 196 |
+
mixture / 32768,
|
| 197 |
+
640,
|
| 198 |
+
320,
|
| 199 |
+
640,
|
| 200 |
+
self.window,
|
| 201 |
+
center=False,
|
| 202 |
+
return_complex=True)
|
| 203 |
+
spec = torch.view_as_real(spec).permute([0, 2, 1, 3])
|
| 204 |
+
specmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2)
|
| 205 |
+
est = x * specmag
|
| 206 |
+
y = torch.cat([est, feat], dim=-1)
|
| 207 |
+
for i in range(len(self.stage2)):
|
| 208 |
+
y = self.stage2[i](y)
|
| 209 |
+
return y
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
class Unet(nn.Module):
|
| 213 |
+
|
| 214 |
+
def __init__(self,
|
| 215 |
+
indim,
|
| 216 |
+
outdim,
|
| 217 |
+
layers=9,
|
| 218 |
+
dims=[256] * 4,
|
| 219 |
+
lorder=20,
|
| 220 |
+
rorder=0,
|
| 221 |
+
dilation=1,
|
| 222 |
+
layer_norm=False,
|
| 223 |
+
dropout=0,
|
| 224 |
+
crm=False,
|
| 225 |
+
vad=False,
|
| 226 |
+
linearout=False):
|
| 227 |
+
super(Unet, self).__init__()
|
| 228 |
+
|
| 229 |
+
self.linear1 = AffineTransform(indim, dims[0])
|
| 230 |
+
self.relu = RectifiedLinear(dims[0], dims[0])
|
| 231 |
+
|
| 232 |
+
self.encoder = nn.ModuleList()
|
| 233 |
+
self.decoder = nn.ModuleList()
|
| 234 |
+
for i in range(len(dims) - 1):
|
| 235 |
+
layer = nn.Sequential(
|
| 236 |
+
nn.Linear(dims[i], dims[i + 1]), nn.ReLU(),
|
| 237 |
+
nn.Linear(dims[i + 1], dims[i + 1], bias=False),
|
| 238 |
+
Conv2d(
|
| 239 |
+
dims[i + 1],
|
| 240 |
+
dims[i + 1],
|
| 241 |
+
lorder,
|
| 242 |
+
groups=dims[i + 1],
|
| 243 |
+
skip_connect=True))
|
| 244 |
+
self.encoder.append(layer)
|
| 245 |
+
for i in range(len(dims) - 1, 0, -1):
|
| 246 |
+
layer = nn.Sequential(
|
| 247 |
+
nn.Linear(dims[i] * 2, dims[i - 1]), nn.ReLU(),
|
| 248 |
+
nn.Linear(dims[i - 1], dims[i - 1], bias=False),
|
| 249 |
+
Conv2d(
|
| 250 |
+
dims[i - 1],
|
| 251 |
+
dims[i - 1],
|
| 252 |
+
lorder,
|
| 253 |
+
groups=dims[i - 1],
|
| 254 |
+
skip_connect=True))
|
| 255 |
+
self.decoder.append(layer)
|
| 256 |
+
self.tf = nn.ModuleList()
|
| 257 |
+
for i in range(layers - 2 * (len(dims) - 1)):
|
| 258 |
+
layer = nn.Sequential(
|
| 259 |
+
nn.Linear(dims[-1], dims[-1]), nn.ReLU(),
|
| 260 |
+
nn.Linear(dims[-1], dims[-1], bias=False),
|
| 261 |
+
Conv2d(
|
| 262 |
+
dims[-1],
|
| 263 |
+
dims[-1],
|
| 264 |
+
lorder,
|
| 265 |
+
groups=dims[-1],
|
| 266 |
+
skip_connect=True))
|
| 267 |
+
self.tf.append(layer)
|
| 268 |
+
|
| 269 |
+
self.linear2 = AffineTransform(dims[0], outdim)
|
| 270 |
+
self.crm = crm
|
| 271 |
+
self.act = nn.Tanh() if self.crm else nn.Sigmoid()
|
| 272 |
+
self.vad = False
|
| 273 |
+
self.layers = layers
|
| 274 |
+
self.linearout = linearout
|
| 275 |
+
|
| 276 |
+
def forward(self, x, ctl=None):
|
| 277 |
+
x = self.linear1(x)
|
| 278 |
+
x = self.relu(x)
|
| 279 |
+
|
| 280 |
+
encoder_out = []
|
| 281 |
+
for i in range(len(self.encoder)):
|
| 282 |
+
x = self.encoder[i](x)
|
| 283 |
+
encoder_out.append(x)
|
| 284 |
+
for i in range(len(self.tf)):
|
| 285 |
+
x = self.tf[i](x)
|
| 286 |
+
for i in range(len(self.decoder)):
|
| 287 |
+
x = torch.cat([x, encoder_out[-1 - i]], dim=-1)
|
| 288 |
+
x = self.decoder[i](x)
|
| 289 |
+
|
| 290 |
+
x = self.linear2(x)
|
| 291 |
+
if self.linearout:
|
| 292 |
+
return x
|
| 293 |
+
return self.act(x)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
class BranchNet(nn.Module):
|
| 297 |
+
|
| 298 |
+
def __init__(self,
|
| 299 |
+
indim,
|
| 300 |
+
outdim,
|
| 301 |
+
layers=9,
|
| 302 |
+
hidden_dim=256,
|
| 303 |
+
lorder=20,
|
| 304 |
+
rorder=0,
|
| 305 |
+
dilation=1,
|
| 306 |
+
layer_norm=False,
|
| 307 |
+
dropout=0,
|
| 308 |
+
crm=False,
|
| 309 |
+
vad=False,
|
| 310 |
+
linearout=False):
|
| 311 |
+
super(BranchNet, self).__init__()
|
| 312 |
+
|
| 313 |
+
self.linear1 = AffineTransform(indim, hidden_dim)
|
| 314 |
+
self.relu = RectifiedLinear(hidden_dim, hidden_dim)
|
| 315 |
+
|
| 316 |
+
self.convs = nn.ModuleList()
|
| 317 |
+
self.deepfsmn = nn.ModuleList()
|
| 318 |
+
self.FREQ = nn.ModuleList()
|
| 319 |
+
self.TIME = nn.ModuleList()
|
| 320 |
+
self.br1 = nn.ModuleList()
|
| 321 |
+
self.br2 = nn.ModuleList()
|
| 322 |
+
for i in range(layers):
|
| 323 |
+
'''
|
| 324 |
+
layer = nn.Sequential(
|
| 325 |
+
nn.Linear(hidden_dim, hidden_dim),
|
| 326 |
+
nn.ReLU(),
|
| 327 |
+
nn.Linear(hidden_dim, hidden_dim, bias=False),
|
| 328 |
+
Conv2d(hidden_dim, hidden_dim, lorder,
|
| 329 |
+
groups=hidden_dim, skip_connect=True)
|
| 330 |
+
)
|
| 331 |
+
self.deepfsmn.append(layer)
|
| 332 |
+
'''
|
| 333 |
+
layer = nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.ReLU())
|
| 334 |
+
self.FREQ.append(layer)
|
| 335 |
+
'''
|
| 336 |
+
layer = nn.GRU(hidden_dim, hidden_dim,
|
| 337 |
+
batch_first=True,
|
| 338 |
+
bidirectional=False)
|
| 339 |
+
self.TIME.append(layer)
|
| 340 |
+
|
| 341 |
+
layer = nn.Sequential(
|
| 342 |
+
nn.Linear(hidden_dim, hidden_dim//2, bias=False),
|
| 343 |
+
Conv2d(hidden_dim//2, hidden_dim//2, lorder,
|
| 344 |
+
groups=hidden_dim//2, skip_connect=True)
|
| 345 |
+
)
|
| 346 |
+
self.br1.append(layer)
|
| 347 |
+
layer = nn.GRU(hidden_dim, hidden_dim//2,
|
| 348 |
+
batch_first=True,
|
| 349 |
+
bidirectional=False)
|
| 350 |
+
self.br2.append(layer)
|
| 351 |
+
'''
|
| 352 |
+
|
| 353 |
+
self.linear2 = AffineTransform(hidden_dim, outdim)
|
| 354 |
+
self.crm = crm
|
| 355 |
+
self.act = nn.Tanh() if self.crm else nn.Sigmoid()
|
| 356 |
+
self.vad = False
|
| 357 |
+
self.layers = layers
|
| 358 |
+
self.linearout = linearout
|
| 359 |
+
|
| 360 |
+
def forward(self, x, ctl=None):
|
| 361 |
+
return self.forward_branch(x)
|
| 362 |
+
|
| 363 |
+
def forward_sepconv(self, x):
|
| 364 |
+
x = torch.unsqueeze(x, 1)
|
| 365 |
+
for i in range(len(self.convs)):
|
| 366 |
+
x = self.convs[i](x)
|
| 367 |
+
x = F.relu(x)
|
| 368 |
+
B, C, H, W = x.shape
|
| 369 |
+
x = x.permute(0, 2, 1, 3)
|
| 370 |
+
x = torch.reshape(x, [B, H, C * W])
|
| 371 |
+
x = self.linear1(x)
|
| 372 |
+
x = self.relu(x)
|
| 373 |
+
for i in range(self.layers):
|
| 374 |
+
x = self.deepfsmn[i](x) + x
|
| 375 |
+
x = self.linear2(x)
|
| 376 |
+
return self.act(x)
|
| 377 |
+
|
| 378 |
+
def forward_branch(self, x):
|
| 379 |
+
x = self.linear1(x)
|
| 380 |
+
x = self.relu(x)
|
| 381 |
+
for i in range(self.layers):
|
| 382 |
+
z = self.FREQ[i](x)
|
| 383 |
+
x = z + x
|
| 384 |
+
x = self.linear2(x)
|
| 385 |
+
if self.linearout:
|
| 386 |
+
return x
|
| 387 |
+
return self.act(x)
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
class TACNet(nn.Module):
|
| 391 |
+
''' transform average concatenate for ad hoc dr
|
| 392 |
+
'''
|
| 393 |
+
|
| 394 |
+
def __init__(self,
|
| 395 |
+
indim,
|
| 396 |
+
outdim,
|
| 397 |
+
layers=9,
|
| 398 |
+
hidden_dim=128,
|
| 399 |
+
lorder=20,
|
| 400 |
+
rorder=0,
|
| 401 |
+
crm=False,
|
| 402 |
+
vad=False,
|
| 403 |
+
linearout=False):
|
| 404 |
+
super(TACNet, self).__init__()
|
| 405 |
+
|
| 406 |
+
self.linear1 = AffineTransform(indim, hidden_dim)
|
| 407 |
+
self.relu = RectifiedLinear(hidden_dim, hidden_dim)
|
| 408 |
+
|
| 409 |
+
if rorder == 0:
|
| 410 |
+
repeats = [
|
| 411 |
+
UniDeepFsmn(hidden_dim, hidden_dim, lorder, hidden_dim)
|
| 412 |
+
for i in range(layers)
|
| 413 |
+
]
|
| 414 |
+
else:
|
| 415 |
+
repeats = [
|
| 416 |
+
DeepFsmn(hidden_dim, hidden_dim, lorder, rorder, hidden_dim)
|
| 417 |
+
for i in range(layers)
|
| 418 |
+
]
|
| 419 |
+
self.deepfsmn = nn.Sequential(*repeats)
|
| 420 |
+
|
| 421 |
+
self.ch_transform = nn.ModuleList([])
|
| 422 |
+
self.ch_average = nn.ModuleList([])
|
| 423 |
+
self.ch_concat = nn.ModuleList([])
|
| 424 |
+
for i in range(layers):
|
| 425 |
+
self.ch_transform.append(
|
| 426 |
+
nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU()))
|
| 427 |
+
self.ch_average.append(
|
| 428 |
+
nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU()))
|
| 429 |
+
self.ch_concat.append(
|
| 430 |
+
nn.Sequential(
|
| 431 |
+
nn.Linear(hidden_dim * 2, hidden_dim), nn.PReLU()))
|
| 432 |
+
|
| 433 |
+
self.linear2 = AffineTransform(hidden_dim, outdim)
|
| 434 |
+
|
| 435 |
+
self.crm = crm
|
| 436 |
+
if self.crm:
|
| 437 |
+
self.sig = nn.Tanh()
|
| 438 |
+
else:
|
| 439 |
+
self.sig = Sigmoid(outdim, outdim)
|
| 440 |
+
|
| 441 |
+
self.vad = vad
|
| 442 |
+
if self.vad:
|
| 443 |
+
self.linear3 = AffineTransform(hidden_dim, 1)
|
| 444 |
+
|
| 445 |
+
self.layers = layers
|
| 446 |
+
self.linearout = linearout
|
| 447 |
+
if self.linearout and self.vad:
|
| 448 |
+
print('Warning: not supported nnet')
|
| 449 |
+
|
| 450 |
+
def forward(self, feat, ctl=None):
|
| 451 |
+
B, T, F = feat.shape
|
| 452 |
+
# assume 4ch
|
| 453 |
+
ch = 4
|
| 454 |
+
zlist = []
|
| 455 |
+
for c in range(ch):
|
| 456 |
+
z = self.linear1(feat[..., c * (F // 4):(c + 1) * (F // 4)])
|
| 457 |
+
z = self.relu(z)
|
| 458 |
+
zlist.append(z)
|
| 459 |
+
for i in range(self.layers):
|
| 460 |
+
# forward
|
| 461 |
+
for c in range(ch):
|
| 462 |
+
zlist[c] = self.deepfsmn[i](zlist[c])
|
| 463 |
+
|
| 464 |
+
# transform
|
| 465 |
+
olist = []
|
| 466 |
+
for c in range(ch):
|
| 467 |
+
z = self.ch_transform[i](zlist[c])
|
| 468 |
+
olist.append(z)
|
| 469 |
+
# average
|
| 470 |
+
avg = 0
|
| 471 |
+
for c in range(ch):
|
| 472 |
+
avg = avg + olist[c]
|
| 473 |
+
avg = avg / ch
|
| 474 |
+
avg = self.ch_average[i](avg)
|
| 475 |
+
# concate
|
| 476 |
+
for c in range(ch):
|
| 477 |
+
tac = torch.cat([olist[c], avg], dim=-1)
|
| 478 |
+
tac = self.ch_concat[i](tac)
|
| 479 |
+
zlist[c] = zlist[c] + tac
|
| 480 |
+
|
| 481 |
+
for c in range(ch):
|
| 482 |
+
zlist[c] = self.sig(self.linear2(zlist[c]))
|
| 483 |
+
mask = torch.cat(zlist, dim=-1)
|
| 484 |
+
return mask
|
| 485 |
+
|
| 486 |
+
def to_kaldi_nnet(self):
|
| 487 |
+
pass
|
runtime/Lib/site_packages/models/audio/ans/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.import_utils import LazyImportModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .frcrn import FRCRNDecorator
|
| 8 |
+
|
| 9 |
+
else:
|
| 10 |
+
_import_structure = {
|
| 11 |
+
'frcrn': ['FRCRNDecorator'],
|
| 12 |
+
'dnoise_net': ['DenoiseNet'],
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
|
| 17 |
+
sys.modules[__name__] = LazyImportModule(
|
| 18 |
+
__name__,
|
| 19 |
+
globals()['__file__'],
|
| 20 |
+
_import_structure,
|
| 21 |
+
module_spec=__spec__,
|
| 22 |
+
extra_objects={},
|
| 23 |
+
)
|
runtime/Lib/site_packages/models/audio/ans/complex_nn.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# The implementation of class ComplexConv2d, ComplexConvTranspose2d and
|
| 4 |
+
# ComplexBatchNorm2d here is modified based on Jongho Choi(sweetcocoa@snu.ac.kr
|
| 5 |
+
# / Seoul National Univ., ESTsoft ) and publicly available at
|
| 6 |
+
# https://github.com/sweetcocoa/DeepComplexUNetPyTorch
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
|
| 11 |
+
from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ComplexUniDeepFsmn(nn.Module):
|
| 15 |
+
|
| 16 |
+
def __init__(self, nIn, nHidden=128, nOut=128):
|
| 17 |
+
super(ComplexUniDeepFsmn, self).__init__()
|
| 18 |
+
|
| 19 |
+
self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
|
| 20 |
+
self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
|
| 21 |
+
self.fsmn_re_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden)
|
| 22 |
+
self.fsmn_im_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden)
|
| 23 |
+
|
| 24 |
+
def forward(self, x):
|
| 25 |
+
r"""
|
| 26 |
+
|
| 27 |
+
Args:
|
| 28 |
+
x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2]
|
| 29 |
+
|
| 30 |
+
Returns:
|
| 31 |
+
[batch, feature, sequence, 2], eg: [6, 99, 1024, 2]
|
| 32 |
+
"""
|
| 33 |
+
#
|
| 34 |
+
b, c, h, T, d = x.size()
|
| 35 |
+
x = torch.reshape(x, (b, c * h, T, d))
|
| 36 |
+
# x: [b,h,T,2], [6, 256, 106, 2]
|
| 37 |
+
x = torch.transpose(x, 1, 2)
|
| 38 |
+
# x: [b,T,h,2], [6, 106, 256, 2]
|
| 39 |
+
|
| 40 |
+
real_L1 = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1])
|
| 41 |
+
imaginary_L1 = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0])
|
| 42 |
+
# GRU output: [99, 6, 128]
|
| 43 |
+
real = self.fsmn_re_L2(real_L1) - self.fsmn_im_L2(imaginary_L1)
|
| 44 |
+
imaginary = self.fsmn_re_L2(imaginary_L1) + self.fsmn_im_L2(real_L1)
|
| 45 |
+
# output: [b,T,h,2], [99, 6, 1024, 2]
|
| 46 |
+
output = torch.stack((real, imaginary), dim=-1)
|
| 47 |
+
|
| 48 |
+
# output: [b,h,T,2], [6, 99, 1024, 2]
|
| 49 |
+
output = torch.transpose(output, 1, 2)
|
| 50 |
+
output = torch.reshape(output, (b, c, h, T, d))
|
| 51 |
+
|
| 52 |
+
return output
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class ComplexUniDeepFsmn_L1(nn.Module):
|
| 56 |
+
|
| 57 |
+
def __init__(self, nIn, nHidden=128, nOut=128):
|
| 58 |
+
super(ComplexUniDeepFsmn_L1, self).__init__()
|
| 59 |
+
self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
|
| 60 |
+
self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
|
| 61 |
+
|
| 62 |
+
def forward(self, x):
|
| 63 |
+
r"""
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2]
|
| 67 |
+
"""
|
| 68 |
+
b, c, h, T, d = x.size()
|
| 69 |
+
# x : [b,T,h,c,2]
|
| 70 |
+
x = torch.transpose(x, 1, 3)
|
| 71 |
+
x = torch.reshape(x, (b * T, h, c, d))
|
| 72 |
+
|
| 73 |
+
real = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1])
|
| 74 |
+
imaginary = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0])
|
| 75 |
+
# output: [b*T,h,c,2], [6*106, h, 256, 2]
|
| 76 |
+
output = torch.stack((real, imaginary), dim=-1)
|
| 77 |
+
|
| 78 |
+
output = torch.reshape(output, (b, T, h, c, d))
|
| 79 |
+
output = torch.transpose(output, 1, 3)
|
| 80 |
+
return output
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class ComplexConv2d(nn.Module):
|
| 84 |
+
# https://github.com/litcoderr/ComplexCNN/blob/master/complexcnn/modules.py
|
| 85 |
+
def __init__(self,
|
| 86 |
+
in_channel,
|
| 87 |
+
out_channel,
|
| 88 |
+
kernel_size,
|
| 89 |
+
stride=1,
|
| 90 |
+
padding=0,
|
| 91 |
+
dilation=1,
|
| 92 |
+
groups=1,
|
| 93 |
+
bias=True,
|
| 94 |
+
**kwargs):
|
| 95 |
+
super().__init__()
|
| 96 |
+
|
| 97 |
+
# Model components
|
| 98 |
+
self.conv_re = nn.Conv2d(
|
| 99 |
+
in_channel,
|
| 100 |
+
out_channel,
|
| 101 |
+
kernel_size,
|
| 102 |
+
stride=stride,
|
| 103 |
+
padding=padding,
|
| 104 |
+
dilation=dilation,
|
| 105 |
+
groups=groups,
|
| 106 |
+
bias=bias,
|
| 107 |
+
**kwargs)
|
| 108 |
+
self.conv_im = nn.Conv2d(
|
| 109 |
+
in_channel,
|
| 110 |
+
out_channel,
|
| 111 |
+
kernel_size,
|
| 112 |
+
stride=stride,
|
| 113 |
+
padding=padding,
|
| 114 |
+
dilation=dilation,
|
| 115 |
+
groups=groups,
|
| 116 |
+
bias=bias,
|
| 117 |
+
**kwargs)
|
| 118 |
+
|
| 119 |
+
def forward(self, x):
|
| 120 |
+
r"""
|
| 121 |
+
|
| 122 |
+
Args:
|
| 123 |
+
x: torch with shape: [batch,channel,axis1,axis2,2]
|
| 124 |
+
"""
|
| 125 |
+
real = self.conv_re(x[..., 0]) - self.conv_im(x[..., 1])
|
| 126 |
+
imaginary = self.conv_re(x[..., 1]) + self.conv_im(x[..., 0])
|
| 127 |
+
output = torch.stack((real, imaginary), dim=-1)
|
| 128 |
+
return output
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class ComplexConvTranspose2d(nn.Module):
|
| 132 |
+
|
| 133 |
+
def __init__(self,
|
| 134 |
+
in_channel,
|
| 135 |
+
out_channel,
|
| 136 |
+
kernel_size,
|
| 137 |
+
stride=1,
|
| 138 |
+
padding=0,
|
| 139 |
+
output_padding=0,
|
| 140 |
+
dilation=1,
|
| 141 |
+
groups=1,
|
| 142 |
+
bias=True,
|
| 143 |
+
**kwargs):
|
| 144 |
+
super().__init__()
|
| 145 |
+
|
| 146 |
+
# Model components
|
| 147 |
+
self.tconv_re = nn.ConvTranspose2d(
|
| 148 |
+
in_channel,
|
| 149 |
+
out_channel,
|
| 150 |
+
kernel_size=kernel_size,
|
| 151 |
+
stride=stride,
|
| 152 |
+
padding=padding,
|
| 153 |
+
output_padding=output_padding,
|
| 154 |
+
groups=groups,
|
| 155 |
+
bias=bias,
|
| 156 |
+
dilation=dilation,
|
| 157 |
+
**kwargs)
|
| 158 |
+
self.tconv_im = nn.ConvTranspose2d(
|
| 159 |
+
in_channel,
|
| 160 |
+
out_channel,
|
| 161 |
+
kernel_size=kernel_size,
|
| 162 |
+
stride=stride,
|
| 163 |
+
padding=padding,
|
| 164 |
+
output_padding=output_padding,
|
| 165 |
+
groups=groups,
|
| 166 |
+
bias=bias,
|
| 167 |
+
dilation=dilation,
|
| 168 |
+
**kwargs)
|
| 169 |
+
|
| 170 |
+
def forward(self, x): # shpae of x : [batch,channel,axis1,axis2,2]
|
| 171 |
+
real = self.tconv_re(x[..., 0]) - self.tconv_im(x[..., 1])
|
| 172 |
+
imaginary = self.tconv_re(x[..., 1]) + self.tconv_im(x[..., 0])
|
| 173 |
+
output = torch.stack((real, imaginary), dim=-1)
|
| 174 |
+
return output
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
class ComplexBatchNorm2d(nn.Module):
|
| 178 |
+
|
| 179 |
+
def __init__(self,
|
| 180 |
+
num_features,
|
| 181 |
+
eps=1e-5,
|
| 182 |
+
momentum=0.1,
|
| 183 |
+
affine=True,
|
| 184 |
+
track_running_stats=True,
|
| 185 |
+
**kwargs):
|
| 186 |
+
super().__init__()
|
| 187 |
+
self.bn_re = nn.BatchNorm2d(
|
| 188 |
+
num_features=num_features,
|
| 189 |
+
momentum=momentum,
|
| 190 |
+
affine=affine,
|
| 191 |
+
eps=eps,
|
| 192 |
+
track_running_stats=track_running_stats,
|
| 193 |
+
**kwargs)
|
| 194 |
+
self.bn_im = nn.BatchNorm2d(
|
| 195 |
+
num_features=num_features,
|
| 196 |
+
momentum=momentum,
|
| 197 |
+
affine=affine,
|
| 198 |
+
eps=eps,
|
| 199 |
+
track_running_stats=track_running_stats,
|
| 200 |
+
**kwargs)
|
| 201 |
+
|
| 202 |
+
def forward(self, x):
|
| 203 |
+
real = self.bn_re(x[..., 0])
|
| 204 |
+
imag = self.bn_im(x[..., 1])
|
| 205 |
+
output = torch.stack((real, imag), dim=-1)
|
| 206 |
+
return output
|
runtime/Lib/site_packages/models/audio/ans/conv_stft.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
import numpy as np
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from scipy.signal import get_window
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def init_kernels(win_len, win_inc, fft_len, win_type=None, invers=False):
|
| 10 |
+
if win_type == 'None' or win_type is None:
|
| 11 |
+
window = np.ones(win_len)
|
| 12 |
+
else:
|
| 13 |
+
window = get_window(win_type, win_len, fftbins=True)**0.5
|
| 14 |
+
|
| 15 |
+
N = fft_len
|
| 16 |
+
fourier_basis = np.fft.rfft(np.eye(N))[:win_len]
|
| 17 |
+
real_kernel = np.real(fourier_basis)
|
| 18 |
+
imag_kernel = np.imag(fourier_basis)
|
| 19 |
+
kernel = np.concatenate([real_kernel, imag_kernel], 1).T
|
| 20 |
+
|
| 21 |
+
if invers:
|
| 22 |
+
kernel = np.linalg.pinv(kernel).T
|
| 23 |
+
|
| 24 |
+
kernel = kernel * window
|
| 25 |
+
kernel = kernel[:, None, :]
|
| 26 |
+
return torch.from_numpy(kernel.astype(np.float32)), torch.from_numpy(
|
| 27 |
+
window[None, :, None].astype(np.float32))
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class ConvSTFT(nn.Module):
|
| 31 |
+
|
| 32 |
+
def __init__(self,
|
| 33 |
+
win_len,
|
| 34 |
+
win_inc,
|
| 35 |
+
fft_len=None,
|
| 36 |
+
win_type='hamming',
|
| 37 |
+
feature_type='real',
|
| 38 |
+
fix=True):
|
| 39 |
+
super(ConvSTFT, self).__init__()
|
| 40 |
+
|
| 41 |
+
if fft_len is None:
|
| 42 |
+
self.fft_len = int(2**np.ceil(np.log2(win_len)))
|
| 43 |
+
else:
|
| 44 |
+
self.fft_len = fft_len
|
| 45 |
+
|
| 46 |
+
kernel, _ = init_kernels(win_len, win_inc, self.fft_len, win_type)
|
| 47 |
+
self.weight = nn.Parameter(kernel, requires_grad=(not fix))
|
| 48 |
+
self.feature_type = feature_type
|
| 49 |
+
self.stride = win_inc
|
| 50 |
+
self.win_len = win_len
|
| 51 |
+
self.dim = self.fft_len
|
| 52 |
+
|
| 53 |
+
def forward(self, inputs):
|
| 54 |
+
if inputs.dim() == 2:
|
| 55 |
+
inputs = torch.unsqueeze(inputs, 1)
|
| 56 |
+
|
| 57 |
+
outputs = F.conv1d(inputs, self.weight, stride=self.stride)
|
| 58 |
+
|
| 59 |
+
if self.feature_type == 'complex':
|
| 60 |
+
return outputs
|
| 61 |
+
else:
|
| 62 |
+
dim = self.dim // 2 + 1
|
| 63 |
+
real = outputs[:, :dim, :]
|
| 64 |
+
imag = outputs[:, dim:, :]
|
| 65 |
+
mags = torch.sqrt(real**2 + imag**2)
|
| 66 |
+
phase = torch.atan2(imag, real)
|
| 67 |
+
return mags, phase
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class ConviSTFT(nn.Module):
|
| 71 |
+
|
| 72 |
+
def __init__(self,
|
| 73 |
+
win_len,
|
| 74 |
+
win_inc,
|
| 75 |
+
fft_len=None,
|
| 76 |
+
win_type='hamming',
|
| 77 |
+
feature_type='real',
|
| 78 |
+
fix=True):
|
| 79 |
+
super(ConviSTFT, self).__init__()
|
| 80 |
+
if fft_len is None:
|
| 81 |
+
self.fft_len = int(2**np.ceil(np.log2(win_len)))
|
| 82 |
+
else:
|
| 83 |
+
self.fft_len = fft_len
|
| 84 |
+
kernel, window = init_kernels(
|
| 85 |
+
win_len, win_inc, self.fft_len, win_type, invers=True)
|
| 86 |
+
self.weight = nn.Parameter(kernel, requires_grad=(not fix))
|
| 87 |
+
self.feature_type = feature_type
|
| 88 |
+
self.win_type = win_type
|
| 89 |
+
self.win_len = win_len
|
| 90 |
+
self.win_inc = win_inc
|
| 91 |
+
self.stride = win_inc
|
| 92 |
+
self.dim = self.fft_len
|
| 93 |
+
self.register_buffer('window', window)
|
| 94 |
+
self.register_buffer('enframe', torch.eye(win_len)[:, None, :])
|
| 95 |
+
|
| 96 |
+
def forward(self, inputs, phase=None):
|
| 97 |
+
"""
|
| 98 |
+
Args:
|
| 99 |
+
inputs : [B, N+2, T] (complex spec) or [B, N//2+1, T] (mags)
|
| 100 |
+
phase: [B, N//2+1, T] (if not none)
|
| 101 |
+
"""
|
| 102 |
+
|
| 103 |
+
if phase is not None:
|
| 104 |
+
real = inputs * torch.cos(phase)
|
| 105 |
+
imag = inputs * torch.sin(phase)
|
| 106 |
+
inputs = torch.cat([real, imag], 1)
|
| 107 |
+
outputs = F.conv_transpose1d(inputs, self.weight, stride=self.stride)
|
| 108 |
+
|
| 109 |
+
# this is from torch-stft: https://github.com/pseeth/torch-stft
|
| 110 |
+
t = self.window.repeat(1, 1, inputs.size(-1))**2
|
| 111 |
+
coff = F.conv_transpose1d(t, self.enframe, stride=self.stride)
|
| 112 |
+
outputs = outputs / (coff + 1e-8)
|
| 113 |
+
return outputs
|
runtime/Lib/site_packages/models/audio/ans/denoise_net.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
# Related papers:
|
| 3 |
+
# Shengkui Zhao, Trung Hieu Nguyen, Bin Ma, “Monaural Speech Enhancement with Complex Convolutional
|
| 4 |
+
# Block Attention Module and Joint Time Frequency Losses”, ICASSP 2021.
|
| 5 |
+
# Shiliang Zhang, Ming Lei, Zhijie Yan, Lirong Dai, “Deep-FSMN for Large Vocabulary Continuous Speech
|
| 6 |
+
# Recognition “, arXiv:1803.05030, 2018.
|
| 7 |
+
|
| 8 |
+
from torch import nn
|
| 9 |
+
|
| 10 |
+
from modelscope.metainfo import Models
|
| 11 |
+
from modelscope.models import MODELS, TorchModel
|
| 12 |
+
from modelscope.models.audio.ans.layers.activations import (RectifiedLinear,
|
| 13 |
+
Sigmoid)
|
| 14 |
+
from modelscope.models.audio.ans.layers.affine_transform import AffineTransform
|
| 15 |
+
from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn
|
| 16 |
+
from modelscope.utils.constant import Tasks
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@MODELS.register_module(
|
| 20 |
+
Tasks.acoustic_noise_suppression, module_name=Models.speech_dfsmn_ans)
|
| 21 |
+
class DfsmnAns(TorchModel):
|
| 22 |
+
"""Denoise model with DFSMN.
|
| 23 |
+
|
| 24 |
+
Args:
|
| 25 |
+
model_dir (str): the model path.
|
| 26 |
+
fsmn_depth (int): the depth of deepfsmn
|
| 27 |
+
lorder (int):
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(self,
|
| 31 |
+
model_dir: str,
|
| 32 |
+
fsmn_depth=9,
|
| 33 |
+
lorder=20,
|
| 34 |
+
*args,
|
| 35 |
+
**kwargs):
|
| 36 |
+
super().__init__(model_dir, *args, **kwargs)
|
| 37 |
+
self.lorder = lorder
|
| 38 |
+
self.linear1 = AffineTransform(120, 256)
|
| 39 |
+
self.relu = RectifiedLinear(256, 256)
|
| 40 |
+
repeats = [
|
| 41 |
+
UniDeepFsmn(256, 256, lorder, 256) for i in range(fsmn_depth)
|
| 42 |
+
]
|
| 43 |
+
self.deepfsmn = nn.Sequential(*repeats)
|
| 44 |
+
self.linear2 = AffineTransform(256, 961)
|
| 45 |
+
self.sig = Sigmoid(961, 961)
|
| 46 |
+
|
| 47 |
+
def forward(self, input):
|
| 48 |
+
"""
|
| 49 |
+
Args:
|
| 50 |
+
input: fbank feature [batch_size,number_of_frame,feature_dimension]
|
| 51 |
+
|
| 52 |
+
Returns:
|
| 53 |
+
mask value [batch_size, number_of_frame, FFT_size/2+1]
|
| 54 |
+
"""
|
| 55 |
+
x1 = self.linear1(input)
|
| 56 |
+
x2 = self.relu(x1)
|
| 57 |
+
x3 = self.deepfsmn(x2)
|
| 58 |
+
x4 = self.linear2(x3)
|
| 59 |
+
x5 = self.sig(x4)
|
| 60 |
+
return x5
|
| 61 |
+
|
| 62 |
+
def to_kaldi_nnet(self):
|
| 63 |
+
re_str = ''
|
| 64 |
+
re_str += '<Nnet>\n'
|
| 65 |
+
re_str += self.linear1.to_kaldi_nnet()
|
| 66 |
+
re_str += self.relu.to_kaldi_nnet()
|
| 67 |
+
for dfsmn in self.deepfsmn:
|
| 68 |
+
re_str += dfsmn.to_kaldi_nnet()
|
| 69 |
+
re_str += self.linear2.to_kaldi_nnet()
|
| 70 |
+
re_str += self.sig.to_kaldi_nnet()
|
| 71 |
+
re_str += '</Nnet>\n'
|
| 72 |
+
|
| 73 |
+
return re_str
|
runtime/Lib/site_packages/models/audio/ans/frcrn.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
import os
|
| 3 |
+
from typing import Dict
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
|
| 9 |
+
from modelscope.metainfo import Models
|
| 10 |
+
from modelscope.models import TorchModel
|
| 11 |
+
from modelscope.models.base import Tensor
|
| 12 |
+
from modelscope.models.builder import MODELS
|
| 13 |
+
from modelscope.utils.constant import ModelFile, Tasks
|
| 14 |
+
from .conv_stft import ConviSTFT, ConvSTFT
|
| 15 |
+
from .unet import UNet
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@MODELS.register_module(
|
| 19 |
+
Tasks.acoustic_noise_suppression,
|
| 20 |
+
module_name=Models.speech_frcrn_ans_cirm_16k)
|
| 21 |
+
class FRCRNDecorator(TorchModel):
|
| 22 |
+
r""" A decorator of FRCRN for integrating into modelscope framework """
|
| 23 |
+
|
| 24 |
+
def __init__(self, model_dir: str, *args, **kwargs):
|
| 25 |
+
"""initialize the frcrn model from the `model_dir` path.
|
| 26 |
+
|
| 27 |
+
Args:
|
| 28 |
+
model_dir (str): the model path.
|
| 29 |
+
"""
|
| 30 |
+
super().__init__(model_dir, *args, **kwargs)
|
| 31 |
+
self.model = FRCRN(*args, **kwargs)
|
| 32 |
+
model_bin_file = os.path.join(model_dir,
|
| 33 |
+
ModelFile.TORCH_MODEL_BIN_FILE)
|
| 34 |
+
if os.path.exists(model_bin_file):
|
| 35 |
+
checkpoint = torch.load(
|
| 36 |
+
model_bin_file, map_location=torch.device('cpu'))
|
| 37 |
+
if isinstance(checkpoint, dict) and 'state_dict' in checkpoint:
|
| 38 |
+
# the new trained model by user is based on FRCRNDecorator
|
| 39 |
+
self.load_state_dict(checkpoint['state_dict'])
|
| 40 |
+
else:
|
| 41 |
+
# The released model on Modelscope is based on FRCRN
|
| 42 |
+
self.model.load_state_dict(checkpoint, strict=False)
|
| 43 |
+
|
| 44 |
+
def forward(self, inputs: Dict[str, Tensor]) -> Dict[str, Tensor]:
|
| 45 |
+
result_list = self.model.forward(inputs['noisy'])
|
| 46 |
+
output = {
|
| 47 |
+
'spec_l1': result_list[0],
|
| 48 |
+
'wav_l1': result_list[1],
|
| 49 |
+
'mask_l1': result_list[2],
|
| 50 |
+
'spec_l2': result_list[3],
|
| 51 |
+
'wav_l2': result_list[4],
|
| 52 |
+
'mask_l2': result_list[5]
|
| 53 |
+
}
|
| 54 |
+
if 'clean' in inputs:
|
| 55 |
+
mix_result = self.model.loss(
|
| 56 |
+
inputs['noisy'], inputs['clean'], result_list, mode='Mix')
|
| 57 |
+
output.update(mix_result)
|
| 58 |
+
sisnr_result = self.model.loss(
|
| 59 |
+
inputs['noisy'], inputs['clean'], result_list, mode='SiSNR')
|
| 60 |
+
output.update(sisnr_result)
|
| 61 |
+
# logger hooker will use items under 'log_vars'
|
| 62 |
+
output['log_vars'] = {k: mix_result[k].item() for k in mix_result}
|
| 63 |
+
output['log_vars'].update(
|
| 64 |
+
{k: sisnr_result[k].item()
|
| 65 |
+
for k in sisnr_result})
|
| 66 |
+
return output
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class FRCRN(nn.Module):
|
| 70 |
+
r""" Frequency Recurrent CRN """
|
| 71 |
+
|
| 72 |
+
def __init__(self,
|
| 73 |
+
complex,
|
| 74 |
+
model_complexity,
|
| 75 |
+
model_depth,
|
| 76 |
+
log_amp,
|
| 77 |
+
padding_mode,
|
| 78 |
+
win_len=400,
|
| 79 |
+
win_inc=100,
|
| 80 |
+
fft_len=512,
|
| 81 |
+
win_type='hann',
|
| 82 |
+
**kwargs):
|
| 83 |
+
r"""
|
| 84 |
+
Args:
|
| 85 |
+
complex: Whether to use complex networks.
|
| 86 |
+
model_complexity: define the model complexity with the number of layers
|
| 87 |
+
model_depth: Only two options are available : 10, 20
|
| 88 |
+
log_amp: Whether to use log amplitude to estimate signals
|
| 89 |
+
padding_mode: Encoder's convolution filter. 'zeros', 'reflect'
|
| 90 |
+
win_len: length of window used for defining one frame of sample points
|
| 91 |
+
win_inc: length of window shifting (equivalent to hop_size)
|
| 92 |
+
fft_len: number of Short Time Fourier Transform (STFT) points
|
| 93 |
+
win_type: windowing type used in STFT, eg. 'hanning', 'hamming'
|
| 94 |
+
"""
|
| 95 |
+
super().__init__()
|
| 96 |
+
self.feat_dim = fft_len // 2 + 1
|
| 97 |
+
|
| 98 |
+
self.win_len = win_len
|
| 99 |
+
self.win_inc = win_inc
|
| 100 |
+
self.fft_len = fft_len
|
| 101 |
+
self.win_type = win_type
|
| 102 |
+
|
| 103 |
+
fix = True
|
| 104 |
+
self.stft = ConvSTFT(
|
| 105 |
+
self.win_len,
|
| 106 |
+
self.win_inc,
|
| 107 |
+
self.fft_len,
|
| 108 |
+
self.win_type,
|
| 109 |
+
feature_type='complex',
|
| 110 |
+
fix=fix)
|
| 111 |
+
self.istft = ConviSTFT(
|
| 112 |
+
self.win_len,
|
| 113 |
+
self.win_inc,
|
| 114 |
+
self.fft_len,
|
| 115 |
+
self.win_type,
|
| 116 |
+
feature_type='complex',
|
| 117 |
+
fix=fix)
|
| 118 |
+
self.unet = UNet(
|
| 119 |
+
1,
|
| 120 |
+
complex=complex,
|
| 121 |
+
model_complexity=model_complexity,
|
| 122 |
+
model_depth=model_depth,
|
| 123 |
+
padding_mode=padding_mode)
|
| 124 |
+
self.unet2 = UNet(
|
| 125 |
+
1,
|
| 126 |
+
complex=complex,
|
| 127 |
+
model_complexity=model_complexity,
|
| 128 |
+
model_depth=model_depth,
|
| 129 |
+
padding_mode=padding_mode)
|
| 130 |
+
|
| 131 |
+
def forward(self, inputs):
|
| 132 |
+
out_list = []
|
| 133 |
+
# [B, D*2, T]
|
| 134 |
+
cmp_spec = self.stft(inputs)
|
| 135 |
+
# [B, 1, D*2, T]
|
| 136 |
+
cmp_spec = torch.unsqueeze(cmp_spec, 1)
|
| 137 |
+
|
| 138 |
+
# to [B, 2, D, T] real_part/imag_part
|
| 139 |
+
cmp_spec = torch.cat([
|
| 140 |
+
cmp_spec[:, :, :self.feat_dim, :],
|
| 141 |
+
cmp_spec[:, :, self.feat_dim:, :],
|
| 142 |
+
], 1)
|
| 143 |
+
|
| 144 |
+
# [B, 2, D, T]
|
| 145 |
+
cmp_spec = torch.unsqueeze(cmp_spec, 4)
|
| 146 |
+
# [B, 1, D, T, 2]
|
| 147 |
+
cmp_spec = torch.transpose(cmp_spec, 1, 4)
|
| 148 |
+
unet1_out = self.unet(cmp_spec)
|
| 149 |
+
cmp_mask1 = torch.tanh(unet1_out)
|
| 150 |
+
unet2_out = self.unet2(unet1_out)
|
| 151 |
+
cmp_mask2 = torch.tanh(unet2_out)
|
| 152 |
+
est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask1)
|
| 153 |
+
out_list.append(est_spec)
|
| 154 |
+
out_list.append(est_wav)
|
| 155 |
+
out_list.append(est_mask)
|
| 156 |
+
cmp_mask2 = cmp_mask2 + cmp_mask1
|
| 157 |
+
est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask2)
|
| 158 |
+
out_list.append(est_spec)
|
| 159 |
+
out_list.append(est_wav)
|
| 160 |
+
out_list.append(est_mask)
|
| 161 |
+
return out_list
|
| 162 |
+
|
| 163 |
+
def apply_mask(self, cmp_spec, cmp_mask):
|
| 164 |
+
est_spec = torch.cat([
|
| 165 |
+
cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 0]
|
| 166 |
+
- cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 1],
|
| 167 |
+
cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 1]
|
| 168 |
+
+ cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 0]
|
| 169 |
+
], 1)
|
| 170 |
+
est_spec = torch.cat([est_spec[:, 0, :, :], est_spec[:, 1, :, :]], 1)
|
| 171 |
+
cmp_mask = torch.squeeze(cmp_mask, 1)
|
| 172 |
+
cmp_mask = torch.cat([cmp_mask[:, :, :, 0], cmp_mask[:, :, :, 1]], 1)
|
| 173 |
+
|
| 174 |
+
est_wav = self.istft(est_spec)
|
| 175 |
+
est_wav = torch.squeeze(est_wav, 1)
|
| 176 |
+
return est_spec, est_wav, cmp_mask
|
| 177 |
+
|
| 178 |
+
def get_params(self, weight_decay=0.0):
|
| 179 |
+
# add L2 penalty
|
| 180 |
+
weights, biases = [], []
|
| 181 |
+
for name, param in self.named_parameters():
|
| 182 |
+
if 'bias' in name:
|
| 183 |
+
biases += [param]
|
| 184 |
+
else:
|
| 185 |
+
weights += [param]
|
| 186 |
+
params = [{
|
| 187 |
+
'params': weights,
|
| 188 |
+
'weight_decay': weight_decay,
|
| 189 |
+
}, {
|
| 190 |
+
'params': biases,
|
| 191 |
+
'weight_decay': 0.0,
|
| 192 |
+
}]
|
| 193 |
+
return params
|
| 194 |
+
|
| 195 |
+
def loss(self, noisy, labels, out_list, mode='Mix'):
|
| 196 |
+
if mode == 'SiSNR':
|
| 197 |
+
count = 0
|
| 198 |
+
while count < len(out_list):
|
| 199 |
+
est_spec = out_list[count]
|
| 200 |
+
count = count + 1
|
| 201 |
+
est_wav = out_list[count]
|
| 202 |
+
count = count + 1
|
| 203 |
+
est_mask = out_list[count]
|
| 204 |
+
count = count + 1
|
| 205 |
+
if count != 3:
|
| 206 |
+
loss = self.loss_1layer(noisy, est_spec, est_wav, labels,
|
| 207 |
+
est_mask, mode)
|
| 208 |
+
return dict(sisnr=loss)
|
| 209 |
+
|
| 210 |
+
elif mode == 'Mix':
|
| 211 |
+
count = 0
|
| 212 |
+
while count < len(out_list):
|
| 213 |
+
est_spec = out_list[count]
|
| 214 |
+
count = count + 1
|
| 215 |
+
est_wav = out_list[count]
|
| 216 |
+
count = count + 1
|
| 217 |
+
est_mask = out_list[count]
|
| 218 |
+
count = count + 1
|
| 219 |
+
if count != 3:
|
| 220 |
+
amp_loss, phase_loss, SiSNR_loss = self.loss_1layer(
|
| 221 |
+
noisy, est_spec, est_wav, labels, est_mask, mode)
|
| 222 |
+
loss = amp_loss + phase_loss + SiSNR_loss
|
| 223 |
+
return dict(loss=loss, amp_loss=amp_loss, phase_loss=phase_loss)
|
| 224 |
+
|
| 225 |
+
def loss_1layer(self, noisy, est, est_wav, labels, cmp_mask, mode='Mix'):
|
| 226 |
+
r""" Compute the loss by mode
|
| 227 |
+
mode == 'Mix'
|
| 228 |
+
est: [B, F*2, T]
|
| 229 |
+
labels: [B, F*2,T]
|
| 230 |
+
mode == 'SiSNR'
|
| 231 |
+
est: [B, T]
|
| 232 |
+
labels: [B, T]
|
| 233 |
+
"""
|
| 234 |
+
if mode == 'SiSNR':
|
| 235 |
+
if labels.dim() == 3:
|
| 236 |
+
labels = torch.squeeze(labels, 1)
|
| 237 |
+
if est_wav.dim() == 3:
|
| 238 |
+
est_wav = torch.squeeze(est_wav, 1)
|
| 239 |
+
return -si_snr(est_wav, labels)
|
| 240 |
+
elif mode == 'Mix':
|
| 241 |
+
|
| 242 |
+
if labels.dim() == 3:
|
| 243 |
+
labels = torch.squeeze(labels, 1)
|
| 244 |
+
if est_wav.dim() == 3:
|
| 245 |
+
est_wav = torch.squeeze(est_wav, 1)
|
| 246 |
+
SiSNR_loss = -si_snr(est_wav, labels)
|
| 247 |
+
|
| 248 |
+
b, d, t = est.size()
|
| 249 |
+
S = self.stft(labels)
|
| 250 |
+
Sr = S[:, :self.feat_dim, :]
|
| 251 |
+
Si = S[:, self.feat_dim:, :]
|
| 252 |
+
Y = self.stft(noisy)
|
| 253 |
+
Yr = Y[:, :self.feat_dim, :]
|
| 254 |
+
Yi = Y[:, self.feat_dim:, :]
|
| 255 |
+
Y_pow = Yr**2 + Yi**2
|
| 256 |
+
gth_mask = torch.cat([(Sr * Yr + Si * Yi) / (Y_pow + 1e-8),
|
| 257 |
+
(Si * Yr - Sr * Yi) / (Y_pow + 1e-8)], 1)
|
| 258 |
+
gth_mask[gth_mask > 2] = 1
|
| 259 |
+
gth_mask[gth_mask < -2] = -1
|
| 260 |
+
amp_loss = F.mse_loss(gth_mask[:, :self.feat_dim, :],
|
| 261 |
+
cmp_mask[:, :self.feat_dim, :]) * d
|
| 262 |
+
phase_loss = F.mse_loss(gth_mask[:, self.feat_dim:, :],
|
| 263 |
+
cmp_mask[:, self.feat_dim:, :]) * d
|
| 264 |
+
return amp_loss, phase_loss, SiSNR_loss
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def l2_norm(s1, s2):
|
| 268 |
+
norm = torch.sum(s1 * s2, -1, keepdim=True)
|
| 269 |
+
return norm
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def si_snr(s1, s2, eps=1e-8):
|
| 273 |
+
s1_s2_norm = l2_norm(s1, s2)
|
| 274 |
+
s2_s2_norm = l2_norm(s2, s2)
|
| 275 |
+
s_target = s1_s2_norm / (s2_s2_norm + eps) * s2
|
| 276 |
+
e_noise = s1 - s_target
|
| 277 |
+
target_norm = l2_norm(s_target, s_target)
|
| 278 |
+
noise_norm = l2_norm(e_noise, e_noise)
|
| 279 |
+
snr = 10 * torch.log10((target_norm) / (noise_norm + eps) + eps)
|
| 280 |
+
return torch.mean(snr)
|
runtime/Lib/site_packages/models/audio/ans/layers/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/ans/layers/activations.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
|
| 5 |
+
from modelscope.models.audio.ans.layers.layer_base import LayerBase
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class RectifiedLinear(LayerBase):
|
| 9 |
+
|
| 10 |
+
def __init__(self, input_dim, output_dim):
|
| 11 |
+
super(RectifiedLinear, self).__init__()
|
| 12 |
+
self.dim = input_dim
|
| 13 |
+
self.relu = nn.ReLU()
|
| 14 |
+
|
| 15 |
+
def forward(self, input):
|
| 16 |
+
return self.relu(input)
|
| 17 |
+
|
| 18 |
+
def to_kaldi_nnet(self):
|
| 19 |
+
re_str = ''
|
| 20 |
+
re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
|
| 21 |
+
return re_str
|
| 22 |
+
|
| 23 |
+
def load_kaldi_nnet(self, instr):
|
| 24 |
+
return instr
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class LogSoftmax(LayerBase):
|
| 28 |
+
|
| 29 |
+
def __init__(self, input_dim, output_dim):
|
| 30 |
+
super(LogSoftmax, self).__init__()
|
| 31 |
+
self.dim = input_dim
|
| 32 |
+
self.ls = nn.LogSoftmax()
|
| 33 |
+
|
| 34 |
+
def forward(self, input):
|
| 35 |
+
return self.ls(input)
|
| 36 |
+
|
| 37 |
+
def to_kaldi_nnet(self):
|
| 38 |
+
re_str = ''
|
| 39 |
+
re_str += '<Softmax> %d %d\n' % (self.dim, self.dim)
|
| 40 |
+
return re_str
|
| 41 |
+
|
| 42 |
+
def load_kaldi_nnet(self, instr):
|
| 43 |
+
return instr
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class Sigmoid(LayerBase):
|
| 47 |
+
|
| 48 |
+
def __init__(self, input_dim, output_dim):
|
| 49 |
+
super(Sigmoid, self).__init__()
|
| 50 |
+
self.dim = input_dim
|
| 51 |
+
self.sig = nn.Sigmoid()
|
| 52 |
+
|
| 53 |
+
def forward(self, input):
|
| 54 |
+
return self.sig(input)
|
| 55 |
+
|
| 56 |
+
def to_kaldi_nnet(self):
|
| 57 |
+
re_str = ''
|
| 58 |
+
re_str += '<Sigmoid> %d %d\n' % (self.dim, self.dim)
|
| 59 |
+
return re_str
|
| 60 |
+
|
| 61 |
+
def load_kaldi_nnet(self, instr):
|
| 62 |
+
return instr
|
runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch as th
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
|
| 6 |
+
from modelscope.models.audio.ans.layers.layer_base import (LayerBase,
|
| 7 |
+
to_kaldi_matrix)
|
| 8 |
+
from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix,
|
| 9 |
+
expect_token_number)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AffineTransform(LayerBase):
|
| 13 |
+
|
| 14 |
+
def __init__(self, input_dim, output_dim):
|
| 15 |
+
super(AffineTransform, self).__init__()
|
| 16 |
+
self.input_dim = input_dim
|
| 17 |
+
self.output_dim = output_dim
|
| 18 |
+
|
| 19 |
+
self.linear = nn.Linear(input_dim, output_dim)
|
| 20 |
+
|
| 21 |
+
def forward(self, input):
|
| 22 |
+
return self.linear(input)
|
| 23 |
+
|
| 24 |
+
def to_kaldi_nnet(self):
|
| 25 |
+
re_str = ''
|
| 26 |
+
|
| 27 |
+
re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
|
| 28 |
+
self.input_dim)
|
| 29 |
+
|
| 30 |
+
re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
|
| 31 |
+
|
| 32 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 33 |
+
|
| 34 |
+
x = linear_weights.squeeze().numpy()
|
| 35 |
+
|
| 36 |
+
re_str += to_kaldi_matrix(x)
|
| 37 |
+
|
| 38 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 39 |
+
|
| 40 |
+
x = linear_bias.squeeze().numpy()
|
| 41 |
+
|
| 42 |
+
re_str += to_kaldi_matrix(x)
|
| 43 |
+
|
| 44 |
+
return re_str
|
| 45 |
+
|
| 46 |
+
def load_kaldi_nnet(self, instr):
|
| 47 |
+
|
| 48 |
+
output = expect_token_number(
|
| 49 |
+
instr,
|
| 50 |
+
'<LearnRateCoef>',
|
| 51 |
+
)
|
| 52 |
+
if output is None:
|
| 53 |
+
raise Exception('AffineTransform format error')
|
| 54 |
+
|
| 55 |
+
instr, lr = output
|
| 56 |
+
|
| 57 |
+
output = expect_token_number(instr, '<BiasLearnRateCoef>')
|
| 58 |
+
if output is None:
|
| 59 |
+
raise Exception('AffineTransform format error')
|
| 60 |
+
|
| 61 |
+
instr, lr = output
|
| 62 |
+
|
| 63 |
+
output = expect_token_number(instr, '<MaxNorm>')
|
| 64 |
+
if output is None:
|
| 65 |
+
raise Exception('AffineTransform format error')
|
| 66 |
+
|
| 67 |
+
instr, lr = output
|
| 68 |
+
|
| 69 |
+
output = expect_kaldi_matrix(instr)
|
| 70 |
+
|
| 71 |
+
if output is None:
|
| 72 |
+
raise Exception('AffineTransform format error')
|
| 73 |
+
|
| 74 |
+
instr, mat = output
|
| 75 |
+
|
| 76 |
+
self.linear.weight = th.nn.Parameter(
|
| 77 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 78 |
+
|
| 79 |
+
output = expect_kaldi_matrix(instr)
|
| 80 |
+
if output is None:
|
| 81 |
+
raise Exception('AffineTransform format error')
|
| 82 |
+
|
| 83 |
+
instr, mat = output
|
| 84 |
+
self.linear.bias = th.nn.Parameter(
|
| 85 |
+
th.from_numpy(mat).type(th.FloatTensor))
|
| 86 |
+
return instr
|
runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import abc
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import six
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def to_kaldi_matrix(np_mat):
|
| 11 |
+
""" function that transform as str numpy mat to standard kaldi str matrix
|
| 12 |
+
|
| 13 |
+
Args:
|
| 14 |
+
np_mat: numpy mat
|
| 15 |
+
"""
|
| 16 |
+
np.set_printoptions(threshold=np.inf, linewidth=np.nan)
|
| 17 |
+
out_str = str(np_mat)
|
| 18 |
+
out_str = out_str.replace('[', '')
|
| 19 |
+
out_str = out_str.replace(']', '')
|
| 20 |
+
return '[ %s ]\n' % out_str
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@six.add_metaclass(abc.ABCMeta)
|
| 24 |
+
class LayerBase(nn.Module):
|
| 25 |
+
|
| 26 |
+
def __init__(self):
|
| 27 |
+
super(LayerBase, self).__init__()
|
| 28 |
+
|
| 29 |
+
@abc.abstractmethod
|
| 30 |
+
def to_kaldi_nnet(self):
|
| 31 |
+
pass
|
runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
|
| 8 |
+
from modelscope.models.audio.ans.layers.layer_base import (LayerBase,
|
| 9 |
+
to_kaldi_matrix)
|
| 10 |
+
from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix,
|
| 11 |
+
expect_token_number)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class UniDeepFsmn(LayerBase):
|
| 15 |
+
|
| 16 |
+
def __init__(self, input_dim, output_dim, lorder=1, hidden_size=None):
|
| 17 |
+
super(UniDeepFsmn, self).__init__()
|
| 18 |
+
|
| 19 |
+
self.input_dim = input_dim
|
| 20 |
+
self.output_dim = output_dim
|
| 21 |
+
self.lorder = lorder
|
| 22 |
+
self.hidden_size = hidden_size
|
| 23 |
+
|
| 24 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 25 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 26 |
+
self.conv1 = nn.Conv2d(
|
| 27 |
+
output_dim,
|
| 28 |
+
output_dim, (lorder, 1), (1, 1),
|
| 29 |
+
groups=output_dim,
|
| 30 |
+
bias=False)
|
| 31 |
+
|
| 32 |
+
def forward(self, input):
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
Args:
|
| 36 |
+
input: torch with shape: batch (b) x sequence(T) x feature (h)
|
| 37 |
+
|
| 38 |
+
Returns:
|
| 39 |
+
batch (b) x channel (c) x sequence(T) x feature (h)
|
| 40 |
+
"""
|
| 41 |
+
f1 = F.relu(self.linear(input))
|
| 42 |
+
p1 = self.project(f1)
|
| 43 |
+
x = torch.unsqueeze(p1, 1)
|
| 44 |
+
# x: batch (b) x channel (c) x sequence(T) x feature (h)
|
| 45 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 46 |
+
# x_per: batch (b) x feature (h) x sequence(T) x channel (c)
|
| 47 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
|
| 48 |
+
|
| 49 |
+
out = x_per + self.conv1(y)
|
| 50 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 51 |
+
# out1: batch (b) x channel (c) x sequence(T) x feature (h)
|
| 52 |
+
return input + out1.squeeze()
|
| 53 |
+
|
| 54 |
+
def to_kaldi_nnet(self):
|
| 55 |
+
re_str = ''
|
| 56 |
+
re_str += '<UniDeepFsmn> %d %d\n'\
|
| 57 |
+
% (self.output_dim, self.input_dim)
|
| 58 |
+
re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n'\
|
| 59 |
+
% (1, self.hidden_size, self.lorder, 1)
|
| 60 |
+
|
| 61 |
+
lfiters = self.state_dict()['conv1.weight']
|
| 62 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 63 |
+
re_str += to_kaldi_matrix(x)
|
| 64 |
+
proj_weights = self.state_dict()['project.weight']
|
| 65 |
+
x = proj_weights.squeeze().numpy()
|
| 66 |
+
re_str += to_kaldi_matrix(x)
|
| 67 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 68 |
+
x = linear_weights.squeeze().numpy()
|
| 69 |
+
re_str += to_kaldi_matrix(x)
|
| 70 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 71 |
+
x = linear_bias.squeeze().numpy()
|
| 72 |
+
re_str += to_kaldi_matrix(x)
|
| 73 |
+
return re_str
|
| 74 |
+
|
| 75 |
+
def load_kaldi_nnet(self, instr):
|
| 76 |
+
output = expect_token_number(
|
| 77 |
+
instr,
|
| 78 |
+
'<LearnRateCoef>',
|
| 79 |
+
)
|
| 80 |
+
if output is None:
|
| 81 |
+
raise Exception('UniDeepFsmn format error')
|
| 82 |
+
instr, lr = output
|
| 83 |
+
|
| 84 |
+
output = expect_token_number(
|
| 85 |
+
instr,
|
| 86 |
+
'<HidSize>',
|
| 87 |
+
)
|
| 88 |
+
if output is None:
|
| 89 |
+
raise Exception('UniDeepFsmn format error')
|
| 90 |
+
instr, hiddensize = output
|
| 91 |
+
self.hidden_size = int(hiddensize)
|
| 92 |
+
|
| 93 |
+
output = expect_token_number(
|
| 94 |
+
instr,
|
| 95 |
+
'<LOrder>',
|
| 96 |
+
)
|
| 97 |
+
if output is None:
|
| 98 |
+
raise Exception('UniDeepFsmn format error')
|
| 99 |
+
instr, lorder = output
|
| 100 |
+
self.lorder = int(lorder)
|
| 101 |
+
|
| 102 |
+
output = expect_token_number(
|
| 103 |
+
instr,
|
| 104 |
+
'<LStride>',
|
| 105 |
+
)
|
| 106 |
+
if output is None:
|
| 107 |
+
raise Exception('UniDeepFsmn format error')
|
| 108 |
+
instr, lstride = output
|
| 109 |
+
self.lstride = lstride
|
| 110 |
+
|
| 111 |
+
output = expect_token_number(
|
| 112 |
+
instr,
|
| 113 |
+
'<MaxNorm>',
|
| 114 |
+
)
|
| 115 |
+
if output is None:
|
| 116 |
+
raise Exception('UniDeepFsmn format error')
|
| 117 |
+
|
| 118 |
+
output = expect_kaldi_matrix(instr)
|
| 119 |
+
if output is None:
|
| 120 |
+
raise Exception('Fsmn format error')
|
| 121 |
+
instr, mat = output
|
| 122 |
+
mat1 = np.fliplr(mat.T).copy()
|
| 123 |
+
self.conv1 = nn.Conv2d(
|
| 124 |
+
self.output_dim,
|
| 125 |
+
self.output_dim, (self.lorder, 1), (1, 1),
|
| 126 |
+
groups=self.output_dim,
|
| 127 |
+
bias=False)
|
| 128 |
+
mat_th = torch.from_numpy(mat1).type(torch.FloatTensor)
|
| 129 |
+
mat_th = mat_th.unsqueeze(1)
|
| 130 |
+
mat_th = mat_th.unsqueeze(3)
|
| 131 |
+
self.conv1.weight = torch.nn.Parameter(mat_th)
|
| 132 |
+
|
| 133 |
+
output = expect_kaldi_matrix(instr)
|
| 134 |
+
if output is None:
|
| 135 |
+
raise Exception('UniDeepFsmn format error')
|
| 136 |
+
instr, mat = output
|
| 137 |
+
|
| 138 |
+
self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
|
| 139 |
+
self.linear = nn.Linear(self.input_dim, self.hidden_size)
|
| 140 |
+
self.project.weight = torch.nn.Parameter(
|
| 141 |
+
torch.from_numpy(mat).type(torch.FloatTensor))
|
| 142 |
+
|
| 143 |
+
output = expect_kaldi_matrix(instr)
|
| 144 |
+
if output is None:
|
| 145 |
+
raise Exception('UniDeepFsmn format error')
|
| 146 |
+
instr, mat = output
|
| 147 |
+
self.linear.weight = torch.nn.Parameter(
|
| 148 |
+
torch.from_numpy(mat).type(torch.FloatTensor))
|
| 149 |
+
|
| 150 |
+
output = expect_kaldi_matrix(instr)
|
| 151 |
+
if output is None:
|
| 152 |
+
raise Exception('UniDeepFsmn format error')
|
| 153 |
+
instr, mat = output
|
| 154 |
+
self.linear.bias = torch.nn.Parameter(
|
| 155 |
+
torch.from_numpy(mat).type(torch.FloatTensor))
|
| 156 |
+
return instr
|
runtime/Lib/site_packages/models/audio/ans/se_module_complex.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
import torch
|
| 3 |
+
from torch import nn
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class SELayer(nn.Module):
|
| 7 |
+
|
| 8 |
+
def __init__(self, channel, reduction=16):
|
| 9 |
+
super(SELayer, self).__init__()
|
| 10 |
+
self.avg_pool = nn.AdaptiveAvgPool2d(1)
|
| 11 |
+
self.fc_r = nn.Sequential(
|
| 12 |
+
nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True),
|
| 13 |
+
nn.Linear(channel // reduction, channel), nn.Sigmoid())
|
| 14 |
+
self.fc_i = nn.Sequential(
|
| 15 |
+
nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True),
|
| 16 |
+
nn.Linear(channel // reduction, channel), nn.Sigmoid())
|
| 17 |
+
|
| 18 |
+
def forward(self, x):
|
| 19 |
+
b, c, _, _, _ = x.size()
|
| 20 |
+
x_r = self.avg_pool(x[:, :, :, :, 0]).view(b, c)
|
| 21 |
+
x_i = self.avg_pool(x[:, :, :, :, 1]).view(b, c)
|
| 22 |
+
y_r = self.fc_r(x_r).view(b, c, 1, 1, 1) - self.fc_i(x_i).view(
|
| 23 |
+
b, c, 1, 1, 1)
|
| 24 |
+
y_i = self.fc_r(x_i).view(b, c, 1, 1, 1) + self.fc_i(x_r).view(
|
| 25 |
+
b, c, 1, 1, 1)
|
| 26 |
+
y = torch.cat([y_r, y_i], 4)
|
| 27 |
+
return x * y
|
runtime/Lib/site_packages/models/audio/ans/unet.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# The implementation here is modified based on
|
| 4 |
+
# Jongho Choi(sweetcocoa@snu.ac.kr / Seoul National Univ., ESTsoft )
|
| 5 |
+
# and publicly available at
|
| 6 |
+
# https://github.com/sweetcocoa/DeepComplexUNetPyTorch
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
|
| 11 |
+
from . import complex_nn
|
| 12 |
+
from .se_module_complex import SELayer
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class Encoder(nn.Module):
|
| 16 |
+
|
| 17 |
+
def __init__(self,
|
| 18 |
+
in_channels,
|
| 19 |
+
out_channels,
|
| 20 |
+
kernel_size,
|
| 21 |
+
stride,
|
| 22 |
+
padding=None,
|
| 23 |
+
complex=False,
|
| 24 |
+
padding_mode='zeros'):
|
| 25 |
+
super().__init__()
|
| 26 |
+
if padding is None:
|
| 27 |
+
padding = [(i - 1) // 2 for i in kernel_size] # 'SAME' padding
|
| 28 |
+
|
| 29 |
+
if complex:
|
| 30 |
+
conv = complex_nn.ComplexConv2d
|
| 31 |
+
bn = complex_nn.ComplexBatchNorm2d
|
| 32 |
+
else:
|
| 33 |
+
conv = nn.Conv2d
|
| 34 |
+
bn = nn.BatchNorm2d
|
| 35 |
+
|
| 36 |
+
self.conv = conv(
|
| 37 |
+
in_channels,
|
| 38 |
+
out_channels,
|
| 39 |
+
kernel_size=kernel_size,
|
| 40 |
+
stride=stride,
|
| 41 |
+
padding=padding,
|
| 42 |
+
padding_mode=padding_mode)
|
| 43 |
+
self.bn = bn(out_channels)
|
| 44 |
+
self.relu = nn.LeakyReLU(inplace=True)
|
| 45 |
+
|
| 46 |
+
def forward(self, x):
|
| 47 |
+
x = self.conv(x)
|
| 48 |
+
x = self.bn(x)
|
| 49 |
+
x = self.relu(x)
|
| 50 |
+
return x
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class Decoder(nn.Module):
|
| 54 |
+
|
| 55 |
+
def __init__(self,
|
| 56 |
+
in_channels,
|
| 57 |
+
out_channels,
|
| 58 |
+
kernel_size,
|
| 59 |
+
stride,
|
| 60 |
+
padding=(0, 0),
|
| 61 |
+
complex=False):
|
| 62 |
+
super().__init__()
|
| 63 |
+
if complex:
|
| 64 |
+
tconv = complex_nn.ComplexConvTranspose2d
|
| 65 |
+
bn = complex_nn.ComplexBatchNorm2d
|
| 66 |
+
else:
|
| 67 |
+
tconv = nn.ConvTranspose2d
|
| 68 |
+
bn = nn.BatchNorm2d
|
| 69 |
+
|
| 70 |
+
self.transconv = tconv(
|
| 71 |
+
in_channels,
|
| 72 |
+
out_channels,
|
| 73 |
+
kernel_size=kernel_size,
|
| 74 |
+
stride=stride,
|
| 75 |
+
padding=padding)
|
| 76 |
+
self.bn = bn(out_channels)
|
| 77 |
+
self.relu = nn.LeakyReLU(inplace=True)
|
| 78 |
+
|
| 79 |
+
def forward(self, x):
|
| 80 |
+
x = self.transconv(x)
|
| 81 |
+
x = self.bn(x)
|
| 82 |
+
x = self.relu(x)
|
| 83 |
+
return x
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class UNet(nn.Module):
|
| 87 |
+
|
| 88 |
+
def __init__(self,
|
| 89 |
+
input_channels=1,
|
| 90 |
+
complex=False,
|
| 91 |
+
model_complexity=45,
|
| 92 |
+
model_depth=20,
|
| 93 |
+
padding_mode='zeros'):
|
| 94 |
+
super().__init__()
|
| 95 |
+
|
| 96 |
+
if complex:
|
| 97 |
+
model_complexity = int(model_complexity // 1.414)
|
| 98 |
+
|
| 99 |
+
self.set_size(
|
| 100 |
+
model_complexity=model_complexity,
|
| 101 |
+
input_channels=input_channels,
|
| 102 |
+
model_depth=model_depth)
|
| 103 |
+
self.encoders = []
|
| 104 |
+
self.model_length = model_depth // 2
|
| 105 |
+
self.fsmn = complex_nn.ComplexUniDeepFsmn(128, 128, 128)
|
| 106 |
+
self.se_layers_enc = []
|
| 107 |
+
self.fsmn_enc = []
|
| 108 |
+
for i in range(self.model_length):
|
| 109 |
+
fsmn_enc = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128)
|
| 110 |
+
self.add_module('fsmn_enc{}'.format(i), fsmn_enc)
|
| 111 |
+
self.fsmn_enc.append(fsmn_enc)
|
| 112 |
+
module = Encoder(
|
| 113 |
+
self.enc_channels[i],
|
| 114 |
+
self.enc_channels[i + 1],
|
| 115 |
+
kernel_size=self.enc_kernel_sizes[i],
|
| 116 |
+
stride=self.enc_strides[i],
|
| 117 |
+
padding=self.enc_paddings[i],
|
| 118 |
+
complex=complex,
|
| 119 |
+
padding_mode=padding_mode)
|
| 120 |
+
self.add_module('encoder{}'.format(i), module)
|
| 121 |
+
self.encoders.append(module)
|
| 122 |
+
se_layer_enc = SELayer(self.enc_channels[i + 1], 8)
|
| 123 |
+
self.add_module('se_layer_enc{}'.format(i), se_layer_enc)
|
| 124 |
+
self.se_layers_enc.append(se_layer_enc)
|
| 125 |
+
self.decoders = []
|
| 126 |
+
self.fsmn_dec = []
|
| 127 |
+
self.se_layers_dec = []
|
| 128 |
+
for i in range(self.model_length):
|
| 129 |
+
fsmn_dec = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128)
|
| 130 |
+
self.add_module('fsmn_dec{}'.format(i), fsmn_dec)
|
| 131 |
+
self.fsmn_dec.append(fsmn_dec)
|
| 132 |
+
module = Decoder(
|
| 133 |
+
self.dec_channels[i] * 2,
|
| 134 |
+
self.dec_channels[i + 1],
|
| 135 |
+
kernel_size=self.dec_kernel_sizes[i],
|
| 136 |
+
stride=self.dec_strides[i],
|
| 137 |
+
padding=self.dec_paddings[i],
|
| 138 |
+
complex=complex)
|
| 139 |
+
self.add_module('decoder{}'.format(i), module)
|
| 140 |
+
self.decoders.append(module)
|
| 141 |
+
if i < self.model_length - 1:
|
| 142 |
+
se_layer_dec = SELayer(self.dec_channels[i + 1], 8)
|
| 143 |
+
self.add_module('se_layer_dec{}'.format(i), se_layer_dec)
|
| 144 |
+
self.se_layers_dec.append(se_layer_dec)
|
| 145 |
+
if complex:
|
| 146 |
+
conv = complex_nn.ComplexConv2d
|
| 147 |
+
else:
|
| 148 |
+
conv = nn.Conv2d
|
| 149 |
+
|
| 150 |
+
linear = conv(self.dec_channels[-1], 1, 1)
|
| 151 |
+
|
| 152 |
+
self.add_module('linear', linear)
|
| 153 |
+
self.complex = complex
|
| 154 |
+
self.padding_mode = padding_mode
|
| 155 |
+
|
| 156 |
+
self.decoders = nn.ModuleList(self.decoders)
|
| 157 |
+
self.encoders = nn.ModuleList(self.encoders)
|
| 158 |
+
self.se_layers_enc = nn.ModuleList(self.se_layers_enc)
|
| 159 |
+
self.se_layers_dec = nn.ModuleList(self.se_layers_dec)
|
| 160 |
+
self.fsmn_enc = nn.ModuleList(self.fsmn_enc)
|
| 161 |
+
self.fsmn_dec = nn.ModuleList(self.fsmn_dec)
|
| 162 |
+
|
| 163 |
+
def forward(self, inputs):
|
| 164 |
+
x = inputs
|
| 165 |
+
# go down
|
| 166 |
+
xs = []
|
| 167 |
+
xs_se = []
|
| 168 |
+
xs_se.append(x)
|
| 169 |
+
for i, encoder in enumerate(self.encoders):
|
| 170 |
+
xs.append(x)
|
| 171 |
+
if i > 0:
|
| 172 |
+
x = self.fsmn_enc[i](x)
|
| 173 |
+
x = encoder(x)
|
| 174 |
+
xs_se.append(self.se_layers_enc[i](x))
|
| 175 |
+
# xs : x0=input x1 ... x9
|
| 176 |
+
x = self.fsmn(x)
|
| 177 |
+
|
| 178 |
+
p = x
|
| 179 |
+
for i, decoder in enumerate(self.decoders):
|
| 180 |
+
p = decoder(p)
|
| 181 |
+
if i < self.model_length - 1:
|
| 182 |
+
p = self.fsmn_dec[i](p)
|
| 183 |
+
if i == self.model_length - 1:
|
| 184 |
+
break
|
| 185 |
+
if i < self.model_length - 2:
|
| 186 |
+
p = self.se_layers_dec[i](p)
|
| 187 |
+
p = torch.cat([p, xs_se[self.model_length - 1 - i]], dim=1)
|
| 188 |
+
|
| 189 |
+
# cmp_spec: [12, 1, 513, 64, 2]
|
| 190 |
+
cmp_spec = self.linear(p)
|
| 191 |
+
return cmp_spec
|
| 192 |
+
|
| 193 |
+
def set_size(self, model_complexity, model_depth=20, input_channels=1):
|
| 194 |
+
|
| 195 |
+
if model_depth == 14:
|
| 196 |
+
self.enc_channels = [
|
| 197 |
+
input_channels, 128, 128, 128, 128, 128, 128, 128
|
| 198 |
+
]
|
| 199 |
+
self.enc_kernel_sizes = [(5, 2), (5, 2), (5, 2), (5, 2), (5, 2),
|
| 200 |
+
(5, 2), (2, 2)]
|
| 201 |
+
self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1),
|
| 202 |
+
(2, 1)]
|
| 203 |
+
self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1),
|
| 204 |
+
(0, 1), (0, 1)]
|
| 205 |
+
self.dec_channels = [64, 128, 128, 128, 128, 128, 128, 1]
|
| 206 |
+
self.dec_kernel_sizes = [(2, 2), (5, 2), (5, 2), (5, 2), (6, 2),
|
| 207 |
+
(5, 2), (5, 2)]
|
| 208 |
+
self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1),
|
| 209 |
+
(2, 1)]
|
| 210 |
+
self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1),
|
| 211 |
+
(0, 1), (0, 1)]
|
| 212 |
+
|
| 213 |
+
elif model_depth == 10:
|
| 214 |
+
self.enc_channels = [
|
| 215 |
+
input_channels,
|
| 216 |
+
16,
|
| 217 |
+
32,
|
| 218 |
+
64,
|
| 219 |
+
128,
|
| 220 |
+
256,
|
| 221 |
+
]
|
| 222 |
+
self.enc_kernel_sizes = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3)]
|
| 223 |
+
self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)]
|
| 224 |
+
self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)]
|
| 225 |
+
self.dec_channels = [128, 128, 64, 32, 16, 1]
|
| 226 |
+
self.dec_kernel_sizes = [(3, 3), (3, 3), (3, 3), (4, 3), (3, 3)]
|
| 227 |
+
self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)]
|
| 228 |
+
self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)]
|
| 229 |
+
|
| 230 |
+
elif model_depth == 20:
|
| 231 |
+
self.enc_channels = [
|
| 232 |
+
input_channels, model_complexity, model_complexity,
|
| 233 |
+
model_complexity * 2, model_complexity * 2,
|
| 234 |
+
model_complexity * 2, model_complexity * 2,
|
| 235 |
+
model_complexity * 2, model_complexity * 2,
|
| 236 |
+
model_complexity * 2, 128
|
| 237 |
+
]
|
| 238 |
+
|
| 239 |
+
self.enc_kernel_sizes = [(7, 1), (1, 7), (6, 4), (7, 5), (5, 3),
|
| 240 |
+
(5, 3), (5, 3), (5, 3), (5, 3), (5, 3)]
|
| 241 |
+
|
| 242 |
+
self.enc_strides = [(1, 1), (1, 1), (2, 2), (2, 1), (2, 2), (2, 1),
|
| 243 |
+
(2, 2), (2, 1), (2, 2), (2, 1)]
|
| 244 |
+
|
| 245 |
+
self.enc_paddings = [
|
| 246 |
+
(3, 0),
|
| 247 |
+
(0, 3),
|
| 248 |
+
None, # (0, 2),
|
| 249 |
+
None,
|
| 250 |
+
None, # (3,1),
|
| 251 |
+
None, # (3,1),
|
| 252 |
+
None, # (1,2),
|
| 253 |
+
None,
|
| 254 |
+
None,
|
| 255 |
+
None
|
| 256 |
+
]
|
| 257 |
+
|
| 258 |
+
self.dec_channels = [
|
| 259 |
+
0, model_complexity * 2, model_complexity * 2,
|
| 260 |
+
model_complexity * 2, model_complexity * 2,
|
| 261 |
+
model_complexity * 2, model_complexity * 2,
|
| 262 |
+
model_complexity * 2, model_complexity * 2,
|
| 263 |
+
model_complexity * 2, model_complexity * 2,
|
| 264 |
+
model_complexity * 2
|
| 265 |
+
]
|
| 266 |
+
|
| 267 |
+
self.dec_kernel_sizes = [(4, 3), (4, 2), (4, 3), (4, 2), (4, 3),
|
| 268 |
+
(4, 2), (6, 3), (7, 4), (1, 7), (7, 1)]
|
| 269 |
+
|
| 270 |
+
self.dec_strides = [(2, 1), (2, 2), (2, 1), (2, 2), (2, 1), (2, 2),
|
| 271 |
+
(2, 1), (2, 2), (1, 1), (1, 1)]
|
| 272 |
+
|
| 273 |
+
self.dec_paddings = [(1, 1), (1, 0), (1, 1), (1, 0), (1, 1),
|
| 274 |
+
(1, 0), (2, 1), (2, 1), (0, 3), (3, 0)]
|
| 275 |
+
else:
|
| 276 |
+
raise ValueError('Unknown model depth : {}'.format(model_depth))
|
runtime/Lib/site_packages/models/audio/asr/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.import_utils import LazyImportModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .generic_automatic_speech_recognition import GenericAutomaticSpeechRecognition
|
| 8 |
+
|
| 9 |
+
else:
|
| 10 |
+
_import_structure = {
|
| 11 |
+
'generic_automatic_speech_recognition':
|
| 12 |
+
['GenericAutomaticSpeechRecognition'],
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
|
| 17 |
+
sys.modules[__name__] = LazyImportModule(
|
| 18 |
+
__name__,
|
| 19 |
+
globals()['__file__'],
|
| 20 |
+
_import_structure,
|
| 21 |
+
module_spec=__spec__,
|
| 22 |
+
extra_objects={},
|
| 23 |
+
)
|
runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
from modelscope.metainfo import Models
|
| 7 |
+
from modelscope.models.base import Model
|
| 8 |
+
from modelscope.models.builder import MODELS
|
| 9 |
+
from modelscope.utils.constant import Frameworks, Tasks
|
| 10 |
+
|
| 11 |
+
__all__ = ['GenericAutomaticSpeechRecognition']
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@MODELS.register_module(
|
| 15 |
+
Tasks.auto_speech_recognition, module_name=Models.generic_asr)
|
| 16 |
+
@MODELS.register_module(
|
| 17 |
+
Tasks.voice_activity_detection, module_name=Models.generic_asr)
|
| 18 |
+
@MODELS.register_module(
|
| 19 |
+
Tasks.speech_separation, module_name=Models.generic_asr)
|
| 20 |
+
@MODELS.register_module(
|
| 21 |
+
Tasks.language_score_prediction, module_name=Models.generic_asr)
|
| 22 |
+
@MODELS.register_module(Tasks.speech_timestamp, module_name=Models.generic_asr)
|
| 23 |
+
class GenericAutomaticSpeechRecognition(Model):
|
| 24 |
+
|
| 25 |
+
def __init__(self, model_dir: str, am_model_name: str,
|
| 26 |
+
model_config: Dict[str, Any], *args, **kwargs):
|
| 27 |
+
"""initialize the info of model.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
model_dir (str): the model path.
|
| 31 |
+
am_model_name (str): the am model name from configuration.json
|
| 32 |
+
model_config (Dict[str, Any]): the detail config about model from configuration.json
|
| 33 |
+
"""
|
| 34 |
+
super().__init__(model_dir, am_model_name, model_config, *args,
|
| 35 |
+
**kwargs)
|
| 36 |
+
self.model_cfg = {
|
| 37 |
+
# the recognition model dir path
|
| 38 |
+
'model_workspace': model_dir,
|
| 39 |
+
# the am model name
|
| 40 |
+
'am_model': am_model_name,
|
| 41 |
+
# the am model file path
|
| 42 |
+
'am_model_path': os.path.join(model_dir, am_model_name),
|
| 43 |
+
# the recognition model config dict
|
| 44 |
+
'model_config': model_config
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
def forward(self) -> Dict[str, Any]:
|
| 48 |
+
"""preload model and return the info of the model
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
return self.model_cfg
|
runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
+
import wenetruntime as wenet
|
| 8 |
+
|
| 9 |
+
from modelscope.metainfo import Models
|
| 10 |
+
from modelscope.models.base import Model
|
| 11 |
+
from modelscope.models.builder import MODELS
|
| 12 |
+
from modelscope.utils.constant import Tasks
|
| 13 |
+
|
| 14 |
+
__all__ = ['WeNetAutomaticSpeechRecognition']
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@MODELS.register_module(
|
| 18 |
+
Tasks.auto_speech_recognition, module_name=Models.wenet_asr)
|
| 19 |
+
class WeNetAutomaticSpeechRecognition(Model):
|
| 20 |
+
|
| 21 |
+
def __init__(self, model_dir: str, am_model_name: str,
|
| 22 |
+
model_config: Dict[str, Any], *args, **kwargs):
|
| 23 |
+
"""initialize the info of model.
|
| 24 |
+
|
| 25 |
+
Args:
|
| 26 |
+
model_dir (str): the model path.
|
| 27 |
+
"""
|
| 28 |
+
super().__init__(model_dir, am_model_name, model_config, *args,
|
| 29 |
+
**kwargs)
|
| 30 |
+
self.decoder = wenet.Decoder(model_dir, lang='chs')
|
| 31 |
+
|
| 32 |
+
def forward(self, inputs: Dict[str, Any]) -> str:
|
| 33 |
+
if inputs['audio_format'] == 'wav':
|
| 34 |
+
rst = self.decoder.decode_wav(inputs['audio'])
|
| 35 |
+
else:
|
| 36 |
+
rst = self.decoder.decode(inputs['audio'])
|
| 37 |
+
text = json.loads(rst)['nbest'][0]['sentence']
|
| 38 |
+
return {'text': text}
|
runtime/Lib/site_packages/models/audio/itn/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.import_utils import LazyImportModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .generic_inverse_text_processing import GenericInverseTextProcessing
|
| 8 |
+
|
| 9 |
+
else:
|
| 10 |
+
_import_structure = {
|
| 11 |
+
'generic_inverse_text_processing': ['GenericInverseTextProcessing'],
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
import sys
|
| 15 |
+
|
| 16 |
+
sys.modules[__name__] = LazyImportModule(
|
| 17 |
+
__name__,
|
| 18 |
+
globals()['__file__'],
|
| 19 |
+
_import_structure,
|
| 20 |
+
module_spec=__spec__,
|
| 21 |
+
extra_objects={},
|
| 22 |
+
)
|
runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
from modelscope.metainfo import Models
|
| 7 |
+
from modelscope.models.base import Model
|
| 8 |
+
from modelscope.models.builder import MODELS
|
| 9 |
+
from modelscope.utils.constant import Frameworks, Tasks
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@MODELS.register_module(
|
| 13 |
+
Tasks.inverse_text_processing, module_name=Models.generic_itn)
|
| 14 |
+
class GenericInverseTextProcessing(Model):
|
| 15 |
+
|
| 16 |
+
def __init__(self, model_dir: str, itn_model_name: str,
|
| 17 |
+
model_config: Dict[str, Any], *args, **kwargs):
|
| 18 |
+
"""initialize the info of model.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
model_dir (str): the model path.
|
| 22 |
+
itn_model_name (str): the itn model name from configuration.json
|
| 23 |
+
model_config (Dict[str, Any]): the detail config about model from configuration.json
|
| 24 |
+
"""
|
| 25 |
+
super().__init__(model_dir, itn_model_name, model_config, *args,
|
| 26 |
+
**kwargs)
|
| 27 |
+
self.model_cfg = {
|
| 28 |
+
# the recognition model dir path
|
| 29 |
+
'model_workspace': model_dir,
|
| 30 |
+
# the itn model name
|
| 31 |
+
'itn_model': itn_model_name,
|
| 32 |
+
# the am model file path
|
| 33 |
+
'itn_model_path': os.path.join(model_dir, itn_model_name),
|
| 34 |
+
# the recognition model config dict
|
| 35 |
+
'model_config': model_config
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
def forward(self) -> Dict[str, Any]:
|
| 39 |
+
"""
|
| 40 |
+
just return the model config
|
| 41 |
+
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
return self.model_cfg
|
runtime/Lib/site_packages/models/audio/kws/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.import_utils import LazyImportModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .generic_key_word_spotting import GenericKeyWordSpotting
|
| 8 |
+
from .farfield.model import FSMNSeleNetV2Decorator
|
| 9 |
+
from .nearfield.model import FSMNDecorator
|
| 10 |
+
|
| 11 |
+
else:
|
| 12 |
+
_import_structure = {
|
| 13 |
+
'generic_key_word_spotting': ['GenericKeyWordSpotting'],
|
| 14 |
+
'farfield.model': ['FSMNSeleNetV2Decorator'],
|
| 15 |
+
'nearfield.model': ['FSMNDecorator'],
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
import sys
|
| 19 |
+
|
| 20 |
+
sys.modules[__name__] = LazyImportModule(
|
| 21 |
+
__name__,
|
| 22 |
+
globals()['__file__'],
|
| 23 |
+
_import_structure,
|
| 24 |
+
module_spec=__spec__,
|
| 25 |
+
extra_objects={},
|
| 26 |
+
)
|
runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
|
| 8 |
+
from .model_def import (HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32,
|
| 9 |
+
printNeonMatrix, printNeonVector)
|
| 10 |
+
|
| 11 |
+
DEBUG = False
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def to_kaldi_matrix(np_mat):
|
| 15 |
+
""" function that transform as str numpy mat to standard kaldi str matrix
|
| 16 |
+
|
| 17 |
+
Args:
|
| 18 |
+
np_mat: numpy mat
|
| 19 |
+
|
| 20 |
+
Returns: str
|
| 21 |
+
"""
|
| 22 |
+
np.set_printoptions(threshold=np.inf, linewidth=np.nan)
|
| 23 |
+
out_str = str(np_mat)
|
| 24 |
+
out_str = out_str.replace('[', '')
|
| 25 |
+
out_str = out_str.replace(']', '')
|
| 26 |
+
return '[ %s ]\n' % out_str
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def print_tensor(torch_tensor):
|
| 30 |
+
""" print torch tensor for debug
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
torch_tensor: a tensor
|
| 34 |
+
"""
|
| 35 |
+
re_str = ''
|
| 36 |
+
x = torch_tensor.detach().squeeze().numpy()
|
| 37 |
+
re_str += to_kaldi_matrix(x)
|
| 38 |
+
re_str += '<!EndOfComponent>\n'
|
| 39 |
+
print(re_str)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class LinearTransform(nn.Module):
|
| 43 |
+
|
| 44 |
+
def __init__(self, input_dim, output_dim):
|
| 45 |
+
super(LinearTransform, self).__init__()
|
| 46 |
+
self.input_dim = input_dim
|
| 47 |
+
self.output_dim = output_dim
|
| 48 |
+
self.linear = nn.Linear(input_dim, output_dim, bias=False)
|
| 49 |
+
|
| 50 |
+
self.debug = False
|
| 51 |
+
self.dataout = None
|
| 52 |
+
|
| 53 |
+
def forward(self, input):
|
| 54 |
+
output = self.linear(input)
|
| 55 |
+
|
| 56 |
+
if self.debug:
|
| 57 |
+
self.dataout = output
|
| 58 |
+
|
| 59 |
+
return output
|
| 60 |
+
|
| 61 |
+
def print_model(self):
|
| 62 |
+
printNeonMatrix(self.linear.weight)
|
| 63 |
+
|
| 64 |
+
def to_kaldi_nnet(self):
|
| 65 |
+
re_str = ''
|
| 66 |
+
re_str += '<LinearTransform> %d %d\n' % (self.output_dim,
|
| 67 |
+
self.input_dim)
|
| 68 |
+
re_str += '<LearnRateCoef> 1\n'
|
| 69 |
+
|
| 70 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 71 |
+
x = linear_weights.squeeze().numpy()
|
| 72 |
+
re_str += to_kaldi_matrix(x)
|
| 73 |
+
re_str += '<!EndOfComponent>\n'
|
| 74 |
+
|
| 75 |
+
return re_str
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class AffineTransform(nn.Module):
|
| 79 |
+
|
| 80 |
+
def __init__(self, input_dim, output_dim):
|
| 81 |
+
super(AffineTransform, self).__init__()
|
| 82 |
+
self.input_dim = input_dim
|
| 83 |
+
self.output_dim = output_dim
|
| 84 |
+
|
| 85 |
+
self.linear = nn.Linear(input_dim, output_dim)
|
| 86 |
+
|
| 87 |
+
self.debug = False
|
| 88 |
+
self.dataout = None
|
| 89 |
+
|
| 90 |
+
def forward(self, input):
|
| 91 |
+
output = self.linear(input)
|
| 92 |
+
|
| 93 |
+
if self.debug:
|
| 94 |
+
self.dataout = output
|
| 95 |
+
|
| 96 |
+
return output
|
| 97 |
+
|
| 98 |
+
def print_model(self):
|
| 99 |
+
printNeonMatrix(self.linear.weight)
|
| 100 |
+
printNeonVector(self.linear.bias)
|
| 101 |
+
|
| 102 |
+
def to_kaldi_nnet(self):
|
| 103 |
+
re_str = ''
|
| 104 |
+
re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
|
| 105 |
+
self.input_dim)
|
| 106 |
+
re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
|
| 107 |
+
|
| 108 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 109 |
+
x = linear_weights.squeeze().numpy()
|
| 110 |
+
re_str += to_kaldi_matrix(x)
|
| 111 |
+
|
| 112 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 113 |
+
x = linear_bias.squeeze().numpy()
|
| 114 |
+
re_str += to_kaldi_matrix(x)
|
| 115 |
+
re_str += '<!EndOfComponent>\n'
|
| 116 |
+
|
| 117 |
+
return re_str
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class Fsmn(nn.Module):
|
| 121 |
+
"""
|
| 122 |
+
FSMN implementation.
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
def __init__(self,
|
| 126 |
+
input_dim,
|
| 127 |
+
output_dim,
|
| 128 |
+
lorder=None,
|
| 129 |
+
rorder=None,
|
| 130 |
+
lstride=None,
|
| 131 |
+
rstride=None):
|
| 132 |
+
super(Fsmn, self).__init__()
|
| 133 |
+
|
| 134 |
+
self.dim = input_dim
|
| 135 |
+
|
| 136 |
+
if lorder is None:
|
| 137 |
+
return
|
| 138 |
+
|
| 139 |
+
self.lorder = lorder
|
| 140 |
+
self.rorder = rorder
|
| 141 |
+
self.lstride = lstride
|
| 142 |
+
self.rstride = rstride
|
| 143 |
+
|
| 144 |
+
self.conv_left = nn.Conv2d(
|
| 145 |
+
self.dim,
|
| 146 |
+
self.dim, (lorder, 1),
|
| 147 |
+
dilation=(lstride, 1),
|
| 148 |
+
groups=self.dim,
|
| 149 |
+
bias=False)
|
| 150 |
+
|
| 151 |
+
if rorder > 0:
|
| 152 |
+
self.conv_right = nn.Conv2d(
|
| 153 |
+
self.dim,
|
| 154 |
+
self.dim, (rorder, 1),
|
| 155 |
+
dilation=(rstride, 1),
|
| 156 |
+
groups=self.dim,
|
| 157 |
+
bias=False)
|
| 158 |
+
else:
|
| 159 |
+
self.conv_right = None
|
| 160 |
+
|
| 161 |
+
self.debug = False
|
| 162 |
+
self.dataout = None
|
| 163 |
+
|
| 164 |
+
def forward(self, input):
|
| 165 |
+
x = torch.unsqueeze(input, 1)
|
| 166 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 167 |
+
|
| 168 |
+
y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
|
| 169 |
+
|
| 170 |
+
if self.conv_right is not None:
|
| 171 |
+
y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
|
| 172 |
+
y_right = y_right[:, :, self.rstride:, :]
|
| 173 |
+
out = x_per + self.conv_left(y_left) + self.conv_right(y_right)
|
| 174 |
+
else:
|
| 175 |
+
out = x_per + self.conv_left(y_left)
|
| 176 |
+
|
| 177 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 178 |
+
output = out1.squeeze(1)
|
| 179 |
+
|
| 180 |
+
if self.debug:
|
| 181 |
+
self.dataout = output
|
| 182 |
+
|
| 183 |
+
return output
|
| 184 |
+
|
| 185 |
+
def print_model(self):
|
| 186 |
+
tmpw = self.conv_left.weight
|
| 187 |
+
tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
|
| 188 |
+
for j in range(tmpw.shape[0]):
|
| 189 |
+
tmpwm[:, j] = tmpw[j, 0, :, 0]
|
| 190 |
+
|
| 191 |
+
printNeonMatrix(tmpwm)
|
| 192 |
+
|
| 193 |
+
if self.conv_right is not None:
|
| 194 |
+
tmpw = self.conv_right.weight
|
| 195 |
+
tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
|
| 196 |
+
for j in range(tmpw.shape[0]):
|
| 197 |
+
tmpwm[:, j] = tmpw[j, 0, :, 0]
|
| 198 |
+
|
| 199 |
+
printNeonMatrix(tmpwm)
|
| 200 |
+
|
| 201 |
+
def to_kaldi_nnet(self):
|
| 202 |
+
re_str = ''
|
| 203 |
+
re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)
|
| 204 |
+
re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (
|
| 205 |
+
1, self.lorder, self.rorder, self.lstride, self.rstride)
|
| 206 |
+
|
| 207 |
+
lfiters = self.state_dict()['conv_left.weight']
|
| 208 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 209 |
+
re_str += to_kaldi_matrix(x)
|
| 210 |
+
|
| 211 |
+
if self.conv_right is not None:
|
| 212 |
+
rfiters = self.state_dict()['conv_right.weight']
|
| 213 |
+
x = (rfiters.squeeze().numpy().T)
|
| 214 |
+
re_str += to_kaldi_matrix(x)
|
| 215 |
+
re_str += '<!EndOfComponent>\n'
|
| 216 |
+
|
| 217 |
+
return re_str
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
class RectifiedLinear(nn.Module):
|
| 221 |
+
|
| 222 |
+
def __init__(self, input_dim, output_dim):
|
| 223 |
+
super(RectifiedLinear, self).__init__()
|
| 224 |
+
self.dim = input_dim
|
| 225 |
+
self.relu = nn.ReLU()
|
| 226 |
+
|
| 227 |
+
def forward(self, input):
|
| 228 |
+
return self.relu(input)
|
| 229 |
+
|
| 230 |
+
def to_kaldi_nnet(self):
|
| 231 |
+
re_str = ''
|
| 232 |
+
re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
|
| 233 |
+
re_str += '<!EndOfComponent>\n'
|
| 234 |
+
return re_str
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
class FSMNNet(nn.Module):
|
| 238 |
+
"""
|
| 239 |
+
FSMN net for keyword spotting
|
| 240 |
+
"""
|
| 241 |
+
|
| 242 |
+
def __init__(self,
|
| 243 |
+
input_dim=200,
|
| 244 |
+
linear_dim=128,
|
| 245 |
+
proj_dim=128,
|
| 246 |
+
lorder=10,
|
| 247 |
+
rorder=1,
|
| 248 |
+
num_syn=5,
|
| 249 |
+
fsmn_layers=4):
|
| 250 |
+
"""
|
| 251 |
+
Args:
|
| 252 |
+
input_dim: input dimension
|
| 253 |
+
linear_dim: fsmn input dimension
|
| 254 |
+
proj_dim: fsmn projection dimension
|
| 255 |
+
lorder: fsmn left order
|
| 256 |
+
rorder: fsmn right order
|
| 257 |
+
num_syn: output dimension
|
| 258 |
+
fsmn_layers: no. of sequential fsmn layers
|
| 259 |
+
"""
|
| 260 |
+
super(FSMNNet, self).__init__()
|
| 261 |
+
|
| 262 |
+
self.input_dim = input_dim
|
| 263 |
+
self.linear_dim = linear_dim
|
| 264 |
+
self.proj_dim = proj_dim
|
| 265 |
+
self.lorder = lorder
|
| 266 |
+
self.rorder = rorder
|
| 267 |
+
self.num_syn = num_syn
|
| 268 |
+
self.fsmn_layers = fsmn_layers
|
| 269 |
+
|
| 270 |
+
self.linear1 = AffineTransform(input_dim, linear_dim)
|
| 271 |
+
self.relu = RectifiedLinear(linear_dim, linear_dim)
|
| 272 |
+
|
| 273 |
+
self.fsmn = self._build_repeats(linear_dim, proj_dim, lorder, rorder,
|
| 274 |
+
fsmn_layers)
|
| 275 |
+
|
| 276 |
+
self.linear2 = AffineTransform(linear_dim, num_syn)
|
| 277 |
+
|
| 278 |
+
@staticmethod
|
| 279 |
+
def _build_repeats(linear_dim=136,
|
| 280 |
+
proj_dim=68,
|
| 281 |
+
lorder=3,
|
| 282 |
+
rorder=2,
|
| 283 |
+
fsmn_layers=5):
|
| 284 |
+
repeats = [
|
| 285 |
+
nn.Sequential(
|
| 286 |
+
LinearTransform(linear_dim, proj_dim),
|
| 287 |
+
Fsmn(proj_dim, proj_dim, lorder, rorder, 1, 1),
|
| 288 |
+
AffineTransform(proj_dim, linear_dim),
|
| 289 |
+
RectifiedLinear(linear_dim, linear_dim))
|
| 290 |
+
for i in range(fsmn_layers)
|
| 291 |
+
]
|
| 292 |
+
|
| 293 |
+
return nn.Sequential(*repeats)
|
| 294 |
+
|
| 295 |
+
def forward(self, input):
|
| 296 |
+
x1 = self.linear1(input)
|
| 297 |
+
x2 = self.relu(x1)
|
| 298 |
+
x3 = self.fsmn(x2)
|
| 299 |
+
x4 = self.linear2(x3)
|
| 300 |
+
return x4
|
| 301 |
+
|
| 302 |
+
def print_model(self):
|
| 303 |
+
self.linear1.print_model()
|
| 304 |
+
|
| 305 |
+
for layer in self.fsmn:
|
| 306 |
+
layer[0].print_model()
|
| 307 |
+
layer[1].print_model()
|
| 308 |
+
layer[2].print_model()
|
| 309 |
+
|
| 310 |
+
self.linear2.print_model()
|
| 311 |
+
|
| 312 |
+
def print_header(self):
|
| 313 |
+
#
|
| 314 |
+
# write total header
|
| 315 |
+
#
|
| 316 |
+
header = [0.0] * HEADER_BLOCK_SIZE * 4
|
| 317 |
+
# numins
|
| 318 |
+
header[0] = 0.0
|
| 319 |
+
# numouts
|
| 320 |
+
header[1] = 0.0
|
| 321 |
+
# dimins
|
| 322 |
+
header[2] = self.input_dim
|
| 323 |
+
# dimouts
|
| 324 |
+
header[3] = self.num_syn
|
| 325 |
+
# numlayers
|
| 326 |
+
header[4] = 3
|
| 327 |
+
|
| 328 |
+
#
|
| 329 |
+
# write each layer's header
|
| 330 |
+
#
|
| 331 |
+
hidx = 1
|
| 332 |
+
|
| 333 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 334 |
+
LayerType.LAYER_DENSE.value)
|
| 335 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 336 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = self.input_dim
|
| 337 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = self.linear_dim
|
| 338 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 339 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 340 |
+
ActivationType.ACTIVATION_RELU.value)
|
| 341 |
+
hidx += 1
|
| 342 |
+
|
| 343 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 344 |
+
LayerType.LAYER_SEQUENTIAL_FSMN.value)
|
| 345 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 346 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim
|
| 347 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = self.proj_dim
|
| 348 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = self.lorder
|
| 349 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = self.rorder
|
| 350 |
+
header[HEADER_BLOCK_SIZE * hidx + 6] = self.fsmn_layers
|
| 351 |
+
header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0
|
| 352 |
+
hidx += 1
|
| 353 |
+
|
| 354 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 355 |
+
LayerType.LAYER_DENSE.value)
|
| 356 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 357 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim
|
| 358 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = self.num_syn
|
| 359 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 360 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 361 |
+
ActivationType.ACTIVATION_SOFTMAX.value)
|
| 362 |
+
|
| 363 |
+
for h in header:
|
| 364 |
+
print(f32ToI32(h))
|
| 365 |
+
|
| 366 |
+
def to_kaldi_nnet(self):
|
| 367 |
+
re_str = ''
|
| 368 |
+
re_str += '<Nnet>\n'
|
| 369 |
+
re_str += self.linear1.to_kaldi_nnet()
|
| 370 |
+
re_str += self.relu.to_kaldi_nnet()
|
| 371 |
+
|
| 372 |
+
for fsmn in self.fsmn:
|
| 373 |
+
re_str += fsmn[0].to_kaldi_nnet()
|
| 374 |
+
re_str += fsmn[1].to_kaldi_nnet()
|
| 375 |
+
re_str += fsmn[2].to_kaldi_nnet()
|
| 376 |
+
re_str += fsmn[3].to_kaldi_nnet()
|
| 377 |
+
|
| 378 |
+
re_str += self.linear2.to_kaldi_nnet()
|
| 379 |
+
re_str += '<Softmax> %d %d\n' % (self.num_syn, self.num_syn)
|
| 380 |
+
re_str += '<!EndOfComponent>\n'
|
| 381 |
+
re_str += '</Nnet>\n'
|
| 382 |
+
|
| 383 |
+
return re_str
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
class DFSMN(nn.Module):
|
| 387 |
+
"""
|
| 388 |
+
One deep fsmn layer
|
| 389 |
+
"""
|
| 390 |
+
|
| 391 |
+
def __init__(self,
|
| 392 |
+
dimproj=64,
|
| 393 |
+
dimlinear=128,
|
| 394 |
+
lorder=20,
|
| 395 |
+
rorder=1,
|
| 396 |
+
lstride=1,
|
| 397 |
+
rstride=1):
|
| 398 |
+
"""
|
| 399 |
+
Args:
|
| 400 |
+
dimproj: projection dimension, input and output dimension of memory blocks
|
| 401 |
+
dimlinear: dimension of mapping layer
|
| 402 |
+
lorder: left order
|
| 403 |
+
rorder: right order
|
| 404 |
+
lstride: left stride
|
| 405 |
+
rstride: right stride
|
| 406 |
+
"""
|
| 407 |
+
super(DFSMN, self).__init__()
|
| 408 |
+
|
| 409 |
+
self.lorder = lorder
|
| 410 |
+
self.rorder = rorder
|
| 411 |
+
self.lstride = lstride
|
| 412 |
+
self.rstride = rstride
|
| 413 |
+
|
| 414 |
+
self.expand = AffineTransform(dimproj, dimlinear)
|
| 415 |
+
self.shrink = LinearTransform(dimlinear, dimproj)
|
| 416 |
+
|
| 417 |
+
self.conv_left = nn.Conv2d(
|
| 418 |
+
dimproj,
|
| 419 |
+
dimproj, (lorder, 1),
|
| 420 |
+
dilation=(lstride, 1),
|
| 421 |
+
groups=dimproj,
|
| 422 |
+
bias=False)
|
| 423 |
+
|
| 424 |
+
if rorder > 0:
|
| 425 |
+
self.conv_right = nn.Conv2d(
|
| 426 |
+
dimproj,
|
| 427 |
+
dimproj, (rorder, 1),
|
| 428 |
+
dilation=(rstride, 1),
|
| 429 |
+
groups=dimproj,
|
| 430 |
+
bias=False)
|
| 431 |
+
else:
|
| 432 |
+
self.conv_right = None
|
| 433 |
+
|
| 434 |
+
def forward(self, input):
|
| 435 |
+
f1 = F.relu(self.expand(input))
|
| 436 |
+
p1 = self.shrink(f1)
|
| 437 |
+
|
| 438 |
+
x = torch.unsqueeze(p1, 1)
|
| 439 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 440 |
+
|
| 441 |
+
y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
|
| 442 |
+
|
| 443 |
+
if self.conv_right is not None:
|
| 444 |
+
y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
|
| 445 |
+
y_right = y_right[:, :, self.rstride:, :]
|
| 446 |
+
out = x_per + self.conv_left(y_left) + self.conv_right(y_right)
|
| 447 |
+
else:
|
| 448 |
+
out = x_per + self.conv_left(y_left)
|
| 449 |
+
|
| 450 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 451 |
+
output = input + out1.squeeze(1)
|
| 452 |
+
|
| 453 |
+
return output
|
| 454 |
+
|
| 455 |
+
def print_model(self):
|
| 456 |
+
self.expand.print_model()
|
| 457 |
+
self.shrink.print_model()
|
| 458 |
+
|
| 459 |
+
tmpw = self.conv_left.weight
|
| 460 |
+
tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
|
| 461 |
+
for j in range(tmpw.shape[0]):
|
| 462 |
+
tmpwm[:, j] = tmpw[j, 0, :, 0]
|
| 463 |
+
|
| 464 |
+
printNeonMatrix(tmpwm)
|
| 465 |
+
|
| 466 |
+
if self.conv_right is not None:
|
| 467 |
+
tmpw = self.conv_right.weight
|
| 468 |
+
tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
|
| 469 |
+
for j in range(tmpw.shape[0]):
|
| 470 |
+
tmpwm[:, j] = tmpw[j, 0, :, 0]
|
| 471 |
+
|
| 472 |
+
printNeonMatrix(tmpwm)
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def build_dfsmn_repeats(linear_dim=128,
|
| 476 |
+
proj_dim=64,
|
| 477 |
+
lorder=20,
|
| 478 |
+
rorder=1,
|
| 479 |
+
fsmn_layers=6):
|
| 480 |
+
"""
|
| 481 |
+
build stacked dfsmn layers
|
| 482 |
+
Args:
|
| 483 |
+
linear_dim:
|
| 484 |
+
proj_dim:
|
| 485 |
+
lorder:
|
| 486 |
+
rorder:
|
| 487 |
+
fsmn_layers:
|
| 488 |
+
|
| 489 |
+
Returns:
|
| 490 |
+
|
| 491 |
+
"""
|
| 492 |
+
repeats = [
|
| 493 |
+
nn.Sequential(DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1))
|
| 494 |
+
for i in range(fsmn_layers)
|
| 495 |
+
]
|
| 496 |
+
|
| 497 |
+
return nn.Sequential(*repeats)
|
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear
|
| 8 |
+
from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class FSMNUnit(nn.Module):
|
| 12 |
+
""" A multi-channel fsmn unit
|
| 13 |
+
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
def __init__(self, dimlinear=128, dimproj=64, lorder=20, rorder=1):
|
| 17 |
+
"""
|
| 18 |
+
Args:
|
| 19 |
+
dimlinear: input / output dimension
|
| 20 |
+
dimproj: fsmn input / output dimension
|
| 21 |
+
lorder: left order
|
| 22 |
+
rorder: right order
|
| 23 |
+
"""
|
| 24 |
+
super(FSMNUnit, self).__init__()
|
| 25 |
+
|
| 26 |
+
self.shrink = LinearTransform(dimlinear, dimproj)
|
| 27 |
+
self.fsmn = Fsmn(dimproj, dimproj, lorder, rorder, 1, 1)
|
| 28 |
+
self.expand = AffineTransform(dimproj, dimlinear)
|
| 29 |
+
|
| 30 |
+
self.debug = False
|
| 31 |
+
self.dataout = None
|
| 32 |
+
|
| 33 |
+
'''
|
| 34 |
+
batch, time, channel, feature
|
| 35 |
+
'''
|
| 36 |
+
|
| 37 |
+
def forward(self, input):
|
| 38 |
+
if torch.cuda.is_available():
|
| 39 |
+
out = torch.zeros(input.shape).cuda()
|
| 40 |
+
else:
|
| 41 |
+
out = torch.zeros(input.shape)
|
| 42 |
+
|
| 43 |
+
for n in range(input.shape[2]):
|
| 44 |
+
out1 = self.shrink(input[:, :, n, :])
|
| 45 |
+
out2 = self.fsmn(out1)
|
| 46 |
+
out[:, :, n, :] = F.relu(self.expand(out2))
|
| 47 |
+
|
| 48 |
+
if self.debug:
|
| 49 |
+
self.dataout = out
|
| 50 |
+
|
| 51 |
+
return out
|
| 52 |
+
|
| 53 |
+
def print_model(self):
|
| 54 |
+
self.shrink.print_model()
|
| 55 |
+
self.fsmn.print_model()
|
| 56 |
+
self.expand.print_model()
|
| 57 |
+
|
| 58 |
+
def to_kaldi_nnet(self):
|
| 59 |
+
re_str = self.shrink.to_kaldi_nnet()
|
| 60 |
+
re_str += self.fsmn.to_kaldi_nnet()
|
| 61 |
+
re_str += self.expand.to_kaldi_nnet()
|
| 62 |
+
|
| 63 |
+
relu = RectifiedLinear(self.expand.linear.out_features,
|
| 64 |
+
self.expand.linear.out_features)
|
| 65 |
+
re_str += relu.to_kaldi_nnet()
|
| 66 |
+
|
| 67 |
+
return re_str
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class FSMNSeleNetV2(nn.Module):
|
| 71 |
+
""" FSMN model with channel selection.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
def __init__(self,
|
| 75 |
+
input_dim=120,
|
| 76 |
+
linear_dim=128,
|
| 77 |
+
proj_dim=64,
|
| 78 |
+
lorder=20,
|
| 79 |
+
rorder=1,
|
| 80 |
+
num_syn=5,
|
| 81 |
+
fsmn_layers=5,
|
| 82 |
+
sele_layer=0):
|
| 83 |
+
"""
|
| 84 |
+
Args:
|
| 85 |
+
input_dim: input dimension
|
| 86 |
+
linear_dim: fsmn input dimension
|
| 87 |
+
proj_dim: fsmn projection dimension
|
| 88 |
+
lorder: fsmn left order
|
| 89 |
+
rorder: fsmn right order
|
| 90 |
+
num_syn: output dimension
|
| 91 |
+
fsmn_layers: no. of fsmn units
|
| 92 |
+
sele_layer: channel selection layer index
|
| 93 |
+
"""
|
| 94 |
+
super(FSMNSeleNetV2, self).__init__()
|
| 95 |
+
|
| 96 |
+
self.sele_layer = sele_layer
|
| 97 |
+
|
| 98 |
+
self.featmap = AffineTransform(input_dim, linear_dim)
|
| 99 |
+
|
| 100 |
+
self.mem = []
|
| 101 |
+
for i in range(fsmn_layers):
|
| 102 |
+
unit = FSMNUnit(linear_dim, proj_dim, lorder, rorder)
|
| 103 |
+
self.mem.append(unit)
|
| 104 |
+
self.add_module('mem_{:d}'.format(i), unit)
|
| 105 |
+
|
| 106 |
+
self.decision = AffineTransform(linear_dim, num_syn)
|
| 107 |
+
|
| 108 |
+
def forward(self, input):
|
| 109 |
+
# multi-channel feature mapping
|
| 110 |
+
if torch.cuda.is_available():
|
| 111 |
+
x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
|
| 112 |
+
self.featmap.linear.out_features).cuda()
|
| 113 |
+
else:
|
| 114 |
+
x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
|
| 115 |
+
self.featmap.linear.out_features)
|
| 116 |
+
|
| 117 |
+
for n in range(input.shape[2]):
|
| 118 |
+
x[:, :, n, :] = F.relu(self.featmap(input[:, :, n, :]))
|
| 119 |
+
|
| 120 |
+
for i, unit in enumerate(self.mem):
|
| 121 |
+
y = unit(x)
|
| 122 |
+
|
| 123 |
+
# perform channel selection
|
| 124 |
+
if i == self.sele_layer:
|
| 125 |
+
pool = nn.MaxPool2d((y.shape[2], 1), stride=(y.shape[2], 1))
|
| 126 |
+
y = pool(y)
|
| 127 |
+
|
| 128 |
+
x = y
|
| 129 |
+
|
| 130 |
+
# remove channel dimension
|
| 131 |
+
y = torch.squeeze(y, -2)
|
| 132 |
+
z = self.decision(y)
|
| 133 |
+
|
| 134 |
+
return z
|
| 135 |
+
|
| 136 |
+
def print_model(self):
|
| 137 |
+
self.featmap.print_model()
|
| 138 |
+
|
| 139 |
+
for unit in self.mem:
|
| 140 |
+
unit.print_model()
|
| 141 |
+
|
| 142 |
+
self.decision.print_model()
|
| 143 |
+
|
| 144 |
+
def print_header(self):
|
| 145 |
+
'''
|
| 146 |
+
get FSMN params
|
| 147 |
+
'''
|
| 148 |
+
input_dim = self.featmap.linear.in_features
|
| 149 |
+
linear_dim = self.featmap.linear.out_features
|
| 150 |
+
proj_dim = self.mem[0].shrink.linear.out_features
|
| 151 |
+
lorder = self.mem[0].fsmn.conv_left.kernel_size[0]
|
| 152 |
+
rorder = 0
|
| 153 |
+
if self.mem[0].fsmn.conv_right is not None:
|
| 154 |
+
rorder = self.mem[0].fsmn.conv_right.kernel_size[0]
|
| 155 |
+
|
| 156 |
+
num_syn = self.decision.linear.out_features
|
| 157 |
+
fsmn_layers = len(self.mem)
|
| 158 |
+
|
| 159 |
+
# no. of output channels, 0.0 means the same as numins
|
| 160 |
+
# numouts = 0.0
|
| 161 |
+
numouts = 1.0
|
| 162 |
+
|
| 163 |
+
#
|
| 164 |
+
# write total header
|
| 165 |
+
#
|
| 166 |
+
header = [0.0] * HEADER_BLOCK_SIZE * 4
|
| 167 |
+
# numins
|
| 168 |
+
header[0] = 0.0
|
| 169 |
+
# numouts
|
| 170 |
+
header[1] = numouts
|
| 171 |
+
# dimins
|
| 172 |
+
header[2] = input_dim
|
| 173 |
+
# dimouts
|
| 174 |
+
header[3] = num_syn
|
| 175 |
+
# numlayers
|
| 176 |
+
header[4] = 3
|
| 177 |
+
|
| 178 |
+
#
|
| 179 |
+
# write each layer's header
|
| 180 |
+
#
|
| 181 |
+
hidx = 1
|
| 182 |
+
|
| 183 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 184 |
+
LayerType.LAYER_DENSE.value)
|
| 185 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 186 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim
|
| 187 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
|
| 188 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 189 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 190 |
+
ActivationType.ACTIVATION_RELU.value)
|
| 191 |
+
hidx += 1
|
| 192 |
+
|
| 193 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 194 |
+
LayerType.LAYER_SEQUENTIAL_FSMN.value)
|
| 195 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 196 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
|
| 197 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = proj_dim
|
| 198 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = lorder
|
| 199 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = rorder
|
| 200 |
+
header[HEADER_BLOCK_SIZE * hidx + 6] = fsmn_layers
|
| 201 |
+
if numouts == 1.0:
|
| 202 |
+
header[HEADER_BLOCK_SIZE * hidx + 7] = float(self.sele_layer)
|
| 203 |
+
else:
|
| 204 |
+
header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0
|
| 205 |
+
hidx += 1
|
| 206 |
+
|
| 207 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 208 |
+
LayerType.LAYER_DENSE.value)
|
| 209 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = numouts
|
| 210 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
|
| 211 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn
|
| 212 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 213 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 214 |
+
ActivationType.ACTIVATION_SOFTMAX.value)
|
| 215 |
+
|
| 216 |
+
for h in header:
|
| 217 |
+
print(f32ToI32(h))
|
| 218 |
+
|
| 219 |
+
def to_kaldi_nnet(self):
|
| 220 |
+
re_str = '<Nnet>\n'
|
| 221 |
+
|
| 222 |
+
re_str = self.featmap.to_kaldi_nnet()
|
| 223 |
+
|
| 224 |
+
relu = RectifiedLinear(self.featmap.linear.out_features,
|
| 225 |
+
self.featmap.linear.out_features)
|
| 226 |
+
re_str += relu.to_kaldi_nnet()
|
| 227 |
+
|
| 228 |
+
for unit in self.mem:
|
| 229 |
+
re_str += unit.to_kaldi_nnet()
|
| 230 |
+
|
| 231 |
+
re_str += self.decision.to_kaldi_nnet()
|
| 232 |
+
|
| 233 |
+
re_str += '<Softmax> %d %d\n' % (self.decision.linear.out_features,
|
| 234 |
+
self.decision.linear.out_features)
|
| 235 |
+
re_str += '<!EndOfComponent>\n'
|
| 236 |
+
re_str += '</Nnet>\n'
|
| 237 |
+
|
| 238 |
+
return re_str
|
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear
|
| 8 |
+
from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class DFSMNUnit(nn.Module):
|
| 12 |
+
""" one multi-channel deep fsmn unit
|
| 13 |
+
Args:
|
| 14 |
+
dimin: input dimension
|
| 15 |
+
dimexpand: feature expansion dimension
|
| 16 |
+
dimout: output dimension
|
| 17 |
+
lorder: left ofder
|
| 18 |
+
rorder: right order
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
def __init__(self,
|
| 22 |
+
dimin=64,
|
| 23 |
+
dimexpand=128,
|
| 24 |
+
dimout=64,
|
| 25 |
+
lorder=10,
|
| 26 |
+
rorder=1):
|
| 27 |
+
super(DFSMNUnit, self).__init__()
|
| 28 |
+
|
| 29 |
+
self.expand = AffineTransform(dimin, dimexpand)
|
| 30 |
+
self.shrink = LinearTransform(dimexpand, dimout)
|
| 31 |
+
self.fsmn = Fsmn(dimout, dimout, lorder, rorder, 1, 1)
|
| 32 |
+
|
| 33 |
+
self.debug = False
|
| 34 |
+
self.dataout = None
|
| 35 |
+
|
| 36 |
+
def forward(self, input):
|
| 37 |
+
"""
|
| 38 |
+
Args:
|
| 39 |
+
input: [batch, time, feature]
|
| 40 |
+
"""
|
| 41 |
+
out1 = F.relu(self.expand(input))
|
| 42 |
+
out2 = self.shrink(out1)
|
| 43 |
+
out3 = self.fsmn(out2)
|
| 44 |
+
|
| 45 |
+
# add skip connection for matched data
|
| 46 |
+
if input.shape[-1] == out3.shape[-1]:
|
| 47 |
+
out3 = input + out3
|
| 48 |
+
if self.debug:
|
| 49 |
+
self.dataout = out3
|
| 50 |
+
return out3
|
| 51 |
+
|
| 52 |
+
def print_model(self):
|
| 53 |
+
self.expand.printModel()
|
| 54 |
+
self.shrink.printModel()
|
| 55 |
+
self.fsmn.printModel()
|
| 56 |
+
|
| 57 |
+
def to_kaldi_nnet(self):
|
| 58 |
+
re_str = self.expand.toKaldiNNet()
|
| 59 |
+
relu = RectifiedLinear(self.expand.linear.out_features,
|
| 60 |
+
self.expand.linear.out_features)
|
| 61 |
+
re_str += relu.toKaldiNNet()
|
| 62 |
+
re_str = self.shrink.toKaldiNNet()
|
| 63 |
+
re_str += self.fsmn.toKaldiNNet()
|
| 64 |
+
return re_str
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class FSMNSeleNetV3(nn.Module):
|
| 68 |
+
""" Deep FSMN model with channel selection performs multi-channel kws.
|
| 69 |
+
Zhang, Shiliang, et al. "Deep-FSMN for large vocabulary continuous speech
|
| 70 |
+
recognition." 2018 IEEE International Conference on Acoustics, Speech and
|
| 71 |
+
Signal Processing (ICASSP). IEEE, 2018.
|
| 72 |
+
|
| 73 |
+
Args:
|
| 74 |
+
input_dim: input dimension
|
| 75 |
+
linear_dim: fsmn input dimension
|
| 76 |
+
proj_dim: fsmn projection dimension
|
| 77 |
+
lorder: fsmn left order
|
| 78 |
+
rorder: fsmn right order
|
| 79 |
+
num_syn: output dimension
|
| 80 |
+
fsmn_layers: no. of fsmn units
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
def __init__(self,
|
| 84 |
+
input_dim=120,
|
| 85 |
+
linear_dim=128,
|
| 86 |
+
proj_dim=64,
|
| 87 |
+
lorder=10,
|
| 88 |
+
rorder=1,
|
| 89 |
+
num_syn=5,
|
| 90 |
+
fsmn_layers=5):
|
| 91 |
+
super(FSMNSeleNetV3, self).__init__()
|
| 92 |
+
|
| 93 |
+
self.mem = []
|
| 94 |
+
# the first unit, mapping input dim to proj dim
|
| 95 |
+
unit = DFSMNUnit(input_dim, linear_dim, proj_dim, lorder, rorder)
|
| 96 |
+
self.mem.append(unit)
|
| 97 |
+
self.add_module('mem_{:d}'.format(0), unit)
|
| 98 |
+
|
| 99 |
+
# deep fsmn layers with skip connection
|
| 100 |
+
for i in range(1, fsmn_layers):
|
| 101 |
+
unit = DFSMNUnit(proj_dim, linear_dim, proj_dim, lorder, rorder)
|
| 102 |
+
self.mem.append(unit)
|
| 103 |
+
self.add_module('mem_{:d}'.format(i), unit)
|
| 104 |
+
|
| 105 |
+
self.expand2 = AffineTransform(proj_dim, linear_dim)
|
| 106 |
+
self.decision = AffineTransform(linear_dim, num_syn)
|
| 107 |
+
|
| 108 |
+
def forward(self, input):
|
| 109 |
+
# multi-channel temp space, [batch, time, channel, feature]
|
| 110 |
+
if torch.cuda.is_available():
|
| 111 |
+
x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
|
| 112 |
+
self.expand2.linear.out_features).cuda()
|
| 113 |
+
else:
|
| 114 |
+
x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
|
| 115 |
+
self.expand2.linear.out_features)
|
| 116 |
+
|
| 117 |
+
for n in range(input.shape[2]):
|
| 118 |
+
chin = input[:, :, n, :]
|
| 119 |
+
|
| 120 |
+
for unit in self.mem:
|
| 121 |
+
chout = unit(chin)
|
| 122 |
+
chin = chout
|
| 123 |
+
|
| 124 |
+
x[:, :, n, :] = F.relu(self.expand2(chout))
|
| 125 |
+
|
| 126 |
+
# perform max pooling
|
| 127 |
+
pool = nn.MaxPool2d((x.shape[2], 1), stride=(x.shape[2], 1))
|
| 128 |
+
y = pool(x)
|
| 129 |
+
|
| 130 |
+
# remove channel dimension
|
| 131 |
+
y = torch.squeeze(y, -2)
|
| 132 |
+
z = self.decision(y)
|
| 133 |
+
|
| 134 |
+
return z
|
| 135 |
+
|
| 136 |
+
def print_model(self):
|
| 137 |
+
for unit in self.mem:
|
| 138 |
+
unit.print_model()
|
| 139 |
+
|
| 140 |
+
self.expand2.printModel()
|
| 141 |
+
self.decision.printModel()
|
| 142 |
+
|
| 143 |
+
def print_header(self):
|
| 144 |
+
""" get DFSMN params
|
| 145 |
+
"""
|
| 146 |
+
input_dim = self.mem[0].expand.linear.in_features
|
| 147 |
+
linear_dim = self.mem[0].expand.linear.out_features
|
| 148 |
+
proj_dim = self.mem[0].shrink.linear.out_features
|
| 149 |
+
lorder = self.mem[0].fsmn.conv_left.kernel_size[0]
|
| 150 |
+
rorder = 0
|
| 151 |
+
if self.mem[0].fsmn.conv_right is not None:
|
| 152 |
+
rorder = self.mem[0].fsmn.conv_right.kernel_size[0]
|
| 153 |
+
|
| 154 |
+
num_syn = self.decision.linear.out_features
|
| 155 |
+
fsmn_layers = len(self.mem)
|
| 156 |
+
|
| 157 |
+
# no. of output channels, 0.0 means the same as numins
|
| 158 |
+
numouts = 1.0
|
| 159 |
+
|
| 160 |
+
#
|
| 161 |
+
# write total header
|
| 162 |
+
#
|
| 163 |
+
header = [0.0] * HEADER_BLOCK_SIZE * 5
|
| 164 |
+
# numins
|
| 165 |
+
header[0] = 0.0
|
| 166 |
+
# numouts
|
| 167 |
+
header[1] = numouts
|
| 168 |
+
# dimins
|
| 169 |
+
header[2] = input_dim
|
| 170 |
+
# dimouts
|
| 171 |
+
header[3] = num_syn
|
| 172 |
+
# numlayers
|
| 173 |
+
header[4] = 4
|
| 174 |
+
|
| 175 |
+
#
|
| 176 |
+
# write each layer's header
|
| 177 |
+
#
|
| 178 |
+
hidx = 1
|
| 179 |
+
|
| 180 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 181 |
+
LayerType.LAYER_DFSMN.value)
|
| 182 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 183 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim
|
| 184 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
|
| 185 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = proj_dim
|
| 186 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = lorder
|
| 187 |
+
header[HEADER_BLOCK_SIZE * hidx + 6] = rorder
|
| 188 |
+
header[HEADER_BLOCK_SIZE * hidx + 7] = fsmn_layers
|
| 189 |
+
hidx += 1
|
| 190 |
+
|
| 191 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 192 |
+
LayerType.LAYER_DENSE.value)
|
| 193 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 194 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = proj_dim
|
| 195 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
|
| 196 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 197 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 198 |
+
ActivationType.ACTIVATION_RELU.value)
|
| 199 |
+
hidx += 1
|
| 200 |
+
|
| 201 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 202 |
+
LayerType.LAYER_MAX_POOLING.value)
|
| 203 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
|
| 204 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
|
| 205 |
+
hidx += 1
|
| 206 |
+
|
| 207 |
+
header[HEADER_BLOCK_SIZE * hidx + 0] = float(
|
| 208 |
+
LayerType.LAYER_DENSE.value)
|
| 209 |
+
header[HEADER_BLOCK_SIZE * hidx + 1] = numouts
|
| 210 |
+
header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
|
| 211 |
+
header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn
|
| 212 |
+
header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
|
| 213 |
+
header[HEADER_BLOCK_SIZE * hidx + 5] = float(
|
| 214 |
+
ActivationType.ACTIVATION_SOFTMAX.value)
|
| 215 |
+
|
| 216 |
+
for h in header:
|
| 217 |
+
print(f32ToI32(h))
|
| 218 |
+
|
| 219 |
+
def to_kaldi_nnet(self):
|
| 220 |
+
re_str = '<Nnet>\n'
|
| 221 |
+
for unit in self.mem:
|
| 222 |
+
re_str += unit.to_kaldi_nnet()
|
| 223 |
+
re_str = self.expand2.toKaldiNNet()
|
| 224 |
+
relu = RectifiedLinear(self.expand2.linear.out_features,
|
| 225 |
+
self.expand2.linear.out_features)
|
| 226 |
+
re_str += relu.toKaldiNNet()
|
| 227 |
+
re_str += self.decision.toKaldiNNet()
|
| 228 |
+
re_str += '<Softmax> %d %d\n' % (self.decision.linear.out_features,
|
| 229 |
+
self.decision.linear.out_features)
|
| 230 |
+
re_str += '<!EndOfComponent>\n'
|
| 231 |
+
re_str += '</Nnet>\n'
|
| 232 |
+
|
| 233 |
+
return re_str
|
runtime/Lib/site_packages/models/audio/kws/farfield/model.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import tempfile
|
| 5 |
+
from typing import Dict, Optional
|
| 6 |
+
|
| 7 |
+
from modelscope.metainfo import Models
|
| 8 |
+
from modelscope.models import TorchModel
|
| 9 |
+
from modelscope.models.base import Tensor
|
| 10 |
+
from modelscope.models.builder import MODELS
|
| 11 |
+
from modelscope.utils.audio.audio_utils import update_conf
|
| 12 |
+
from modelscope.utils.constant import Tasks
|
| 13 |
+
from .fsmn_sele_v2 import FSMNSeleNetV2
|
| 14 |
+
from .fsmn_sele_v3 import FSMNSeleNetV3
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@MODELS.register_module(
|
| 18 |
+
Tasks.keyword_spotting, module_name=Models.speech_dfsmn_kws_char_farfield)
|
| 19 |
+
class FSMNSeleNetV2Decorator(TorchModel):
|
| 20 |
+
r""" A decorator of FSMNSeleNetV2 for integrating into modelscope framework """
|
| 21 |
+
|
| 22 |
+
MODEL_CLASS = FSMNSeleNetV2
|
| 23 |
+
MODEL_TXT = 'model.txt'
|
| 24 |
+
SC_CONFIG = 'sound_connect.conf'
|
| 25 |
+
|
| 26 |
+
def __init__(self,
|
| 27 |
+
model_dir: str,
|
| 28 |
+
training: Optional[bool] = False,
|
| 29 |
+
*args,
|
| 30 |
+
**kwargs):
|
| 31 |
+
"""initialize the dfsmn model from the `model_dir` path.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
model_dir (str): the model path.
|
| 35 |
+
"""
|
| 36 |
+
super().__init__(model_dir, *args, **kwargs)
|
| 37 |
+
if training:
|
| 38 |
+
self.model = self.MODEL_CLASS(*args, **kwargs)
|
| 39 |
+
else:
|
| 40 |
+
sc_config_file = os.path.join(model_dir, self.SC_CONFIG)
|
| 41 |
+
model_txt_file = os.path.join(model_dir, self.MODEL_TXT)
|
| 42 |
+
self.tmp_dir = tempfile.TemporaryDirectory()
|
| 43 |
+
new_config_file = os.path.join(self.tmp_dir.name, self.SC_CONFIG)
|
| 44 |
+
|
| 45 |
+
self._sc = None
|
| 46 |
+
if os.path.exists(model_txt_file):
|
| 47 |
+
conf_dict = dict(kws_model=model_txt_file)
|
| 48 |
+
update_conf(sc_config_file, new_config_file, conf_dict)
|
| 49 |
+
import py_sound_connect
|
| 50 |
+
self._sc = py_sound_connect.SoundConnect(new_config_file)
|
| 51 |
+
self.size_in = self._sc.bytesPerBlockIn()
|
| 52 |
+
self.size_out = self._sc.bytesPerBlockOut()
|
| 53 |
+
else:
|
| 54 |
+
raise Exception(
|
| 55 |
+
f'Invalid model directory! Failed to load model file:'
|
| 56 |
+
f' {model_txt_file}.')
|
| 57 |
+
|
| 58 |
+
def __del__(self):
|
| 59 |
+
if hasattr(self, 'tmp_dir'):
|
| 60 |
+
self.tmp_dir.cleanup()
|
| 61 |
+
|
| 62 |
+
def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]:
|
| 63 |
+
return self.model.forward(input)
|
| 64 |
+
|
| 65 |
+
def forward_decode(self, data: bytes):
|
| 66 |
+
result = {'pcm': self._sc.process(data, self.size_out)}
|
| 67 |
+
state = self._sc.kwsState()
|
| 68 |
+
if state == 2:
|
| 69 |
+
result['kws'] = {
|
| 70 |
+
'keyword':
|
| 71 |
+
self._sc.kwsKeyword(self._sc.kwsSpottedKeywordIndex()),
|
| 72 |
+
'offset': self._sc.kwsKeywordOffset(),
|
| 73 |
+
'channel': self._sc.kwsBestChannel(),
|
| 74 |
+
'length': self._sc.kwsKeywordLength(),
|
| 75 |
+
'confidence': self._sc.kwsConfidence()
|
| 76 |
+
}
|
| 77 |
+
return result
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@MODELS.register_module(
|
| 81 |
+
Tasks.keyword_spotting,
|
| 82 |
+
module_name=Models.speech_dfsmn_kws_char_farfield_iot)
|
| 83 |
+
class FSMNSeleNetV3Decorator(FSMNSeleNetV2Decorator):
|
| 84 |
+
r""" A decorator of FSMNSeleNetV3 for integrating into modelscope framework """
|
| 85 |
+
|
| 86 |
+
MODEL_CLASS = FSMNSeleNetV3
|
| 87 |
+
|
| 88 |
+
def __init__(self,
|
| 89 |
+
model_dir: str,
|
| 90 |
+
training: Optional[bool] = False,
|
| 91 |
+
*args,
|
| 92 |
+
**kwargs):
|
| 93 |
+
"""initialize the dfsmn model from the `model_dir` path.
|
| 94 |
+
|
| 95 |
+
Args:
|
| 96 |
+
model_dir (str): the model path.
|
| 97 |
+
"""
|
| 98 |
+
super().__init__(model_dir, training, *args, **kwargs)
|
runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
import struct
|
| 5 |
+
from enum import Enum
|
| 6 |
+
|
| 7 |
+
HEADER_BLOCK_SIZE = 10
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class LayerType(Enum):
|
| 11 |
+
LAYER_DENSE = 1
|
| 12 |
+
LAYER_GRU = 2
|
| 13 |
+
LAYER_ATTENTION = 3
|
| 14 |
+
LAYER_FSMN = 4
|
| 15 |
+
LAYER_SEQUENTIAL_FSMN = 5
|
| 16 |
+
LAYER_FSMN_SELE = 6
|
| 17 |
+
LAYER_GRU_ATTENTION = 7
|
| 18 |
+
LAYER_DFSMN = 8
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ActivationType(Enum):
|
| 22 |
+
ACTIVATION_NONE = 0
|
| 23 |
+
ACTIVATION_RELU = 1
|
| 24 |
+
ACTIVATION_TANH = 2
|
| 25 |
+
ACTIVATION_SIGMOID = 3
|
| 26 |
+
ACTIVATION_SOFTMAX = 4
|
| 27 |
+
ACTIVATION_LOGSOFTMAX = 5
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def f32ToI32(f):
|
| 31 |
+
"""
|
| 32 |
+
print layer
|
| 33 |
+
"""
|
| 34 |
+
bs = struct.pack('f', f)
|
| 35 |
+
|
| 36 |
+
ba = bytearray()
|
| 37 |
+
ba.append(bs[0])
|
| 38 |
+
ba.append(bs[1])
|
| 39 |
+
ba.append(bs[2])
|
| 40 |
+
ba.append(bs[3])
|
| 41 |
+
|
| 42 |
+
return struct.unpack('i', ba)[0]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def printNeonMatrix(w):
|
| 46 |
+
"""
|
| 47 |
+
print matrix with neon padding
|
| 48 |
+
"""
|
| 49 |
+
numrows, numcols = w.shape
|
| 50 |
+
numnecols = math.ceil(numcols / 4)
|
| 51 |
+
|
| 52 |
+
for i in range(numrows):
|
| 53 |
+
for j in range(numcols):
|
| 54 |
+
print(f32ToI32(w[i, j]))
|
| 55 |
+
|
| 56 |
+
for j in range(numnecols * 4 - numcols):
|
| 57 |
+
print(0)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def printNeonVector(b):
|
| 61 |
+
"""
|
| 62 |
+
print vector with neon padding
|
| 63 |
+
"""
|
| 64 |
+
size = b.shape[0]
|
| 65 |
+
nesize = math.ceil(size / 4)
|
| 66 |
+
|
| 67 |
+
for i in range(size):
|
| 68 |
+
print(f32ToI32(b[i]))
|
| 69 |
+
|
| 70 |
+
for i in range(nesize * 4 - size):
|
| 71 |
+
print(0)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def printDense(layer):
|
| 75 |
+
"""
|
| 76 |
+
save dense layer
|
| 77 |
+
"""
|
| 78 |
+
statedict = layer.state_dict()
|
| 79 |
+
printNeonMatrix(statedict['weight'])
|
| 80 |
+
printNeonVector(statedict['bias'])
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def printGRU(layer):
|
| 84 |
+
"""
|
| 85 |
+
save gru layer
|
| 86 |
+
"""
|
| 87 |
+
statedict = layer.state_dict()
|
| 88 |
+
weight = [statedict['weight_ih_l0'], statedict['weight_hh_l0']]
|
| 89 |
+
bias = [statedict['bias_ih_l0'], statedict['bias_hh_l0']]
|
| 90 |
+
numins, numouts = weight[0].shape
|
| 91 |
+
numins = numins // 3
|
| 92 |
+
|
| 93 |
+
# output input weights
|
| 94 |
+
w_rx = weight[0][:numins, :]
|
| 95 |
+
w_zx = weight[0][numins:numins * 2, :]
|
| 96 |
+
w_x = weight[0][numins * 2:, :]
|
| 97 |
+
printNeonMatrix(w_zx)
|
| 98 |
+
printNeonMatrix(w_rx)
|
| 99 |
+
printNeonMatrix(w_x)
|
| 100 |
+
|
| 101 |
+
# output recurrent weights
|
| 102 |
+
w_rh = weight[1][:numins, :]
|
| 103 |
+
w_zh = weight[1][numins:numins * 2, :]
|
| 104 |
+
w_h = weight[1][numins * 2:, :]
|
| 105 |
+
printNeonMatrix(w_zh)
|
| 106 |
+
printNeonMatrix(w_rh)
|
| 107 |
+
printNeonMatrix(w_h)
|
| 108 |
+
|
| 109 |
+
# output input bias
|
| 110 |
+
b_rx = bias[0][:numins]
|
| 111 |
+
b_zx = bias[0][numins:numins * 2]
|
| 112 |
+
b_x = bias[0][numins * 2:]
|
| 113 |
+
printNeonVector(b_zx)
|
| 114 |
+
printNeonVector(b_rx)
|
| 115 |
+
printNeonVector(b_x)
|
| 116 |
+
|
| 117 |
+
# output recurrent bias
|
| 118 |
+
b_rh = bias[1][:numins]
|
| 119 |
+
b_zh = bias[1][numins:numins * 2]
|
| 120 |
+
b_h = bias[1][numins * 2:]
|
| 121 |
+
printNeonVector(b_zh)
|
| 122 |
+
printNeonVector(b_rh)
|
| 123 |
+
printNeonVector(b_h)
|
runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
from modelscope.metainfo import Models
|
| 7 |
+
from modelscope.models.base import Model
|
| 8 |
+
from modelscope.models.builder import MODELS
|
| 9 |
+
from modelscope.utils.constant import Tasks
|
| 10 |
+
|
| 11 |
+
__all__ = ['GenericKeyWordSpotting']
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@MODELS.register_module(Tasks.keyword_spotting, module_name=Models.kws_kwsbp)
|
| 15 |
+
class GenericKeyWordSpotting(Model):
|
| 16 |
+
|
| 17 |
+
def __init__(self, model_dir: str, *args, **kwargs):
|
| 18 |
+
"""initialize the info of model.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
model_dir (str): the model path.
|
| 22 |
+
"""
|
| 23 |
+
super().__init__(model_dir, *args, **kwargs)
|
| 24 |
+
self.model_cfg = {
|
| 25 |
+
'model_workspace': model_dir,
|
| 26 |
+
'config_path': os.path.join(model_dir, 'config.yaml')
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
def forward(self) -> Dict[str, Any]:
|
| 30 |
+
"""return the info of the model
|
| 31 |
+
"""
|
| 32 |
+
return self.model_cfg
|
runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) 2020 Binbin Zhang
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class GlobalCMVN(torch.nn.Module):
|
| 23 |
+
|
| 24 |
+
def __init__(self,
|
| 25 |
+
mean: torch.Tensor,
|
| 26 |
+
istd: torch.Tensor,
|
| 27 |
+
norm_var: bool = True):
|
| 28 |
+
"""
|
| 29 |
+
Args:
|
| 30 |
+
mean (torch.Tensor): mean stats
|
| 31 |
+
istd (torch.Tensor): inverse std, std which is 1.0 / std
|
| 32 |
+
"""
|
| 33 |
+
super().__init__()
|
| 34 |
+
assert mean.shape == istd.shape
|
| 35 |
+
self.norm_var = norm_var
|
| 36 |
+
# The buffer can be accessed from this module using self.mean
|
| 37 |
+
self.register_buffer('mean', mean)
|
| 38 |
+
self.register_buffer('istd', istd)
|
| 39 |
+
|
| 40 |
+
def forward(self, x: torch.Tensor):
|
| 41 |
+
"""
|
| 42 |
+
Args:
|
| 43 |
+
x (torch.Tensor): (batch, max_len, feat_dim)
|
| 44 |
+
|
| 45 |
+
Returns:
|
| 46 |
+
(torch.Tensor): normalized feature
|
| 47 |
+
"""
|
| 48 |
+
x = x - self.mean
|
| 49 |
+
if self.norm_var:
|
| 50 |
+
x = x * self.istd
|
| 51 |
+
return x
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def load_kaldi_cmvn(cmvn_file):
|
| 55 |
+
""" Load the kaldi format cmvn stats file and no need to calculate
|
| 56 |
+
|
| 57 |
+
Args:
|
| 58 |
+
cmvn_file: cmvn stats file in kaldi format
|
| 59 |
+
|
| 60 |
+
Returns:
|
| 61 |
+
a numpy array of [means, vars]
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
means = None
|
| 65 |
+
variance = None
|
| 66 |
+
with open(cmvn_file) as f:
|
| 67 |
+
all_lines = f.readlines()
|
| 68 |
+
for idx, line in enumerate(all_lines):
|
| 69 |
+
if line.find('AddShift') != -1:
|
| 70 |
+
segs = line.strip().split(' ')
|
| 71 |
+
assert len(segs) == 3
|
| 72 |
+
next_line = all_lines[idx + 1]
|
| 73 |
+
means_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
|
| 74 |
+
means_list = means_str.strip().split(' ')
|
| 75 |
+
means = [0 - float(s) for s in means_list]
|
| 76 |
+
assert len(means) == int(segs[1])
|
| 77 |
+
elif line.find('Rescale') != -1:
|
| 78 |
+
segs = line.strip().split(' ')
|
| 79 |
+
assert len(segs) == 3
|
| 80 |
+
next_line = all_lines[idx + 1]
|
| 81 |
+
vars_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
|
| 82 |
+
vars_list = vars_str.strip().split(' ')
|
| 83 |
+
variance = [float(s) for s in vars_list]
|
| 84 |
+
assert len(variance) == int(segs[1])
|
| 85 |
+
elif line.find('Splice') != -1:
|
| 86 |
+
segs = line.strip().split(' ')
|
| 87 |
+
assert len(segs) == 3
|
| 88 |
+
next_line = all_lines[idx + 1]
|
| 89 |
+
splice_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
|
| 90 |
+
splice_list = splice_str.strip().split(' ')
|
| 91 |
+
assert len(splice_list) * int(segs[2]) == int(segs[1])
|
| 92 |
+
copy_times = len(splice_list)
|
| 93 |
+
else:
|
| 94 |
+
continue
|
| 95 |
+
|
| 96 |
+
cmvn = np.array([means, variance])
|
| 97 |
+
cmvn = np.tile(cmvn, (1, copy_times))
|
| 98 |
+
|
| 99 |
+
return cmvn
|
runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py
ADDED
|
@@ -0,0 +1,521 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
'''
|
| 2 |
+
FSMN implementation.
|
| 3 |
+
|
| 4 |
+
Copyright: 2022-03-09 yueyue.nyy
|
| 5 |
+
'''
|
| 6 |
+
|
| 7 |
+
from typing import Tuple
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def toKaldiMatrix(np_mat):
|
| 16 |
+
np.set_printoptions(threshold=np.inf, linewidth=np.nan)
|
| 17 |
+
out_str = str(np_mat)
|
| 18 |
+
out_str = out_str.replace('[', '')
|
| 19 |
+
out_str = out_str.replace(']', '')
|
| 20 |
+
return '[ %s ]\n' % out_str
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def printTensor(torch_tensor):
|
| 24 |
+
re_str = ''
|
| 25 |
+
x = torch_tensor.detach().squeeze().numpy()
|
| 26 |
+
re_str += toKaldiMatrix(x)
|
| 27 |
+
# re_str += '<!EndOfComponent>\n'
|
| 28 |
+
print(re_str)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class LinearTransform(nn.Module):
|
| 32 |
+
|
| 33 |
+
def __init__(self, input_dim, output_dim):
|
| 34 |
+
super(LinearTransform, self).__init__()
|
| 35 |
+
self.input_dim = input_dim
|
| 36 |
+
self.output_dim = output_dim
|
| 37 |
+
self.linear = nn.Linear(input_dim, output_dim, bias=False)
|
| 38 |
+
self.quant = torch.quantization.QuantStub()
|
| 39 |
+
self.dequant = torch.quantization.DeQuantStub()
|
| 40 |
+
|
| 41 |
+
def forward(self, input):
|
| 42 |
+
output = self.quant(input)
|
| 43 |
+
output = self.linear(output)
|
| 44 |
+
output = self.dequant(output)
|
| 45 |
+
|
| 46 |
+
return output
|
| 47 |
+
|
| 48 |
+
def to_kaldi_net(self):
|
| 49 |
+
re_str = ''
|
| 50 |
+
re_str += '<LinearTransform> %d %d\n' % (self.output_dim,
|
| 51 |
+
self.input_dim)
|
| 52 |
+
re_str += '<LearnRateCoef> 1\n'
|
| 53 |
+
|
| 54 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 55 |
+
x = linear_weights.squeeze().numpy()
|
| 56 |
+
re_str += toKaldiMatrix(x)
|
| 57 |
+
# re_str += '<!EndOfComponent>\n'
|
| 58 |
+
|
| 59 |
+
return re_str
|
| 60 |
+
|
| 61 |
+
def to_pytorch_net(self, fread):
|
| 62 |
+
linear_line = fread.readline()
|
| 63 |
+
linear_split = linear_line.strip().split()
|
| 64 |
+
assert len(linear_split) == 3
|
| 65 |
+
assert linear_split[0] == '<LinearTransform>'
|
| 66 |
+
self.output_dim = int(linear_split[1])
|
| 67 |
+
self.input_dim = int(linear_split[2])
|
| 68 |
+
|
| 69 |
+
learn_rate_line = fread.readline()
|
| 70 |
+
assert learn_rate_line.find('LearnRateCoef') != -1
|
| 71 |
+
|
| 72 |
+
self.linear.reset_parameters()
|
| 73 |
+
|
| 74 |
+
# linear_weights = self.state_dict()['linear.weight']
|
| 75 |
+
# print(linear_weights.shape)
|
| 76 |
+
new_weights = torch.zeros((self.output_dim, self.input_dim),
|
| 77 |
+
dtype=torch.float32)
|
| 78 |
+
for i in range(self.output_dim):
|
| 79 |
+
line = fread.readline()
|
| 80 |
+
splits = line.strip().strip('[]').strip().split()
|
| 81 |
+
assert len(splits) == self.input_dim
|
| 82 |
+
cols = torch.tensor([float(item) for item in splits],
|
| 83 |
+
dtype=torch.float32)
|
| 84 |
+
new_weights[i, :] = cols
|
| 85 |
+
|
| 86 |
+
self.linear.weight.data = new_weights
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class AffineTransform(nn.Module):
|
| 90 |
+
|
| 91 |
+
def __init__(self, input_dim, output_dim):
|
| 92 |
+
super(AffineTransform, self).__init__()
|
| 93 |
+
self.input_dim = input_dim
|
| 94 |
+
self.output_dim = output_dim
|
| 95 |
+
|
| 96 |
+
self.linear = nn.Linear(input_dim, output_dim)
|
| 97 |
+
self.quant = torch.quantization.QuantStub()
|
| 98 |
+
self.dequant = torch.quantization.DeQuantStub()
|
| 99 |
+
|
| 100 |
+
def forward(self, input):
|
| 101 |
+
output = self.quant(input)
|
| 102 |
+
output = self.linear(output)
|
| 103 |
+
output = self.dequant(output)
|
| 104 |
+
|
| 105 |
+
return output
|
| 106 |
+
|
| 107 |
+
def to_kaldi_net(self):
|
| 108 |
+
re_str = ''
|
| 109 |
+
re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
|
| 110 |
+
self.input_dim)
|
| 111 |
+
re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
|
| 112 |
+
|
| 113 |
+
linear_weights = self.state_dict()['linear.weight']
|
| 114 |
+
x = linear_weights.squeeze().numpy()
|
| 115 |
+
re_str += toKaldiMatrix(x)
|
| 116 |
+
|
| 117 |
+
linear_bias = self.state_dict()['linear.bias']
|
| 118 |
+
x = linear_bias.squeeze().numpy()
|
| 119 |
+
re_str += toKaldiMatrix(x)
|
| 120 |
+
# re_str += '<!EndOfComponent>\n'
|
| 121 |
+
|
| 122 |
+
return re_str
|
| 123 |
+
|
| 124 |
+
def to_pytorch_net(self, fread):
|
| 125 |
+
affine_line = fread.readline()
|
| 126 |
+
affine_split = affine_line.strip().split()
|
| 127 |
+
assert len(affine_split) == 3
|
| 128 |
+
assert affine_split[0] == '<AffineTransform>'
|
| 129 |
+
self.output_dim = int(affine_split[1])
|
| 130 |
+
self.input_dim = int(affine_split[2])
|
| 131 |
+
print('AffineTransform output/input dim: %d %d' %
|
| 132 |
+
(self.output_dim, self.input_dim))
|
| 133 |
+
|
| 134 |
+
learn_rate_line = fread.readline()
|
| 135 |
+
assert learn_rate_line.find('LearnRateCoef') != -1
|
| 136 |
+
|
| 137 |
+
# linear_weights = self.state_dict()['linear.weight']
|
| 138 |
+
# print(linear_weights.shape)
|
| 139 |
+
self.linear.reset_parameters()
|
| 140 |
+
|
| 141 |
+
new_weights = torch.zeros((self.output_dim, self.input_dim),
|
| 142 |
+
dtype=torch.float32)
|
| 143 |
+
for i in range(self.output_dim):
|
| 144 |
+
line = fread.readline()
|
| 145 |
+
splits = line.strip().strip('[]').strip().split()
|
| 146 |
+
assert len(splits) == self.input_dim
|
| 147 |
+
cols = torch.tensor([float(item) for item in splits],
|
| 148 |
+
dtype=torch.float32)
|
| 149 |
+
new_weights[i, :] = cols
|
| 150 |
+
|
| 151 |
+
self.linear.weight.data = new_weights
|
| 152 |
+
|
| 153 |
+
# linear_bias = self.state_dict()['linear.bias']
|
| 154 |
+
# print(linear_bias.shape)
|
| 155 |
+
bias_line = fread.readline()
|
| 156 |
+
splits = bias_line.strip().strip('[]').strip().split()
|
| 157 |
+
assert len(splits) == self.output_dim
|
| 158 |
+
new_bias = torch.tensor([float(item) for item in splits],
|
| 159 |
+
dtype=torch.float32)
|
| 160 |
+
|
| 161 |
+
self.linear.bias.data = new_bias
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
class FSMNBlock(nn.Module):
|
| 165 |
+
|
| 166 |
+
def __init__(
|
| 167 |
+
self,
|
| 168 |
+
input_dim: int,
|
| 169 |
+
output_dim: int,
|
| 170 |
+
lorder=None,
|
| 171 |
+
rorder=None,
|
| 172 |
+
lstride=1,
|
| 173 |
+
rstride=1,
|
| 174 |
+
):
|
| 175 |
+
super(FSMNBlock, self).__init__()
|
| 176 |
+
|
| 177 |
+
self.dim = input_dim
|
| 178 |
+
|
| 179 |
+
if lorder is None:
|
| 180 |
+
return
|
| 181 |
+
|
| 182 |
+
self.lorder = lorder
|
| 183 |
+
self.rorder = rorder
|
| 184 |
+
self.lstride = lstride
|
| 185 |
+
self.rstride = rstride
|
| 186 |
+
|
| 187 |
+
self.conv_left = nn.Conv2d(
|
| 188 |
+
self.dim,
|
| 189 |
+
self.dim, [lorder, 1],
|
| 190 |
+
dilation=[lstride, 1],
|
| 191 |
+
groups=self.dim,
|
| 192 |
+
bias=False)
|
| 193 |
+
|
| 194 |
+
if rorder > 0:
|
| 195 |
+
self.conv_right = nn.Conv2d(
|
| 196 |
+
self.dim,
|
| 197 |
+
self.dim, [rorder, 1],
|
| 198 |
+
dilation=[rstride, 1],
|
| 199 |
+
groups=self.dim,
|
| 200 |
+
bias=False)
|
| 201 |
+
else:
|
| 202 |
+
self.conv_right = None
|
| 203 |
+
|
| 204 |
+
self.quant = torch.quantization.QuantStub()
|
| 205 |
+
self.dequant = torch.quantization.DeQuantStub()
|
| 206 |
+
|
| 207 |
+
def forward(self, input):
|
| 208 |
+
x = torch.unsqueeze(input, 1)
|
| 209 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 210 |
+
|
| 211 |
+
y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
|
| 212 |
+
y_left = self.quant(y_left)
|
| 213 |
+
y_left = self.conv_left(y_left)
|
| 214 |
+
y_left = self.dequant(y_left)
|
| 215 |
+
out = x_per + y_left
|
| 216 |
+
|
| 217 |
+
if self.conv_right is not None:
|
| 218 |
+
y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
|
| 219 |
+
y_right = y_right[:, :, self.rstride:, :]
|
| 220 |
+
y_right = self.quant(y_right)
|
| 221 |
+
y_right = self.conv_right(y_right)
|
| 222 |
+
y_right = self.dequant(y_right)
|
| 223 |
+
out += y_right
|
| 224 |
+
|
| 225 |
+
out_per = out.permute(0, 3, 2, 1)
|
| 226 |
+
output = out_per.squeeze(1)
|
| 227 |
+
|
| 228 |
+
return output
|
| 229 |
+
|
| 230 |
+
def to_kaldi_net(self):
|
| 231 |
+
re_str = ''
|
| 232 |
+
re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)
|
| 233 |
+
re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (
|
| 234 |
+
1, self.lorder, self.rorder, self.lstride, self.rstride)
|
| 235 |
+
|
| 236 |
+
# print(self.conv_left.weight,self.conv_right.weight)
|
| 237 |
+
lfiters = self.state_dict()['conv_left.weight']
|
| 238 |
+
x = np.flipud(lfiters.squeeze().numpy().T)
|
| 239 |
+
re_str += toKaldiMatrix(x)
|
| 240 |
+
|
| 241 |
+
if self.conv_right is not None:
|
| 242 |
+
rfiters = self.state_dict()['conv_right.weight']
|
| 243 |
+
x = (rfiters.squeeze().numpy().T)
|
| 244 |
+
re_str += toKaldiMatrix(x)
|
| 245 |
+
# re_str += '<!EndOfComponent>\n'
|
| 246 |
+
|
| 247 |
+
return re_str
|
| 248 |
+
|
| 249 |
+
def to_pytorch_net(self, fread):
|
| 250 |
+
fsmn_line = fread.readline()
|
| 251 |
+
fsmn_split = fsmn_line.strip().split()
|
| 252 |
+
assert len(fsmn_split) == 3
|
| 253 |
+
assert fsmn_split[0] == '<Fsmn>'
|
| 254 |
+
self.dim = int(fsmn_split[1])
|
| 255 |
+
|
| 256 |
+
params_line = fread.readline()
|
| 257 |
+
params_split = params_line.strip().strip('[]').strip().split()
|
| 258 |
+
assert len(params_split) == 12
|
| 259 |
+
assert params_split[0] == '<LearnRateCoef>'
|
| 260 |
+
assert params_split[2] == '<LOrder>'
|
| 261 |
+
self.lorder = int(params_split[3])
|
| 262 |
+
assert params_split[4] == '<ROrder>'
|
| 263 |
+
self.rorder = int(params_split[5])
|
| 264 |
+
assert params_split[6] == '<LStride>'
|
| 265 |
+
self.lstride = int(params_split[7])
|
| 266 |
+
assert params_split[8] == '<RStride>'
|
| 267 |
+
self.rstride = int(params_split[9])
|
| 268 |
+
assert params_split[10] == '<MaxNorm>'
|
| 269 |
+
|
| 270 |
+
# lfilters = self.state_dict()['conv_left.weight']
|
| 271 |
+
# print(lfilters.shape)
|
| 272 |
+
print('read conv_left weight')
|
| 273 |
+
new_lfilters = torch.zeros((self.lorder, 1, self.dim, 1),
|
| 274 |
+
dtype=torch.float32)
|
| 275 |
+
for i in range(self.lorder):
|
| 276 |
+
print('read conv_left weight -- %d' % i)
|
| 277 |
+
line = fread.readline()
|
| 278 |
+
splits = line.strip().strip('[]').strip().split()
|
| 279 |
+
assert len(splits) == self.dim
|
| 280 |
+
cols = torch.tensor([float(item) for item in splits],
|
| 281 |
+
dtype=torch.float32)
|
| 282 |
+
new_lfilters[self.lorder - 1 - i, 0, :, 0] = cols
|
| 283 |
+
|
| 284 |
+
new_lfilters = torch.transpose(new_lfilters, 0, 2)
|
| 285 |
+
# print(new_lfilters.shape)
|
| 286 |
+
|
| 287 |
+
self.conv_left.reset_parameters()
|
| 288 |
+
self.conv_left.weight.data = new_lfilters
|
| 289 |
+
# print(self.conv_left.weight.shape)
|
| 290 |
+
|
| 291 |
+
if self.rorder > 0:
|
| 292 |
+
# rfilters = self.state_dict()['conv_right.weight']
|
| 293 |
+
# print(rfilters.shape)
|
| 294 |
+
print('read conv_right weight')
|
| 295 |
+
new_rfilters = torch.zeros((self.rorder, 1, self.dim, 1),
|
| 296 |
+
dtype=torch.float32)
|
| 297 |
+
line = fread.readline()
|
| 298 |
+
for i in range(self.rorder):
|
| 299 |
+
print('read conv_right weight -- %d' % i)
|
| 300 |
+
line = fread.readline()
|
| 301 |
+
splits = line.strip().strip('[]').strip().split()
|
| 302 |
+
assert len(splits) == self.dim
|
| 303 |
+
cols = torch.tensor([float(item) for item in splits],
|
| 304 |
+
dtype=torch.float32)
|
| 305 |
+
new_rfilters[i, 0, :, 0] = cols
|
| 306 |
+
|
| 307 |
+
new_rfilters = torch.transpose(new_rfilters, 0, 2)
|
| 308 |
+
# print(new_rfilters.shape)
|
| 309 |
+
self.conv_right.reset_parameters()
|
| 310 |
+
self.conv_right.weight.data = new_rfilters
|
| 311 |
+
# print(self.conv_right.weight.shape)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
class RectifiedLinear(nn.Module):
|
| 315 |
+
|
| 316 |
+
def __init__(self, input_dim, output_dim):
|
| 317 |
+
super(RectifiedLinear, self).__init__()
|
| 318 |
+
self.dim = input_dim
|
| 319 |
+
self.relu = nn.ReLU()
|
| 320 |
+
self.dropout = nn.Dropout(0.1)
|
| 321 |
+
|
| 322 |
+
def forward(self, input):
|
| 323 |
+
out = self.relu(input)
|
| 324 |
+
# out = self.dropout(out)
|
| 325 |
+
return out
|
| 326 |
+
|
| 327 |
+
def to_kaldi_net(self):
|
| 328 |
+
re_str = ''
|
| 329 |
+
re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
|
| 330 |
+
# re_str += '<!EndOfComponent>\n'
|
| 331 |
+
return re_str
|
| 332 |
+
|
| 333 |
+
# re_str = ''
|
| 334 |
+
# re_str += '<ParametricRelu> %d %d\n' % (self.dim, self.dim)
|
| 335 |
+
# re_str += '<AlphaLearnRateCoef> 0 <BetaLearnRateCoef> 0\n'
|
| 336 |
+
# re_str += toKaldiMatrix(np.ones((self.dim), dtype = 'int32'))
|
| 337 |
+
# re_str += toKaldiMatrix(np.zeros((self.dim), dtype = 'int32'))
|
| 338 |
+
# re_str += '<!EndOfComponent>\n'
|
| 339 |
+
# return re_str
|
| 340 |
+
|
| 341 |
+
def to_pytorch_net(self, fread):
|
| 342 |
+
line = fread.readline()
|
| 343 |
+
splits = line.strip().split()
|
| 344 |
+
assert len(splits) == 3
|
| 345 |
+
assert splits[0] == '<RectifiedLinear>'
|
| 346 |
+
assert int(splits[1]) == int(splits[2])
|
| 347 |
+
assert int(splits[1]) == self.dim
|
| 348 |
+
self.dim = int(splits[1])
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _build_repeats(
|
| 352 |
+
fsmn_layers: int,
|
| 353 |
+
linear_dim: int,
|
| 354 |
+
proj_dim: int,
|
| 355 |
+
lorder: int,
|
| 356 |
+
rorder: int,
|
| 357 |
+
lstride=1,
|
| 358 |
+
rstride=1,
|
| 359 |
+
):
|
| 360 |
+
repeats = [
|
| 361 |
+
nn.Sequential(
|
| 362 |
+
LinearTransform(linear_dim, proj_dim),
|
| 363 |
+
FSMNBlock(proj_dim, proj_dim, lorder, rorder, 1, 1),
|
| 364 |
+
AffineTransform(proj_dim, linear_dim),
|
| 365 |
+
RectifiedLinear(linear_dim, linear_dim))
|
| 366 |
+
for i in range(fsmn_layers)
|
| 367 |
+
]
|
| 368 |
+
|
| 369 |
+
return nn.Sequential(*repeats)
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
class FSMN(nn.Module):
|
| 373 |
+
|
| 374 |
+
def __init__(
|
| 375 |
+
self,
|
| 376 |
+
input_dim: int,
|
| 377 |
+
input_affine_dim: int,
|
| 378 |
+
fsmn_layers: int,
|
| 379 |
+
linear_dim: int,
|
| 380 |
+
proj_dim: int,
|
| 381 |
+
lorder: int,
|
| 382 |
+
rorder: int,
|
| 383 |
+
lstride: int,
|
| 384 |
+
rstride: int,
|
| 385 |
+
output_affine_dim: int,
|
| 386 |
+
output_dim: int,
|
| 387 |
+
):
|
| 388 |
+
"""
|
| 389 |
+
Args:
|
| 390 |
+
input_dim: input dimension
|
| 391 |
+
input_affine_dim: input affine layer dimension
|
| 392 |
+
fsmn_layers: no. of fsmn units
|
| 393 |
+
linear_dim: fsmn input dimension
|
| 394 |
+
proj_dim: fsmn projection dimension
|
| 395 |
+
lorder: fsmn left order
|
| 396 |
+
rorder: fsmn right order
|
| 397 |
+
lstride: fsmn left stride
|
| 398 |
+
rstride: fsmn right stride
|
| 399 |
+
output_affine_dim: output affine layer dimension
|
| 400 |
+
output_dim: output dimension
|
| 401 |
+
"""
|
| 402 |
+
super(FSMN, self).__init__()
|
| 403 |
+
|
| 404 |
+
self.input_dim = input_dim
|
| 405 |
+
self.input_affine_dim = input_affine_dim
|
| 406 |
+
self.fsmn_layers = fsmn_layers
|
| 407 |
+
self.linear_dim = linear_dim
|
| 408 |
+
self.proj_dim = proj_dim
|
| 409 |
+
self.lorder = lorder
|
| 410 |
+
self.rorder = rorder
|
| 411 |
+
self.lstride = lstride
|
| 412 |
+
self.rstride = rstride
|
| 413 |
+
self.output_affine_dim = output_affine_dim
|
| 414 |
+
self.output_dim = output_dim
|
| 415 |
+
|
| 416 |
+
self.in_linear1 = AffineTransform(input_dim, input_affine_dim)
|
| 417 |
+
self.in_linear2 = AffineTransform(input_affine_dim, linear_dim)
|
| 418 |
+
self.relu = RectifiedLinear(linear_dim, linear_dim)
|
| 419 |
+
|
| 420 |
+
self.fsmn = _build_repeats(fsmn_layers, linear_dim, proj_dim, lorder,
|
| 421 |
+
rorder, lstride, rstride)
|
| 422 |
+
|
| 423 |
+
self.out_linear1 = AffineTransform(linear_dim, output_affine_dim)
|
| 424 |
+
self.out_linear2 = AffineTransform(output_affine_dim, output_dim)
|
| 425 |
+
# self.softmax = nn.Softmax(dim = -1)
|
| 426 |
+
|
| 427 |
+
def fuse_modules(self):
|
| 428 |
+
pass
|
| 429 |
+
|
| 430 |
+
def forward(
|
| 431 |
+
self,
|
| 432 |
+
input: torch.Tensor,
|
| 433 |
+
in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float)
|
| 434 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 435 |
+
"""
|
| 436 |
+
Args:
|
| 437 |
+
input (torch.Tensor): Input tensor (B, T, D)
|
| 438 |
+
in_cache(torch.Tensor): (B, D, C), C is the accumulated cache size
|
| 439 |
+
"""
|
| 440 |
+
|
| 441 |
+
# print("FSMN forward!!!!")
|
| 442 |
+
# print(input.shape)
|
| 443 |
+
# print(input)
|
| 444 |
+
# print(self.in_linear1.input_dim)
|
| 445 |
+
# print(self.in_linear1.output_dim)
|
| 446 |
+
|
| 447 |
+
x1 = self.in_linear1(input)
|
| 448 |
+
x2 = self.in_linear2(x1)
|
| 449 |
+
x3 = self.relu(x2)
|
| 450 |
+
x4 = self.fsmn(x3)
|
| 451 |
+
x5 = self.out_linear1(x4)
|
| 452 |
+
x6 = self.out_linear2(x5)
|
| 453 |
+
# x7 = self.softmax(x6)
|
| 454 |
+
|
| 455 |
+
# return x7, None
|
| 456 |
+
return x6, in_cache
|
| 457 |
+
|
| 458 |
+
def to_kaldi_net(self):
|
| 459 |
+
re_str = ''
|
| 460 |
+
re_str += '<Nnet>\n'
|
| 461 |
+
re_str += self.in_linear1.to_kaldi_net()
|
| 462 |
+
re_str += self.in_linear2.to_kaldi_net()
|
| 463 |
+
re_str += self.relu.to_kaldi_net()
|
| 464 |
+
|
| 465 |
+
for fsmn in self.fsmn:
|
| 466 |
+
re_str += fsmn[0].to_kaldi_net()
|
| 467 |
+
re_str += fsmn[1].to_kaldi_net()
|
| 468 |
+
re_str += fsmn[2].to_kaldi_net()
|
| 469 |
+
re_str += fsmn[3].to_kaldi_net()
|
| 470 |
+
|
| 471 |
+
re_str += self.out_linear1.to_kaldi_net()
|
| 472 |
+
re_str += self.out_linear2.to_kaldi_net()
|
| 473 |
+
re_str += '<Softmax> %d %d\n' % (self.output_dim, self.output_dim)
|
| 474 |
+
# re_str += '<!EndOfComponent>\n'
|
| 475 |
+
re_str += '</Nnet>\n'
|
| 476 |
+
|
| 477 |
+
return re_str
|
| 478 |
+
|
| 479 |
+
def to_pytorch_net(self, kaldi_file):
|
| 480 |
+
with open(kaldi_file, 'r', encoding='utf8') as fread:
|
| 481 |
+
fread = open(kaldi_file, 'r')
|
| 482 |
+
nnet_start_line = fread.readline()
|
| 483 |
+
assert nnet_start_line.strip() == '<Nnet>'
|
| 484 |
+
|
| 485 |
+
self.in_linear1.to_pytorch_net(fread)
|
| 486 |
+
self.in_linear2.to_pytorch_net(fread)
|
| 487 |
+
self.relu.to_pytorch_net(fread)
|
| 488 |
+
|
| 489 |
+
for fsmn in self.fsmn:
|
| 490 |
+
fsmn[0].to_pytorch_net(fread)
|
| 491 |
+
fsmn[1].to_pytorch_net(fread)
|
| 492 |
+
fsmn[2].to_pytorch_net(fread)
|
| 493 |
+
fsmn[3].to_pytorch_net(fread)
|
| 494 |
+
|
| 495 |
+
self.out_linear1.to_pytorch_net(fread)
|
| 496 |
+
self.out_linear2.to_pytorch_net(fread)
|
| 497 |
+
|
| 498 |
+
softmax_line = fread.readline()
|
| 499 |
+
softmax_split = softmax_line.strip().split()
|
| 500 |
+
assert softmax_split[0].strip() == '<Softmax>'
|
| 501 |
+
assert int(softmax_split[1]) == self.output_dim
|
| 502 |
+
assert int(softmax_split[2]) == self.output_dim
|
| 503 |
+
# '<!EndOfComponent>\n'
|
| 504 |
+
|
| 505 |
+
nnet_end_line = fread.readline()
|
| 506 |
+
assert nnet_end_line.strip() == '</Nnet>'
|
| 507 |
+
fread.close()
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
if __name__ == '__main__':
|
| 511 |
+
fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599)
|
| 512 |
+
print(fsmn)
|
| 513 |
+
|
| 514 |
+
num_params = sum(p.numel() for p in fsmn.parameters())
|
| 515 |
+
print('the number of model params: {}'.format(num_params))
|
| 516 |
+
x = torch.zeros(128, 200, 400) # batch-size * time * dim
|
| 517 |
+
y, _ = fsmn(x) # batch-size * time * dim
|
| 518 |
+
print('input shape: {}'.format(x.shape))
|
| 519 |
+
print('output shape: {}'.format(y.shape))
|
| 520 |
+
|
| 521 |
+
print(fsmn.to_kaldi_net())
|
runtime/Lib/site_packages/models/audio/kws/nearfield/model.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import sys
|
| 5 |
+
import tempfile
|
| 6 |
+
from typing import Dict, Optional, Tuple
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
|
| 11 |
+
from modelscope.metainfo import Models
|
| 12 |
+
from modelscope.models import TorchModel
|
| 13 |
+
from modelscope.models.base import Tensor
|
| 14 |
+
from modelscope.models.builder import MODELS
|
| 15 |
+
from modelscope.utils.audio.audio_utils import update_conf
|
| 16 |
+
from modelscope.utils.constant import Tasks
|
| 17 |
+
from .cmvn import GlobalCMVN, load_kaldi_cmvn
|
| 18 |
+
from .fsmn import FSMN
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@MODELS.register_module(
|
| 22 |
+
Tasks.keyword_spotting,
|
| 23 |
+
module_name=Models.speech_kws_fsmn_char_ctc_nearfield)
|
| 24 |
+
class FSMNDecorator(TorchModel):
|
| 25 |
+
r""" A decorator of FSMN for integrating into modelscope framework """
|
| 26 |
+
|
| 27 |
+
def __init__(self,
|
| 28 |
+
model_dir: str,
|
| 29 |
+
cmvn_file: str = None,
|
| 30 |
+
backbone: dict = None,
|
| 31 |
+
input_dim: int = 400,
|
| 32 |
+
output_dim: int = 2599,
|
| 33 |
+
training: Optional[bool] = False,
|
| 34 |
+
*args,
|
| 35 |
+
**kwargs):
|
| 36 |
+
"""initialize the fsmn model from the `model_dir` path.
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
model_dir (str): the model path.
|
| 40 |
+
cmvn_file (str): cmvn file
|
| 41 |
+
backbone (dict): params related to backbone
|
| 42 |
+
input_dim (int): input dimension of network
|
| 43 |
+
output_dim (int): output dimension of network
|
| 44 |
+
training (bool): training or inference mode
|
| 45 |
+
"""
|
| 46 |
+
super().__init__(model_dir, *args, **kwargs)
|
| 47 |
+
|
| 48 |
+
self.model = None
|
| 49 |
+
self.model_cfg = None
|
| 50 |
+
|
| 51 |
+
if training:
|
| 52 |
+
self.model = self.init_model(cmvn_file, backbone, input_dim,
|
| 53 |
+
output_dim)
|
| 54 |
+
else:
|
| 55 |
+
self.model_cfg = {
|
| 56 |
+
'model_workspace': model_dir,
|
| 57 |
+
'config_path': os.path.join(model_dir, 'config.yaml')
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
def __del__(self):
|
| 61 |
+
if hasattr(self, 'tmp_dir'):
|
| 62 |
+
self.tmp_dir.cleanup()
|
| 63 |
+
|
| 64 |
+
def forward(self, input) -> Dict[str, Tensor]:
|
| 65 |
+
"""
|
| 66 |
+
Args:
|
| 67 |
+
input (torch.Tensor): Input tensor (B, T, D)
|
| 68 |
+
"""
|
| 69 |
+
if self.model is not None and input is not None:
|
| 70 |
+
return self.model.forward(input)
|
| 71 |
+
else:
|
| 72 |
+
return self.model_cfg
|
| 73 |
+
|
| 74 |
+
def init_model(self, cmvn_file, backbone, input_dim, output_dim):
|
| 75 |
+
if cmvn_file is not None:
|
| 76 |
+
mean, istd = load_kaldi_cmvn(cmvn_file)
|
| 77 |
+
global_cmvn = GlobalCMVN(
|
| 78 |
+
torch.from_numpy(mean).float(),
|
| 79 |
+
torch.from_numpy(istd).float(),
|
| 80 |
+
)
|
| 81 |
+
else:
|
| 82 |
+
global_cmvn = None
|
| 83 |
+
|
| 84 |
+
hidden_dim = 128
|
| 85 |
+
preprocessing = None
|
| 86 |
+
|
| 87 |
+
input_affine_dim = backbone['input_affine_dim']
|
| 88 |
+
num_layers = backbone['num_layers']
|
| 89 |
+
linear_dim = backbone['linear_dim']
|
| 90 |
+
proj_dim = backbone['proj_dim']
|
| 91 |
+
left_order = backbone['left_order']
|
| 92 |
+
right_order = backbone['right_order']
|
| 93 |
+
left_stride = backbone['left_stride']
|
| 94 |
+
right_stride = backbone['right_stride']
|
| 95 |
+
output_affine_dim = backbone['output_affine_dim']
|
| 96 |
+
backbone = FSMN(input_dim, input_affine_dim, num_layers, linear_dim,
|
| 97 |
+
proj_dim, left_order, right_order, left_stride,
|
| 98 |
+
right_stride, output_affine_dim, output_dim)
|
| 99 |
+
|
| 100 |
+
classifier = None
|
| 101 |
+
activation = None
|
| 102 |
+
|
| 103 |
+
kws_model = KWSModel(input_dim, output_dim, hidden_dim, global_cmvn,
|
| 104 |
+
preprocessing, backbone, classifier, activation)
|
| 105 |
+
return kws_model
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
class KWSModel(nn.Module):
|
| 109 |
+
"""Our model consists of four parts:
|
| 110 |
+
1. global_cmvn: Optional, (idim, idim)
|
| 111 |
+
2. preprocessing: feature dimension projection, (idim, hdim)
|
| 112 |
+
3. backbone: backbone or feature extractor of the whole network, (hdim, hdim)
|
| 113 |
+
4. classifier: output layer or classifier of KWS model, (hdim, odim)
|
| 114 |
+
5. activation:
|
| 115 |
+
nn.Sigmoid for wakeup word
|
| 116 |
+
nn.Identity for speech command dataset
|
| 117 |
+
"""
|
| 118 |
+
|
| 119 |
+
def __init__(
|
| 120 |
+
self,
|
| 121 |
+
idim: int,
|
| 122 |
+
odim: int,
|
| 123 |
+
hdim: int,
|
| 124 |
+
global_cmvn: Optional[nn.Module],
|
| 125 |
+
preprocessing: Optional[nn.Module],
|
| 126 |
+
backbone: nn.Module,
|
| 127 |
+
classifier: nn.Module,
|
| 128 |
+
activation: nn.Module,
|
| 129 |
+
):
|
| 130 |
+
"""
|
| 131 |
+
Args:
|
| 132 |
+
idim (int): input dimension of network
|
| 133 |
+
odim (int): output dimension of network
|
| 134 |
+
hdim (int): hidden dimension of network
|
| 135 |
+
global_cmvn (nn.Module): cmvn for input feature, (idim, idim)
|
| 136 |
+
preprocessing (nn.Module): feature dimension projection, (idim, hdim)
|
| 137 |
+
backbone (nn.Module): backbone or feature extractor of the whole network, (hdim, hdim)
|
| 138 |
+
classifier (nn.Module): output layer or classifier of KWS model, (hdim, odim)
|
| 139 |
+
activation (nn.Module): nn.Identity for training, nn.Sigmoid for inference
|
| 140 |
+
"""
|
| 141 |
+
super().__init__()
|
| 142 |
+
self.idim = idim
|
| 143 |
+
self.odim = odim
|
| 144 |
+
self.hdim = hdim
|
| 145 |
+
self.global_cmvn = global_cmvn
|
| 146 |
+
self.preprocessing = preprocessing
|
| 147 |
+
self.backbone = backbone
|
| 148 |
+
self.classifier = classifier
|
| 149 |
+
self.activation = activation
|
| 150 |
+
|
| 151 |
+
def to_kaldi_net(self):
|
| 152 |
+
return self.backbone.to_kaldi_net()
|
| 153 |
+
|
| 154 |
+
def to_pytorch_net(self, kaldi_file):
|
| 155 |
+
return self.backbone.to_pytorch_net(kaldi_file)
|
| 156 |
+
|
| 157 |
+
def forward(
|
| 158 |
+
self,
|
| 159 |
+
x: torch.Tensor,
|
| 160 |
+
in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float)
|
| 161 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 162 |
+
if self.global_cmvn is not None:
|
| 163 |
+
x = self.global_cmvn(x)
|
| 164 |
+
if self.preprocessing is not None:
|
| 165 |
+
x = self.preprocessing(x)
|
| 166 |
+
|
| 167 |
+
x, out_cache = self.backbone(x, in_cache)
|
| 168 |
+
|
| 169 |
+
if self.classifier is not None:
|
| 170 |
+
x = self.classifier(x)
|
| 171 |
+
if self.activation is not None:
|
| 172 |
+
x = self.activation(x)
|
| 173 |
+
return x, out_cache
|
| 174 |
+
|
| 175 |
+
def fuse_modules(self):
|
| 176 |
+
if self.preprocessing is not None:
|
| 177 |
+
self.preprocessing.fuse_modules()
|
| 178 |
+
self.backbone.fuse_modules()
|
runtime/Lib/site_packages/models/audio/punc/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Any, Dict
|
| 5 |
+
|
| 6 |
+
from modelscope.metainfo import Models
|
| 7 |
+
from modelscope.models.base import Model
|
| 8 |
+
from modelscope.models.builder import MODELS
|
| 9 |
+
from modelscope.utils.constant import Frameworks, Tasks
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@MODELS.register_module(Tasks.punctuation, module_name=Models.generic_punc)
|
| 13 |
+
class PunctuationProcessing(Model):
|
| 14 |
+
|
| 15 |
+
def __init__(self, model_dir: str, punc_model_name: str,
|
| 16 |
+
punc_model_config: Dict[str, Any], *args, **kwargs):
|
| 17 |
+
"""initialize the info of model.
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
model_dir (str): the model path.
|
| 21 |
+
punc_model_name (str): the itn model name from configuration.json
|
| 22 |
+
punc_model_config (Dict[str, Any]): the detail config about model from configuration.json
|
| 23 |
+
"""
|
| 24 |
+
super().__init__(model_dir, punc_model_name, punc_model_config, *args,
|
| 25 |
+
**kwargs)
|
| 26 |
+
self.model_cfg = {
|
| 27 |
+
# the recognition model dir path
|
| 28 |
+
'model_workspace': model_dir,
|
| 29 |
+
# the itn model name
|
| 30 |
+
'punc_model': punc_model_name,
|
| 31 |
+
# the am model file path
|
| 32 |
+
'punc_model_path': os.path.join(model_dir, punc_model_name),
|
| 33 |
+
# the recognition model config dict
|
| 34 |
+
'model_config': punc_model_config
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
def forward(self) -> Dict[str, Any]:
|
| 38 |
+
"""
|
| 39 |
+
just return the model config
|
| 40 |
+
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
return self.model_cfg
|
runtime/Lib/site_packages/models/audio/separation/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from modelscope.utils.import_utils import LazyImportModule
|
| 5 |
+
|
| 6 |
+
if TYPE_CHECKING:
|
| 7 |
+
from .mossformer import MossFormer
|
| 8 |
+
from .m2.mossformer import MossFormer2
|
| 9 |
+
|
| 10 |
+
else:
|
| 11 |
+
_import_structure = {
|
| 12 |
+
'mossformer': ['MossFormer'],
|
| 13 |
+
'm2.mossformer': ['MossFormer2'],
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
import sys
|
| 17 |
+
|
| 18 |
+
sys.modules[__name__] = LazyImportModule(
|
| 19 |
+
__name__,
|
| 20 |
+
globals()['__file__'],
|
| 21 |
+
_import_structure,
|
| 22 |
+
module_spec=__spec__,
|
| 23 |
+
extra_objects={},
|
| 24 |
+
)
|
runtime/Lib/site_packages/models/audio/separation/layer_norm.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2018 Northwestern Polytechnical University (author: Ke Wang)
|
| 2 |
+
# made publicly available under the MIT License
|
| 3 |
+
# at https://github.com/wangkenpu/Conv-TasNet-PyTorch/blob/64188ffa48971218fdd68b66906970f215d7eca2/model/layer_norm.py
|
| 4 |
+
|
| 5 |
+
from __future__ import absolute_import, division, print_function
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class CLayerNorm(nn.LayerNorm):
|
| 12 |
+
"""Channel-wise layer normalization."""
|
| 13 |
+
|
| 14 |
+
def __init__(self, *args, **kwargs):
|
| 15 |
+
super(CLayerNorm, self).__init__(*args, **kwargs)
|
| 16 |
+
|
| 17 |
+
def forward(self, sample):
|
| 18 |
+
"""Forward function.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
sample: [batch_size, channels, length]
|
| 22 |
+
"""
|
| 23 |
+
if sample.dim() != 3:
|
| 24 |
+
raise RuntimeError('{} only accept 3-D tensor as input'.format(
|
| 25 |
+
self.__name__))
|
| 26 |
+
# [N, C, T] -> [N, T, C]
|
| 27 |
+
sample = torch.transpose(sample, 1, 2)
|
| 28 |
+
# LayerNorm
|
| 29 |
+
sample = super().forward(sample)
|
| 30 |
+
# [N, T, C] -> [N, C, T]
|
| 31 |
+
sample = torch.transpose(sample, 1, 2)
|
| 32 |
+
return sample
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class GLayerNorm(nn.Module):
|
| 36 |
+
"""Global Layer Normalization for TasNet."""
|
| 37 |
+
|
| 38 |
+
def __init__(self, channels, eps=1e-5):
|
| 39 |
+
super(GLayerNorm, self).__init__()
|
| 40 |
+
self.eps = eps
|
| 41 |
+
self.norm_dim = channels
|
| 42 |
+
self.gamma = nn.Parameter(torch.Tensor(channels))
|
| 43 |
+
self.beta = nn.Parameter(torch.Tensor(channels))
|
| 44 |
+
self.reset_parameters()
|
| 45 |
+
|
| 46 |
+
def reset_parameters(self):
|
| 47 |
+
nn.init.ones_(self.gamma)
|
| 48 |
+
nn.init.zeros_(self.beta)
|
| 49 |
+
|
| 50 |
+
def forward(self, sample):
|
| 51 |
+
"""Forward function.
|
| 52 |
+
|
| 53 |
+
Args:
|
| 54 |
+
sample: [batch_size, channels, length]
|
| 55 |
+
"""
|
| 56 |
+
if sample.dim() != 3:
|
| 57 |
+
raise RuntimeError('{} only accept 3-D tensor as input'.format(
|
| 58 |
+
self.__name__))
|
| 59 |
+
# [N, C, T] -> [N, T, C]
|
| 60 |
+
sample = torch.transpose(sample, 1, 2)
|
| 61 |
+
# Mean and variance [N, 1, 1]
|
| 62 |
+
mean = torch.mean(sample, (1, 2), keepdim=True)
|
| 63 |
+
var = torch.mean((sample - mean)**2, (1, 2), keepdim=True)
|
| 64 |
+
sample = (sample
|
| 65 |
+
- mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta
|
| 66 |
+
# [N, T, C] -> [N, C, T]
|
| 67 |
+
sample = torch.transpose(sample, 1, 2)
|
| 68 |
+
return sample
|
runtime/Lib/site_packages/models/audio/separation/m2/__init__.py
ADDED
|
File without changes
|
runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.init as init
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
EPS = 1e-8
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class GlobalLayerNorm(nn.Module):
|
| 12 |
+
"""Calculate Global Layer Normalization.
|
| 13 |
+
|
| 14 |
+
Args:
|
| 15 |
+
dim : (int or list or torch.Size)
|
| 16 |
+
Input shape from an expected input of size.
|
| 17 |
+
eps : float
|
| 18 |
+
A value added to the denominator for numerical stability.
|
| 19 |
+
elementwise_affine : bool
|
| 20 |
+
A boolean value that when set to True,
|
| 21 |
+
this module has learnable per-element affine parameters
|
| 22 |
+
initialized to ones (for weights) and zeros (for biases).
|
| 23 |
+
|
| 24 |
+
Example:
|
| 25 |
+
-------
|
| 26 |
+
>>> x = torch.randn(5, 10, 20)
|
| 27 |
+
>>> GLN = GlobalLayerNorm(10, 3)
|
| 28 |
+
>>> x_norm = GLN(x)
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True):
|
| 32 |
+
super(GlobalLayerNorm, self).__init__()
|
| 33 |
+
self.dim = dim
|
| 34 |
+
self.eps = eps
|
| 35 |
+
self.elementwise_affine = elementwise_affine
|
| 36 |
+
|
| 37 |
+
if self.elementwise_affine:
|
| 38 |
+
if shape == 3:
|
| 39 |
+
self.weight = nn.Parameter(torch.ones(self.dim, 1))
|
| 40 |
+
self.bias = nn.Parameter(torch.zeros(self.dim, 1))
|
| 41 |
+
if shape == 4:
|
| 42 |
+
self.weight = nn.Parameter(torch.ones(self.dim, 1, 1))
|
| 43 |
+
self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1))
|
| 44 |
+
else:
|
| 45 |
+
self.register_parameter('weight', None)
|
| 46 |
+
self.register_parameter('bias', None)
|
| 47 |
+
|
| 48 |
+
def forward(self, x):
|
| 49 |
+
"""Returns the normalized tensor.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
x : torch.Tensor
|
| 53 |
+
Tensor of size [N, C, K, S] or [N, C, L].
|
| 54 |
+
"""
|
| 55 |
+
# x = N x C x K x S or N x C x L
|
| 56 |
+
# N x 1 x 1
|
| 57 |
+
# cln: mean,var N x 1 x K x S
|
| 58 |
+
# gln: mean,var N x 1 x 1
|
| 59 |
+
if x.dim() == 3:
|
| 60 |
+
mean = torch.mean(x, (1, 2), keepdim=True)
|
| 61 |
+
var = torch.mean((x - mean)**2, (1, 2), keepdim=True)
|
| 62 |
+
if self.elementwise_affine:
|
| 63 |
+
# yapf: disable
|
| 64 |
+
x = (self.weight * (x - mean) / torch.sqrt(var + self.eps)
|
| 65 |
+
+ self.bias)
|
| 66 |
+
# yapf: enable
|
| 67 |
+
else:
|
| 68 |
+
x = (x - mean) / torch.sqrt(var + self.eps)
|
| 69 |
+
|
| 70 |
+
if x.dim() == 4:
|
| 71 |
+
mean = torch.mean(x, (1, 2, 3), keepdim=True)
|
| 72 |
+
var = torch.mean((x - mean)**2, (1, 2, 3), keepdim=True)
|
| 73 |
+
if self.elementwise_affine:
|
| 74 |
+
# yapf: disable
|
| 75 |
+
x = (self.weight * (x - mean) / torch.sqrt(var + self.eps)
|
| 76 |
+
+ self.bias)
|
| 77 |
+
# yapf: enable
|
| 78 |
+
else:
|
| 79 |
+
x = (x - mean) / torch.sqrt(var + self.eps)
|
| 80 |
+
return x
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class CumulativeLayerNorm(nn.LayerNorm):
|
| 84 |
+
"""Calculate Cumulative Layer Normalization.
|
| 85 |
+
|
| 86 |
+
Args:
|
| 87 |
+
dim : int
|
| 88 |
+
Dimension that you want to normalize.
|
| 89 |
+
elementwise_affine : True
|
| 90 |
+
Learnable per-element affine parameters.
|
| 91 |
+
|
| 92 |
+
Example:
|
| 93 |
+
-------
|
| 94 |
+
>>> x = torch.randn(5, 10, 20)
|
| 95 |
+
>>> CLN = CumulativeLayerNorm(10)
|
| 96 |
+
>>> x_norm = CLN(x)
|
| 97 |
+
"""
|
| 98 |
+
|
| 99 |
+
def __init__(self, dim, elementwise_affine=True):
|
| 100 |
+
super(CumulativeLayerNorm, self).__init__(
|
| 101 |
+
dim, elementwise_affine=elementwise_affine, eps=1e-8)
|
| 102 |
+
|
| 103 |
+
def forward(self, x):
|
| 104 |
+
"""Returns the normalized tensor.
|
| 105 |
+
|
| 106 |
+
Args:
|
| 107 |
+
x : torch.Tensor
|
| 108 |
+
Tensor size [N, C, K, S] or [N, C, L]
|
| 109 |
+
"""
|
| 110 |
+
# x: N x C x K x S or N x C x L
|
| 111 |
+
# N x K x S x C
|
| 112 |
+
if x.dim() == 4:
|
| 113 |
+
x = x.permute(0, 2, 3, 1).contiguous()
|
| 114 |
+
# N x K x S x C == only channel norm
|
| 115 |
+
x = super().forward(x)
|
| 116 |
+
# N x C x K x S
|
| 117 |
+
x = x.permute(0, 3, 1, 2).contiguous()
|
| 118 |
+
if x.dim() == 3:
|
| 119 |
+
x = torch.transpose(x, 1, 2)
|
| 120 |
+
# N x L x C == only channel norm
|
| 121 |
+
x = super().forward(x)
|
| 122 |
+
# N x C x L
|
| 123 |
+
x = torch.transpose(x, 1, 2)
|
| 124 |
+
return x
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class Transpose(nn.Module):
|
| 128 |
+
""" Wrapper class of torch.transpose() for Sequential module. """
|
| 129 |
+
|
| 130 |
+
def __init__(self, shape: tuple):
|
| 131 |
+
super(Transpose, self).__init__()
|
| 132 |
+
self.shape = shape
|
| 133 |
+
|
| 134 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 135 |
+
return x.transpose(*self.shape)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class DepthwiseConv1d(nn.Module):
|
| 139 |
+
"""When groups == in_channels and out_channels == K * in_channels, where K is a positive integer,
|
| 140 |
+
this operation is termed in literature as depthwise convolution.
|
| 141 |
+
|
| 142 |
+
Args:
|
| 143 |
+
in_channels (int): Number of channels in the input
|
| 144 |
+
out_channels (int): Number of channels produced by the convolution
|
| 145 |
+
kernel_size (int or tuple): Size of the convolving kernel
|
| 146 |
+
stride (int, optional): Stride of the convolution. Default: 1
|
| 147 |
+
padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0
|
| 148 |
+
bias (bool, optional): If True, adds a learnable bias to the output. Default: True
|
| 149 |
+
Inputs: inputs
|
| 150 |
+
- **inputs** (batch, in_channels, time): Tensor containing input vector
|
| 151 |
+
Returns: outputs
|
| 152 |
+
- **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution.
|
| 153 |
+
"""
|
| 154 |
+
|
| 155 |
+
def __init__(
|
| 156 |
+
self,
|
| 157 |
+
in_channels: int,
|
| 158 |
+
out_channels: int,
|
| 159 |
+
kernel_size: int,
|
| 160 |
+
stride: int = 1,
|
| 161 |
+
padding: int = 0,
|
| 162 |
+
bias: bool = False,
|
| 163 |
+
) -> None:
|
| 164 |
+
super(DepthwiseConv1d, self).__init__()
|
| 165 |
+
assert out_channels % in_channels == 0, 'out_channels should be constant multiple of in_channels'
|
| 166 |
+
self.conv = nn.Conv1d(
|
| 167 |
+
in_channels=in_channels,
|
| 168 |
+
out_channels=out_channels,
|
| 169 |
+
kernel_size=kernel_size,
|
| 170 |
+
groups=in_channels,
|
| 171 |
+
stride=stride,
|
| 172 |
+
padding=padding,
|
| 173 |
+
bias=bias,
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
def forward(self, inputs: Tensor) -> Tensor:
|
| 177 |
+
return self.conv(inputs)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class ConvModule(nn.Module):
|
| 181 |
+
"""
|
| 182 |
+
Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU).
|
| 183 |
+
This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution
|
| 184 |
+
to aid training deep models.
|
| 185 |
+
|
| 186 |
+
Args:
|
| 187 |
+
in_channels (int): Number of channels in the input
|
| 188 |
+
kernel_size (int or tuple, optional): Size of the convolving kernel Default: 17
|
| 189 |
+
dropout_p (float, optional): probability of dropout
|
| 190 |
+
Inputs: inputs
|
| 191 |
+
inputs (batch, time, dim): Tensor contains input sequences
|
| 192 |
+
Outputs: outputs
|
| 193 |
+
outputs (batch, time, dim): Tensor produces by conformer convolution module.
|
| 194 |
+
"""
|
| 195 |
+
|
| 196 |
+
def __init__(
|
| 197 |
+
self,
|
| 198 |
+
in_channels: int,
|
| 199 |
+
kernel_size: int = 17,
|
| 200 |
+
expansion_factor: int = 2,
|
| 201 |
+
dropout_p: float = 0.1,
|
| 202 |
+
) -> None:
|
| 203 |
+
super(ConvModule, self).__init__()
|
| 204 |
+
assert (
|
| 205 |
+
kernel_size - 1
|
| 206 |
+
) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding"
|
| 207 |
+
assert expansion_factor == 2, 'Currently, Only Supports expansion_factor 2'
|
| 208 |
+
|
| 209 |
+
self.sequential = nn.Sequential(
|
| 210 |
+
Transpose(shape=(1, 2)),
|
| 211 |
+
DepthwiseConv1d(
|
| 212 |
+
in_channels,
|
| 213 |
+
in_channels,
|
| 214 |
+
kernel_size,
|
| 215 |
+
stride=1,
|
| 216 |
+
padding=(kernel_size - 1) // 2),
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
def forward(self, inputs: Tensor) -> Tensor:
|
| 220 |
+
return inputs + self.sequential(inputs).transpose(1, 2)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class DilatedDenseNet(nn.Module):
|
| 224 |
+
|
| 225 |
+
def __init__(self, depth=4, lorder=20, in_channels=64):
|
| 226 |
+
super(DilatedDenseNet, self).__init__()
|
| 227 |
+
self.depth = depth
|
| 228 |
+
self.in_channels = in_channels
|
| 229 |
+
self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.)
|
| 230 |
+
self.twidth = lorder * 2 - 1
|
| 231 |
+
self.kernel_size = (self.twidth, 1)
|
| 232 |
+
for i in range(self.depth):
|
| 233 |
+
dil = 2**i
|
| 234 |
+
pad_length = lorder + (dil - 1) * (lorder - 1) - 1
|
| 235 |
+
setattr(self, 'pad{}'.format(i + 1),
|
| 236 |
+
nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.))
|
| 237 |
+
setattr(
|
| 238 |
+
self, 'conv{}'.format(i + 1),
|
| 239 |
+
nn.Conv2d(
|
| 240 |
+
self.in_channels * (i + 1),
|
| 241 |
+
self.in_channels,
|
| 242 |
+
kernel_size=self.kernel_size,
|
| 243 |
+
dilation=(dil, 1),
|
| 244 |
+
groups=self.in_channels,
|
| 245 |
+
bias=False))
|
| 246 |
+
setattr(self, 'norm{}'.format(i + 1),
|
| 247 |
+
nn.InstanceNorm2d(in_channels, affine=True))
|
| 248 |
+
setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels))
|
| 249 |
+
|
| 250 |
+
def forward(self, x):
|
| 251 |
+
x = torch.unsqueeze(x, 1)
|
| 252 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 253 |
+
skip = x_per
|
| 254 |
+
for i in range(self.depth):
|
| 255 |
+
out = getattr(self, 'pad{}'.format(i + 1))(skip)
|
| 256 |
+
out = getattr(self, 'conv{}'.format(i + 1))(out)
|
| 257 |
+
out = getattr(self, 'norm{}'.format(i + 1))(out)
|
| 258 |
+
out = getattr(self, 'prelu{}'.format(i + 1))(out)
|
| 259 |
+
skip = torch.cat([out, skip], dim=1)
|
| 260 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 261 |
+
return out1.squeeze(1)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
class FFConvMDilated(nn.Module):
|
| 265 |
+
|
| 266 |
+
def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1):
|
| 267 |
+
super().__init__()
|
| 268 |
+
self.mdl = nn.Sequential(
|
| 269 |
+
norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(),
|
| 270 |
+
DilatedDenseNet(depth=2, lorder=17, in_channels=dim_out),
|
| 271 |
+
nn.Dropout(dropout))
|
| 272 |
+
|
| 273 |
+
def forward(
|
| 274 |
+
self,
|
| 275 |
+
x,
|
| 276 |
+
):
|
| 277 |
+
output = self.mdl(x)
|
| 278 |
+
return output
|
runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Alibaba, Inc. and its affiliates.
|
| 2 |
+
|
| 3 |
+
import torch as th
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class UniDeepFsmn(nn.Module):
|
| 9 |
+
|
| 10 |
+
def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
|
| 11 |
+
super(UniDeepFsmn, self).__init__()
|
| 12 |
+
|
| 13 |
+
self.input_dim = input_dim
|
| 14 |
+
self.output_dim = output_dim
|
| 15 |
+
if lorder is None:
|
| 16 |
+
return
|
| 17 |
+
self.lorder = lorder
|
| 18 |
+
self.hidden_size = hidden_size
|
| 19 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 20 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 21 |
+
self.conv1 = nn.Conv2d(
|
| 22 |
+
output_dim,
|
| 23 |
+
output_dim, [lorder + lorder - 1, 1], [1, 1],
|
| 24 |
+
groups=output_dim,
|
| 25 |
+
bias=False)
|
| 26 |
+
|
| 27 |
+
def forward(self, input):
|
| 28 |
+
f1 = F.relu(self.linear(input))
|
| 29 |
+
p1 = self.project(f1)
|
| 30 |
+
x = th.unsqueeze(p1, 1)
|
| 31 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 32 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1])
|
| 33 |
+
out = x_per + self.conv1(y)
|
| 34 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 35 |
+
return input + out1.squeeze()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class UniDeepFsmnDual(nn.Module):
|
| 39 |
+
|
| 40 |
+
def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
|
| 41 |
+
super(UniDeepFsmnDual, self).__init__()
|
| 42 |
+
|
| 43 |
+
self.input_dim = input_dim
|
| 44 |
+
self.output_dim = output_dim
|
| 45 |
+
if lorder is None:
|
| 46 |
+
return
|
| 47 |
+
self.lorder = lorder
|
| 48 |
+
self.hidden_size = hidden_size
|
| 49 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 50 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 51 |
+
self.conv1 = nn.Conv2d(
|
| 52 |
+
output_dim,
|
| 53 |
+
output_dim, [lorder + lorder - 1, 1], [1, 1],
|
| 54 |
+
groups=output_dim,
|
| 55 |
+
bias=False)
|
| 56 |
+
self.conv2 = nn.Conv2d(
|
| 57 |
+
output_dim,
|
| 58 |
+
output_dim, [lorder + lorder - 1, 1], [1, 1],
|
| 59 |
+
groups=output_dim // 4,
|
| 60 |
+
bias=False)
|
| 61 |
+
|
| 62 |
+
def forward(self, input):
|
| 63 |
+
|
| 64 |
+
f1 = F.relu(self.linear(input))
|
| 65 |
+
p1 = self.project(f1)
|
| 66 |
+
x = th.unsqueeze(p1, 1)
|
| 67 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 68 |
+
y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1])
|
| 69 |
+
conv1_out = x_per + self.conv1(y)
|
| 70 |
+
z = F.pad(conv1_out, [0, 0, self.lorder - 1, self.lorder - 1])
|
| 71 |
+
out = conv1_out + self.conv2(z)
|
| 72 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 73 |
+
return input + out1.squeeze()
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class DilatedDenseNet(nn.Module):
|
| 77 |
+
|
| 78 |
+
def __init__(self, depth=4, lorder=20, in_channels=64):
|
| 79 |
+
super(DilatedDenseNet, self).__init__()
|
| 80 |
+
self.depth = depth
|
| 81 |
+
self.in_channels = in_channels
|
| 82 |
+
self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.)
|
| 83 |
+
self.twidth = lorder * 2 - 1
|
| 84 |
+
self.kernel_size = (self.twidth, 1)
|
| 85 |
+
for i in range(self.depth):
|
| 86 |
+
dil = 2**i
|
| 87 |
+
pad_length = lorder + (dil - 1) * (lorder - 1) - 1
|
| 88 |
+
setattr(self, 'pad{}'.format(i + 1),
|
| 89 |
+
nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.))
|
| 90 |
+
setattr(
|
| 91 |
+
self, 'conv{}'.format(i + 1),
|
| 92 |
+
nn.Conv2d(
|
| 93 |
+
self.in_channels * (i + 1),
|
| 94 |
+
self.in_channels,
|
| 95 |
+
kernel_size=self.kernel_size,
|
| 96 |
+
dilation=(dil, 1),
|
| 97 |
+
groups=self.in_channels,
|
| 98 |
+
bias=False))
|
| 99 |
+
setattr(self, 'norm{}'.format(i + 1),
|
| 100 |
+
nn.InstanceNorm2d(in_channels, affine=True))
|
| 101 |
+
setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels))
|
| 102 |
+
|
| 103 |
+
def forward(self, x):
|
| 104 |
+
skip = x
|
| 105 |
+
for i in range(self.depth):
|
| 106 |
+
out = getattr(self, 'pad{}'.format(i + 1))(skip)
|
| 107 |
+
out = getattr(self, 'conv{}'.format(i + 1))(out)
|
| 108 |
+
out = getattr(self, 'norm{}'.format(i + 1))(out)
|
| 109 |
+
out = getattr(self, 'prelu{}'.format(i + 1))(out)
|
| 110 |
+
skip = th.cat([out, skip], dim=1)
|
| 111 |
+
return out
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class UniDeepFsmnDilated(nn.Module):
|
| 115 |
+
|
| 116 |
+
def __init__(self,
|
| 117 |
+
input_dim,
|
| 118 |
+
output_dim,
|
| 119 |
+
lorder=None,
|
| 120 |
+
hidden_size=None,
|
| 121 |
+
depth=2):
|
| 122 |
+
super(UniDeepFsmnDilated, self).__init__()
|
| 123 |
+
|
| 124 |
+
self.input_dim = input_dim
|
| 125 |
+
self.output_dim = output_dim
|
| 126 |
+
self.depth = depth
|
| 127 |
+
if lorder is None:
|
| 128 |
+
return
|
| 129 |
+
self.lorder = lorder
|
| 130 |
+
self.hidden_size = hidden_size
|
| 131 |
+
self.linear = nn.Linear(input_dim, hidden_size)
|
| 132 |
+
self.project = nn.Linear(hidden_size, output_dim, bias=False)
|
| 133 |
+
self.conv = DilatedDenseNet(
|
| 134 |
+
depth=self.depth, lorder=lorder, in_channels=output_dim)
|
| 135 |
+
|
| 136 |
+
def forward(self, input):
|
| 137 |
+
f1 = F.relu(self.linear(input))
|
| 138 |
+
p1 = self.project(f1)
|
| 139 |
+
x = th.unsqueeze(p1, 1)
|
| 140 |
+
x_per = x.permute(0, 3, 2, 1)
|
| 141 |
+
out = self.conv(x_per)
|
| 142 |
+
out1 = out.permute(0, 3, 2, 1)
|
| 143 |
+
|
| 144 |
+
return input + out1.squeeze()
|
runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2018 Northwestern Polytechnical University (author: Ke Wang)
|
| 2 |
+
|
| 3 |
+
from __future__ import absolute_import, division, print_function
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CLayerNorm(nn.LayerNorm):
|
| 10 |
+
"""Channel-wise layer normalization."""
|
| 11 |
+
|
| 12 |
+
def __init__(self, *args, **kwargs):
|
| 13 |
+
super(CLayerNorm, self).__init__(*args, **kwargs)
|
| 14 |
+
|
| 15 |
+
def forward(self, sample):
|
| 16 |
+
"""Forward function.
|
| 17 |
+
|
| 18 |
+
Args:
|
| 19 |
+
sample: [batch_size, channels, length]
|
| 20 |
+
"""
|
| 21 |
+
if sample.dim() != 3:
|
| 22 |
+
raise RuntimeError('{} only accept 3-D tensor as input'.format(
|
| 23 |
+
self.__name__))
|
| 24 |
+
# [N, C, T] -> [N, T, C]
|
| 25 |
+
sample = torch.transpose(sample, 1, 2)
|
| 26 |
+
# LayerNorm
|
| 27 |
+
sample = super().forward(sample)
|
| 28 |
+
# [N, T, C] -> [N, C, T]
|
| 29 |
+
sample = torch.transpose(sample, 1, 2)
|
| 30 |
+
return sample
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class ILayerNorm(nn.InstanceNorm1d):
|
| 34 |
+
"""Channel-wise layer normalization."""
|
| 35 |
+
|
| 36 |
+
def __init__(self, *args, **kwargs):
|
| 37 |
+
super(ILayerNorm, self).__init__(*args, **kwargs)
|
| 38 |
+
|
| 39 |
+
def forward(self, sample):
|
| 40 |
+
"""Forward function.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
sample: [batch_size, channels, length]
|
| 44 |
+
"""
|
| 45 |
+
if sample.dim() != 3:
|
| 46 |
+
raise RuntimeError('{} only accept 3-D tensor as input'.format(
|
| 47 |
+
self.__name__))
|
| 48 |
+
# [N, C, T] -> [N, T, C]
|
| 49 |
+
sample = torch.transpose(sample, 1, 2)
|
| 50 |
+
# LayerNorm
|
| 51 |
+
sample = super().forward(sample)
|
| 52 |
+
# [N, T, C] -> [N, C, T]
|
| 53 |
+
sample = torch.transpose(sample, 1, 2)
|
| 54 |
+
return sample
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class GLayerNorm(nn.Module):
|
| 58 |
+
"""Global Layer Normalization for TasNet."""
|
| 59 |
+
|
| 60 |
+
def __init__(self, channels, eps=1e-5):
|
| 61 |
+
super(GLayerNorm, self).__init__()
|
| 62 |
+
self.eps = eps
|
| 63 |
+
self.norm_dim = channels
|
| 64 |
+
self.gamma = nn.Parameter(torch.Tensor(channels))
|
| 65 |
+
self.beta = nn.Parameter(torch.Tensor(channels))
|
| 66 |
+
# self.register_parameter('weight', self.gamma)
|
| 67 |
+
# self.register_parameter('bias', self.beta)
|
| 68 |
+
self.reset_parameters()
|
| 69 |
+
|
| 70 |
+
def reset_parameters(self):
|
| 71 |
+
nn.init.ones_(self.gamma)
|
| 72 |
+
nn.init.zeros_(self.beta)
|
| 73 |
+
|
| 74 |
+
def forward(self, sample):
|
| 75 |
+
"""Forward function.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
sample: [batch_size, channels, length]
|
| 79 |
+
"""
|
| 80 |
+
if sample.dim() != 3:
|
| 81 |
+
raise RuntimeError('{} only accept 3-D tensor as input'.format(
|
| 82 |
+
self.__name__))
|
| 83 |
+
# [N, C, T] -> [N, T, C]
|
| 84 |
+
sample = torch.transpose(sample, 1, 2)
|
| 85 |
+
# Mean and variance [N, 1, 1]
|
| 86 |
+
mean = torch.mean(sample, (1, 2), keepdim=True)
|
| 87 |
+
var = torch.mean((sample - mean)**2, (1, 2), keepdim=True)
|
| 88 |
+
sample = (sample
|
| 89 |
+
- mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta
|
| 90 |
+
# [N, T, C] -> [N, C, T]
|
| 91 |
+
sample = torch.transpose(sample, 1, 2)
|
| 92 |
+
return sample
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class _LayerNorm(nn.Module):
|
| 96 |
+
"""Layer Normalization base class."""
|
| 97 |
+
|
| 98 |
+
def __init__(self, channel_size):
|
| 99 |
+
super(_LayerNorm, self).__init__()
|
| 100 |
+
self.channel_size = channel_size
|
| 101 |
+
self.gamma = nn.Parameter(torch.ones(channel_size), requires_grad=True)
|
| 102 |
+
self.beta = nn.Parameter(torch.zeros(channel_size), requires_grad=True)
|
| 103 |
+
|
| 104 |
+
def apply_gain_and_bias(self, normed_x):
|
| 105 |
+
""" Assumes input of size `[batch, chanel, *]`. """
|
| 106 |
+
return (self.gamma * normed_x.transpose(1, -1) + self.beta).transpose(
|
| 107 |
+
1, -1)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
class GlobLayerNorm(_LayerNorm):
|
| 111 |
+
"""Global Layer Normalization (globLN)."""
|
| 112 |
+
|
| 113 |
+
def forward(self, x):
|
| 114 |
+
""" Applies forward pass.
|
| 115 |
+
Works for any input size > 2D.
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
x (:class:`torch.Tensor`): Shape `[batch, chan, *]`
|
| 119 |
+
Returns:
|
| 120 |
+
:class:`torch.Tensor`: gLN_x `[batch, chan, *]`
|
| 121 |
+
"""
|
| 122 |
+
dims = list(range(1, len(x.shape)))
|
| 123 |
+
mean = x.mean(dim=dims, keepdim=True)
|
| 124 |
+
var = torch.pow(x - mean, 2).mean(dim=dims, keepdim=True)
|
| 125 |
+
return self.apply_gain_and_bias((x - mean) / (var + 1e-8).sqrt())
|