jam134 commited on
Commit
5d3f98b
·
verified ·
1 Parent(s): 0d7d3f0

Upload 1580 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. runtime/Lib/site_packages/models/__init__.py +13 -0
  2. runtime/Lib/site_packages/models/audio/__init__.py +3 -0
  3. runtime/Lib/site_packages/models/audio/aec/__init__.py +0 -0
  4. runtime/Lib/site_packages/models/audio/aec/layers/__init__.py +0 -0
  5. runtime/Lib/site_packages/models/audio/aec/layers/activations.py +62 -0
  6. runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py +80 -0
  7. runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py +180 -0
  8. runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py +52 -0
  9. runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py +484 -0
  10. runtime/Lib/site_packages/models/audio/aec/network/__init__.py +0 -0
  11. runtime/Lib/site_packages/models/audio/aec/network/loss.py +396 -0
  12. runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py +250 -0
  13. runtime/Lib/site_packages/models/audio/aec/network/se_net.py +487 -0
  14. runtime/Lib/site_packages/models/audio/ans/__init__.py +23 -0
  15. runtime/Lib/site_packages/models/audio/ans/complex_nn.py +206 -0
  16. runtime/Lib/site_packages/models/audio/ans/conv_stft.py +113 -0
  17. runtime/Lib/site_packages/models/audio/ans/denoise_net.py +73 -0
  18. runtime/Lib/site_packages/models/audio/ans/frcrn.py +280 -0
  19. runtime/Lib/site_packages/models/audio/ans/layers/__init__.py +0 -0
  20. runtime/Lib/site_packages/models/audio/ans/layers/activations.py +62 -0
  21. runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py +86 -0
  22. runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py +31 -0
  23. runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py +156 -0
  24. runtime/Lib/site_packages/models/audio/ans/se_module_complex.py +27 -0
  25. runtime/Lib/site_packages/models/audio/ans/unet.py +276 -0
  26. runtime/Lib/site_packages/models/audio/asr/__init__.py +23 -0
  27. runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py +51 -0
  28. runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py +38 -0
  29. runtime/Lib/site_packages/models/audio/itn/__init__.py +22 -0
  30. runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py +44 -0
  31. runtime/Lib/site_packages/models/audio/kws/__init__.py +26 -0
  32. runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py +0 -0
  33. runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py +497 -0
  34. runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py +238 -0
  35. runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py +233 -0
  36. runtime/Lib/site_packages/models/audio/kws/farfield/model.py +98 -0
  37. runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py +123 -0
  38. runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py +32 -0
  39. runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py +0 -0
  40. runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py +99 -0
  41. runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py +521 -0
  42. runtime/Lib/site_packages/models/audio/kws/nearfield/model.py +178 -0
  43. runtime/Lib/site_packages/models/audio/punc/__init__.py +0 -0
  44. runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py +43 -0
  45. runtime/Lib/site_packages/models/audio/separation/__init__.py +24 -0
  46. runtime/Lib/site_packages/models/audio/separation/layer_norm.py +68 -0
  47. runtime/Lib/site_packages/models/audio/separation/m2/__init__.py +0 -0
  48. runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py +278 -0
  49. runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py +144 -0
  50. runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py +125 -0
runtime/Lib/site_packages/models/__init__.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.error import (AUDIO_IMPORT_ERROR,
5
+ TENSORFLOW_IMPORT_WARNING)
6
+ from modelscope.utils.import_utils import is_torch_available
7
+ from . import audio, cv, multi_modal, nlp
8
+ from .base import Head, Model
9
+ from .builder import BACKBONES, HEADS, MODELS, build_model
10
+
11
+ if is_torch_available():
12
+ from .base.base_torch_model import TorchModel
13
+ from .base.base_torch_head import TorchHead
runtime/Lib/site_packages/models/audio/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ from . import ans, asr, itn, kws, separation, sv, tts
runtime/Lib/site_packages/models/audio/aec/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/aec/layers/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/aec/layers/activations.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch.nn as nn
4
+
5
+ from .layer_base import LayerBase
6
+
7
+
8
+ class RectifiedLinear(LayerBase):
9
+
10
+ def __init__(self, input_dim, output_dim):
11
+ super(RectifiedLinear, self).__init__()
12
+ self.dim = input_dim
13
+ self.relu = nn.ReLU()
14
+
15
+ def forward(self, input):
16
+ return self.relu(input)
17
+
18
+ def to_kaldi_nnet(self):
19
+ re_str = ''
20
+ re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
21
+ return re_str
22
+
23
+ def load_kaldi_nnet(self, instr):
24
+ return instr
25
+
26
+
27
+ class LogSoftmax(LayerBase):
28
+
29
+ def __init__(self, input_dim, output_dim):
30
+ super(LogSoftmax, self).__init__()
31
+ self.dim = input_dim
32
+ self.ls = nn.LogSoftmax()
33
+
34
+ def forward(self, input):
35
+ return self.ls(input)
36
+
37
+ def to_kaldi_nnet(self):
38
+ re_str = ''
39
+ re_str += '<Softmax> %d %d\n' % (self.dim, self.dim)
40
+ return re_str
41
+
42
+ def load_kaldi_nnet(self, instr):
43
+ return instr
44
+
45
+
46
+ class Sigmoid(LayerBase):
47
+
48
+ def __init__(self, input_dim, output_dim):
49
+ super(Sigmoid, self).__init__()
50
+ self.dim = input_dim
51
+ self.sig = nn.Sigmoid()
52
+
53
+ def forward(self, input):
54
+ return self.sig(input)
55
+
56
+ def to_kaldi_nnet(self):
57
+ re_str = ''
58
+ re_str += '<Sigmoid> %d %d\n' % (self.dim, self.dim)
59
+ return re_str
60
+
61
+ def load_kaldi_nnet(self, instr):
62
+ return instr
runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import numpy as np
4
+ import torch as th
5
+ import torch.nn as nn
6
+
7
+ from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
8
+ to_kaldi_matrix)
9
+
10
+
11
+ class AffineTransform(LayerBase):
12
+
13
+ def __init__(self, input_dim, output_dim):
14
+ super(AffineTransform, self).__init__()
15
+ self.input_dim = input_dim
16
+ self.output_dim = output_dim
17
+ self.linear = nn.Linear(input_dim, output_dim)
18
+
19
+ def forward(self, input):
20
+ return self.linear(input)
21
+
22
+ def to_kaldi_nnet(self):
23
+ re_str = ''
24
+ re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
25
+ self.input_dim)
26
+ re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
27
+ linear_weights = self.state_dict()['linear.weight']
28
+ x = linear_weights.squeeze().numpy()
29
+ re_str += to_kaldi_matrix(x)
30
+ linear_bias = self.state_dict()['linear.bias']
31
+ x = linear_bias.squeeze().numpy()
32
+ re_str += to_kaldi_matrix(x)
33
+ return re_str
34
+
35
+ def to_raw_nnet(self, fid):
36
+ linear_weights = self.state_dict()['linear.weight']
37
+ x = linear_weights.squeeze().numpy()
38
+ x.tofile(fid)
39
+
40
+ linear_bias = self.state_dict()['linear.bias']
41
+ x = linear_bias.squeeze().numpy()
42
+ x.tofile(fid)
43
+
44
+ def load_kaldi_nnet(self, instr):
45
+ output = expect_token_number(
46
+ instr,
47
+ '<LearnRateCoef>',
48
+ )
49
+ if output is None:
50
+ raise Exception('AffineTransform format error for <LearnRateCoef>')
51
+ instr, lr = output
52
+
53
+ output = expect_token_number(instr, '<BiasLearnRateCoef>')
54
+ if output is None:
55
+ raise Exception(
56
+ 'AffineTransform format error for <BiasLearnRateCoef>')
57
+ instr, lr = output
58
+
59
+ output = expect_token_number(instr, '<MaxNorm>')
60
+ if output is None:
61
+ raise Exception('AffineTransform format error for <MaxNorm>')
62
+ instr, lr = output
63
+
64
+ output = expect_kaldi_matrix(instr)
65
+ if output is None:
66
+ raise Exception('AffineTransform format error for parsing matrix')
67
+ instr, mat = output
68
+
69
+ print(mat.shape)
70
+ self.linear.weight = th.nn.Parameter(
71
+ th.from_numpy(mat).type(th.FloatTensor))
72
+
73
+ output = expect_kaldi_matrix(instr)
74
+ if output is None:
75
+ raise Exception('AffineTransform format error for parsing matrix')
76
+ instr, mat = output
77
+ mat = np.squeeze(mat)
78
+ self.linear.bias = th.nn.Parameter(
79
+ th.from_numpy(mat).type(th.FloatTensor))
80
+ return instr
runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import numpy as np
4
+ import torch as th
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+ from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
9
+ to_kaldi_matrix)
10
+
11
+
12
+ class DeepFsmn(LayerBase):
13
+
14
+ def __init__(self,
15
+ input_dim,
16
+ output_dim,
17
+ lorder=None,
18
+ rorder=None,
19
+ hidden_size=None,
20
+ layer_norm=False,
21
+ dropout=0):
22
+ super(DeepFsmn, self).__init__()
23
+
24
+ self.input_dim = input_dim
25
+ self.output_dim = output_dim
26
+
27
+ if lorder is None:
28
+ return
29
+
30
+ self.lorder = lorder
31
+ self.rorder = rorder
32
+ self.hidden_size = hidden_size
33
+ self.layer_norm = layer_norm
34
+
35
+ self.linear = nn.Linear(input_dim, hidden_size)
36
+ self.norm = nn.LayerNorm(hidden_size)
37
+ self.drop1 = nn.Dropout(p=dropout)
38
+ self.drop2 = nn.Dropout(p=dropout)
39
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
40
+
41
+ self.conv1 = nn.Conv2d(
42
+ output_dim,
43
+ output_dim, [lorder, 1], [1, 1],
44
+ groups=output_dim,
45
+ bias=False)
46
+ self.conv2 = nn.Conv2d(
47
+ output_dim,
48
+ output_dim, [rorder, 1], [1, 1],
49
+ groups=output_dim,
50
+ bias=False)
51
+
52
+ def forward(self, input):
53
+
54
+ f1 = F.relu(self.linear(input))
55
+
56
+ f1 = self.drop1(f1)
57
+ if self.layer_norm:
58
+ f1 = self.norm(f1)
59
+
60
+ p1 = self.project(f1)
61
+
62
+ x = th.unsqueeze(p1, 1)
63
+
64
+ x_per = x.permute(0, 3, 2, 1)
65
+
66
+ y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
67
+ yr = F.pad(x_per, [0, 0, 0, self.rorder])
68
+ yr = yr[:, :, 1:, :]
69
+
70
+ out = x_per + self.conv1(y) + self.conv2(yr)
71
+ out = self.drop2(out)
72
+
73
+ out1 = out.permute(0, 3, 2, 1)
74
+
75
+ return input + out1.squeeze()
76
+
77
+ def to_kaldi_nnet(self):
78
+ re_str = ''
79
+ re_str += '<UniDeepFsmn> %d %d\n'\
80
+ % (self.output_dim, self.input_dim)
81
+ re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n'\
82
+ % (1, self.hidden_size, self.lorder, 1)
83
+ lfiters = self.state_dict()['conv1.weight']
84
+ x = np.flipud(lfiters.squeeze().numpy().T)
85
+ re_str += to_kaldi_matrix(x)
86
+ proj_weights = self.state_dict()['project.weight']
87
+ x = proj_weights.squeeze().numpy()
88
+ re_str += to_kaldi_matrix(x)
89
+ linear_weights = self.state_dict()['linear.weight']
90
+ x = linear_weights.squeeze().numpy()
91
+ re_str += to_kaldi_matrix(x)
92
+ linear_bias = self.state_dict()['linear.bias']
93
+ x = linear_bias.squeeze().numpy()
94
+ re_str += to_kaldi_matrix(x)
95
+ return re_str
96
+
97
+ def load_kaldi_nnet(self, instr):
98
+ output = expect_token_number(
99
+ instr,
100
+ '<LearnRateCoef>',
101
+ )
102
+ if output is None:
103
+ raise Exception('UniDeepFsmn format error for <LearnRateCoef>')
104
+ instr, lr = output
105
+
106
+ output = expect_token_number(
107
+ instr,
108
+ '<HidSize>',
109
+ )
110
+ if output is None:
111
+ raise Exception('UniDeepFsmn format error for <HidSize>')
112
+ instr, hiddensize = output
113
+ self.hidden_size = int(hiddensize)
114
+
115
+ output = expect_token_number(
116
+ instr,
117
+ '<LOrder>',
118
+ )
119
+ if output is None:
120
+ raise Exception('UniDeepFsmn format error for <LOrder>')
121
+ instr, lorder = output
122
+ self.lorder = int(lorder)
123
+
124
+ output = expect_token_number(
125
+ instr,
126
+ '<LStride>',
127
+ )
128
+ if output is None:
129
+ raise Exception('UniDeepFsmn format error for <LStride>')
130
+ instr, lstride = output
131
+ self.lstride = lstride
132
+
133
+ output = expect_token_number(
134
+ instr,
135
+ '<MaxNorm>',
136
+ )
137
+ if output is None:
138
+ raise Exception('UniDeepFsmn format error for <MaxNorm>')
139
+
140
+ output = expect_kaldi_matrix(instr)
141
+ if output is None:
142
+ raise Exception('UniDeepFsmn format error for parsing matrix')
143
+ instr, mat = output
144
+ mat1 = np.fliplr(mat.T).copy()
145
+ self.conv1 = nn.Conv2d(
146
+ self.output_dim,
147
+ self.output_dim, [self.lorder, 1], [1, 1],
148
+ groups=self.output_dim,
149
+ bias=False)
150
+ mat_th = th.from_numpy(mat1).type(th.FloatTensor)
151
+ mat_th = mat_th.unsqueeze(1)
152
+ mat_th = mat_th.unsqueeze(3)
153
+ self.conv1.weight = th.nn.Parameter(mat_th)
154
+
155
+ output = expect_kaldi_matrix(instr)
156
+ if output is None:
157
+ raise Exception('UniDeepFsmn format error for parsing matrix')
158
+ instr, mat = output
159
+
160
+ self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
161
+ self.linear = nn.Linear(self.input_dim, self.hidden_size)
162
+
163
+ self.project.weight = th.nn.Parameter(
164
+ th.from_numpy(mat).type(th.FloatTensor))
165
+
166
+ output = expect_kaldi_matrix(instr)
167
+ if output is None:
168
+ raise Exception('UniDeepFsmn format error for parsing matrix')
169
+ instr, mat = output
170
+ self.linear.weight = th.nn.Parameter(
171
+ th.from_numpy(mat).type(th.FloatTensor))
172
+
173
+ output = expect_kaldi_matrix(instr)
174
+ if output is None:
175
+ raise Exception('UniDeepFsmn format error for parsing matrix')
176
+ instr, mat = output
177
+ self.linear.bias = th.nn.Parameter(
178
+ th.from_numpy(mat).type(th.FloatTensor))
179
+
180
+ return instr
runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import abc
4
+ import re
5
+
6
+ import numpy as np
7
+ import torch.nn as nn
8
+
9
+
10
+ def expect_token_number(instr, token):
11
+ first_token = re.match(r'^\s*' + token, instr)
12
+ if first_token is None:
13
+ return None
14
+ instr = instr[first_token.end():]
15
+ lr = re.match(r'^\s*(-?\d+\.?\d*e?-?\d*?)', instr)
16
+ if lr is None:
17
+ return None
18
+ return instr[lr.end():], lr.groups()[0]
19
+
20
+
21
+ def expect_kaldi_matrix(instr):
22
+ pos2 = instr.find('[', 0)
23
+ pos3 = instr.find(']', pos2)
24
+ mat = []
25
+ for stt in instr[pos2 + 1:pos3].split('\n'):
26
+ tmp_mat = np.fromstring(stt, dtype=np.float32, sep=' ')
27
+ if tmp_mat.size > 0:
28
+ mat.append(tmp_mat)
29
+ return instr[pos3 + 1:], np.array(mat)
30
+
31
+
32
+ def to_kaldi_matrix(np_mat):
33
+ """
34
+ function that transform as str numpy mat to standard kaldi str matrix
35
+ :param np_mat: numpy mat
36
+ :return: str
37
+ """
38
+ np.set_printoptions(threshold=np.inf, linewidth=np.nan, suppress=True)
39
+ out_str = str(np_mat)
40
+ out_str = out_str.replace('[', '')
41
+ out_str = out_str.replace(']', '')
42
+ return '[ %s ]\n' % out_str
43
+
44
+
45
+ class LayerBase(nn.Module, metaclass=abc.ABCMeta):
46
+
47
+ def __init__(self):
48
+ super(LayerBase, self).__init__()
49
+
50
+ @abc.abstractmethod
51
+ def to_kaldi_nnet(self):
52
+ pass
runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py ADDED
@@ -0,0 +1,484 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import numpy as np
4
+ import torch as th
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+ from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number,
9
+ to_kaldi_matrix)
10
+
11
+
12
+ class SepConv(nn.Module):
13
+
14
+ def __init__(self,
15
+ in_channels,
16
+ filters,
17
+ out_channels,
18
+ kernel_size=(5, 2),
19
+ dilation=(1, 1)):
20
+ """ :param kernel_size (time, frequency)
21
+
22
+ """
23
+ super(SepConv, self).__init__()
24
+ # depthwise + pointwise
25
+ self.dconv = nn.Conv2d(
26
+ in_channels,
27
+ in_channels * filters,
28
+ kernel_size,
29
+ dilation=dilation,
30
+ groups=in_channels)
31
+ self.pconv = nn.Conv2d(
32
+ in_channels * filters, out_channels, kernel_size=1)
33
+ self.padding = dilation[0] * (kernel_size[0] - 1)
34
+
35
+ def forward(self, input):
36
+ ''' input: [B, C, T, F]
37
+ '''
38
+ x = F.pad(input, [0, 0, self.padding, 0])
39
+ x = self.dconv(x)
40
+ x = self.pconv(x)
41
+ return x
42
+
43
+
44
+ class Conv2d(nn.Module):
45
+
46
+ def __init__(self,
47
+ input_dim,
48
+ output_dim,
49
+ lorder=20,
50
+ rorder=0,
51
+ groups=1,
52
+ bias=False,
53
+ skip_connect=True):
54
+ super(Conv2d, self).__init__()
55
+ self.lorder = lorder
56
+ self.conv = nn.Conv2d(
57
+ input_dim, output_dim, [lorder, 1], groups=groups, bias=bias)
58
+ self.rorder = rorder
59
+ if self.rorder:
60
+ self.conv2 = nn.Conv2d(
61
+ input_dim, output_dim, [rorder, 1], groups=groups, bias=bias)
62
+ self.skip_connect = skip_connect
63
+
64
+ def forward(self, input):
65
+ # [B, 1, T, F]
66
+ x = th.unsqueeze(input, 1)
67
+ # [B, F, T, 1]
68
+ x_per = x.permute(0, 3, 2, 1)
69
+ y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
70
+ out = self.conv(y)
71
+ if self.rorder:
72
+ yr = F.pad(x_per, [0, 0, 0, self.rorder])
73
+ yr = yr[:, :, 1:, :]
74
+ out += self.conv2(yr)
75
+ out = out.permute(0, 3, 2, 1).squeeze(1)
76
+ if self.skip_connect:
77
+ out = out + input
78
+ return out
79
+
80
+
81
+ class SelfAttLayer(nn.Module):
82
+
83
+ def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
84
+ super(SelfAttLayer, self).__init__()
85
+
86
+ self.input_dim = input_dim
87
+ self.output_dim = output_dim
88
+
89
+ if lorder is None:
90
+ return
91
+
92
+ self.lorder = lorder
93
+ self.hidden_size = hidden_size
94
+
95
+ self.linear = nn.Linear(input_dim, hidden_size)
96
+
97
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
98
+
99
+ self.att = nn.Linear(input_dim, lorder, bias=False)
100
+
101
+ def forward(self, input):
102
+
103
+ f1 = F.relu(self.linear(input))
104
+
105
+ p1 = self.project(f1)
106
+
107
+ x = th.unsqueeze(p1, 1)
108
+
109
+ x_per = x.permute(0, 3, 2, 1)
110
+
111
+ y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
112
+
113
+ # z [B, F, T, lorder]
114
+ z = x_per
115
+ for i in range(1, self.lorder):
116
+ z = th.cat([z, y[:, :, self.lorder - 1 - i:-i, :]], axis=-1)
117
+
118
+ # [B, T, lorder]
119
+ att = F.softmax(self.att(input), dim=-1)
120
+ att = th.unsqueeze(att, 1)
121
+ z = th.sum(z * att, axis=-1)
122
+
123
+ out1 = z.permute(0, 2, 1)
124
+
125
+ return input + out1
126
+
127
+
128
+ class TFFsmn(nn.Module):
129
+
130
+ def __init__(self,
131
+ input_dim,
132
+ output_dim,
133
+ lorder=None,
134
+ hidden_size=None,
135
+ dilation=1,
136
+ layer_norm=False,
137
+ dropout=0,
138
+ skip_connect=True):
139
+ super(TFFsmn, self).__init__()
140
+
141
+ self.skip_connect = skip_connect
142
+
143
+ self.linear = nn.Linear(input_dim, hidden_size)
144
+ self.norm = nn.Identity()
145
+ if layer_norm:
146
+ self.norm = nn.LayerNorm(input_dim)
147
+ self.act = nn.ReLU()
148
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
149
+
150
+ self.conv1 = nn.Conv2d(
151
+ output_dim,
152
+ output_dim, [lorder, 1],
153
+ dilation=[dilation, 1],
154
+ groups=output_dim,
155
+ bias=False)
156
+ self.padding_left = dilation * (lorder - 1)
157
+ dorder = 5
158
+ self.conv2 = nn.Conv2d(1, 1, [dorder, 1], bias=False)
159
+ self.padding_freq = dorder - 1
160
+
161
+ def forward(self, input):
162
+ return self.compute1(input)
163
+
164
+ def compute1(self, input):
165
+ ''' linear-dconv-relu(norm)-linear-dconv
166
+ '''
167
+ x = self.linear(input)
168
+ # [B, 1, F, T]
169
+ x = th.unsqueeze(x, 1).permute(0, 1, 3, 2)
170
+ z = F.pad(x, [0, 0, self.padding_freq, 0])
171
+ z = self.conv2(z) + x
172
+ x = z.permute(0, 3, 2, 1).squeeze(-1)
173
+ x = self.act(x)
174
+ x = self.norm(x)
175
+ x = self.project(x)
176
+ x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
177
+ # [B, F, T+lorder-1, 1]
178
+ y = F.pad(x, [0, 0, self.padding_left, 0])
179
+ out = self.conv1(y)
180
+ if self.skip_connect:
181
+ out = out + x
182
+ out = out.permute(0, 3, 2, 1).squeeze()
183
+
184
+ return input + out
185
+
186
+
187
+ class CNNFsmn(nn.Module):
188
+ ''' use cnn to reduce parameters
189
+ '''
190
+
191
+ def __init__(self,
192
+ input_dim,
193
+ output_dim,
194
+ lorder=None,
195
+ hidden_size=None,
196
+ dilation=1,
197
+ layer_norm=False,
198
+ dropout=0,
199
+ skip_connect=True):
200
+ super(CNNFsmn, self).__init__()
201
+
202
+ self.input_dim = input_dim
203
+ self.output_dim = output_dim
204
+ self.skip_connect = skip_connect
205
+
206
+ if lorder is None:
207
+ return
208
+
209
+ self.lorder = lorder
210
+ self.hidden_size = hidden_size
211
+
212
+ self.linear = nn.Linear(input_dim, hidden_size)
213
+ self.act = nn.ReLU()
214
+ kernel_size = (3, 8)
215
+ stride = (1, 4)
216
+ self.conv = nn.Sequential(
217
+ nn.ConstantPad2d((stride[1], 0, kernel_size[0] - 1, 0), 0),
218
+ nn.Conv2d(1, stride[1], kernel_size=kernel_size, stride=stride))
219
+
220
+ self.dconv = nn.Conv2d(
221
+ output_dim,
222
+ output_dim, [lorder, 1],
223
+ dilation=[dilation, 1],
224
+ groups=output_dim,
225
+ bias=False)
226
+ self.padding_left = dilation * (lorder - 1)
227
+
228
+ def forward(self, input):
229
+ return self.compute2(input)
230
+
231
+ def compute1(self, input):
232
+ ''' linear-relu(norm)-conv2d-relu?-dconv
233
+ '''
234
+ # [B, T, F]
235
+ x = self.linear(input)
236
+ x = self.act(x)
237
+ x = th.unsqueeze(x, 1)
238
+ x = self.conv(x)
239
+ # [B, C, T, F] -> [B, 1, T, F]
240
+ b, c, t, f = x.shape
241
+ x = x.view([b, 1, t, -1])
242
+ x = x.permute(0, 3, 2, 1)
243
+ # [B, F, T+lorder-1, 1]
244
+ y = F.pad(x, [0, 0, self.padding_left, 0])
245
+ out = self.dconv(y)
246
+ if self.skip_connect:
247
+ out = out + x
248
+ out = out.permute(0, 3, 2, 1).squeeze()
249
+ return input + out
250
+
251
+ def compute2(self, input):
252
+ ''' conv2d-relu-linear-relu?-dconv
253
+ '''
254
+ x = th.unsqueeze(input, 1)
255
+ x = self.conv(x)
256
+ x = self.act(x)
257
+ # [B, C, T, F] -> [B, T, F]
258
+ b, c, t, f = x.shape
259
+ x = x.view([b, t, -1])
260
+ x = self.linear(x)
261
+ x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
262
+ y = F.pad(x, [0, 0, self.padding_left, 0])
263
+ out = self.dconv(y)
264
+ if self.skip_connect:
265
+ out = out + x
266
+ out = out.permute(0, 3, 2, 1).squeeze()
267
+ return input + out
268
+
269
+
270
+ class UniDeepFsmn(LayerBase):
271
+
272
+ def __init__(self,
273
+ input_dim,
274
+ output_dim,
275
+ lorder=None,
276
+ hidden_size=None,
277
+ dilation=1,
278
+ layer_norm=False,
279
+ dropout=0,
280
+ skip_connect=True):
281
+ super(UniDeepFsmn, self).__init__()
282
+
283
+ self.input_dim = input_dim
284
+ self.output_dim = output_dim
285
+ self.skip_connect = skip_connect
286
+
287
+ if lorder is None:
288
+ return
289
+
290
+ self.lorder = lorder
291
+ self.hidden_size = hidden_size
292
+
293
+ self.linear = nn.Linear(input_dim, hidden_size)
294
+ self.norm = nn.Identity()
295
+ if layer_norm:
296
+ self.norm = nn.LayerNorm(input_dim)
297
+ self.act = nn.ReLU()
298
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
299
+
300
+ self.conv1 = nn.Conv2d(
301
+ output_dim,
302
+ output_dim, [lorder, 1],
303
+ dilation=[dilation, 1],
304
+ groups=output_dim,
305
+ bias=False)
306
+ self.padding_left = dilation * (lorder - 1)
307
+
308
+ def forward(self, input):
309
+ return self.compute1(input)
310
+
311
+ def compute1(self, input):
312
+ ''' linear-relu(norm)-linear-dconv
313
+ '''
314
+ # [B, T, F]
315
+ x = self.linear(input)
316
+ x = self.act(x)
317
+ x = self.norm(x)
318
+ x = self.project(x)
319
+ x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
320
+ # [B, F, T+lorder-1, 1]
321
+ y = F.pad(x, [0, 0, self.padding_left, 0])
322
+ out = self.conv1(y)
323
+ if self.skip_connect:
324
+ out = out + x
325
+ out = out.permute(0, 3, 2, 1).squeeze()
326
+
327
+ return input + out
328
+
329
+ def compute2(self, input):
330
+ ''' linear-dconv-linear-relu(norm)
331
+ '''
332
+ x = self.project(input)
333
+ x = th.unsqueeze(x, 1).permute(0, 3, 2, 1)
334
+ y = F.pad(x, [0, 0, self.padding_left, 0])
335
+ out = self.conv1(y)
336
+ if self.skip_connect:
337
+ out = out + x
338
+ out = out.permute(0, 3, 2, 1).squeeze()
339
+ x = self.linear(out)
340
+ x = self.act(x)
341
+ x = self.norm(x)
342
+
343
+ return input + x
344
+
345
+ def compute3(self, input):
346
+ ''' dconv-linear-relu(norm)-linear
347
+ '''
348
+ x = th.unsqueeze(input, 1).permute(0, 3, 2, 1)
349
+ y = F.pad(x, [0, 0, self.padding_left, 0])
350
+ out = self.conv1(y)
351
+ if self.skip_connect:
352
+ out = out + x
353
+ out = out.permute(0, 3, 2, 1).squeeze()
354
+ x = self.linear(out)
355
+ x = self.act(x)
356
+ x = self.norm(x)
357
+ x = self.project(x)
358
+
359
+ return input + x
360
+
361
+ def to_kaldi_nnet(self):
362
+ re_str = ''
363
+ re_str += '<UniDeepFsmn> %d %d\n' \
364
+ % (self.output_dim, self.input_dim)
365
+ re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n' \
366
+ % (1, self.hidden_size, self.lorder, 1)
367
+ lfiters = self.state_dict()['conv1.weight']
368
+ x = np.flipud(lfiters.squeeze().numpy().T)
369
+ re_str += to_kaldi_matrix(x)
370
+ proj_weights = self.state_dict()['project.weight']
371
+ x = proj_weights.squeeze().numpy()
372
+ re_str += to_kaldi_matrix(x)
373
+ linear_weights = self.state_dict()['linear.weight']
374
+ x = linear_weights.squeeze().numpy()
375
+ re_str += to_kaldi_matrix(x)
376
+ linear_bias = self.state_dict()['linear.bias']
377
+ x = linear_bias.squeeze().numpy()
378
+ re_str += to_kaldi_matrix(x)
379
+ return re_str
380
+
381
+ def to_raw_nnet(self, fid):
382
+ lfiters = self.state_dict()['conv1.weight']
383
+ x = np.flipud(lfiters.squeeze().numpy().T)
384
+ x.tofile(fid)
385
+
386
+ proj_weights = self.state_dict()['project.weight']
387
+ x = proj_weights.squeeze().numpy()
388
+ x.tofile(fid)
389
+
390
+ linear_weights = self.state_dict()['linear.weight']
391
+ x = linear_weights.squeeze().numpy()
392
+ x.tofile(fid)
393
+
394
+ linear_bias = self.state_dict()['linear.bias']
395
+ x = linear_bias.squeeze().numpy()
396
+ x.tofile(fid)
397
+
398
+ def load_kaldi_nnet(self, instr):
399
+ output = expect_token_number(
400
+ instr,
401
+ '<LearnRateCoef>',
402
+ )
403
+ if output is None:
404
+ raise Exception('UniDeepFsmn format error for <LearnRateCoef>')
405
+ instr, lr = output
406
+
407
+ output = expect_token_number(
408
+ instr,
409
+ '<HidSize>',
410
+ )
411
+ if output is None:
412
+ raise Exception('UniDeepFsmn format error for <HidSize>')
413
+ instr, hiddensize = output
414
+ self.hidden_size = int(hiddensize)
415
+
416
+ output = expect_token_number(
417
+ instr,
418
+ '<LOrder>',
419
+ )
420
+ if output is None:
421
+ raise Exception('UniDeepFsmn format error for <LOrder>')
422
+ instr, lorder = output
423
+ self.lorder = int(lorder)
424
+
425
+ output = expect_token_number(
426
+ instr,
427
+ '<LStride>',
428
+ )
429
+ if output is None:
430
+ raise Exception('UniDeepFsmn format error for <LStride>')
431
+ instr, lstride = output
432
+ self.lstride = lstride
433
+
434
+ output = expect_token_number(
435
+ instr,
436
+ '<MaxNorm>',
437
+ )
438
+ if output is None:
439
+ raise Exception('UniDeepFsmn format error for <MaxNorm>')
440
+
441
+ output = expect_kaldi_matrix(instr)
442
+ if output is None:
443
+ raise Exception('UniDeepFsmn format error for parsing matrix')
444
+ instr, mat = output
445
+ mat1 = np.fliplr(mat.T).copy()
446
+
447
+ self.conv1 = nn.Conv2d(
448
+ self.output_dim,
449
+ self.output_dim, [self.lorder, 1], [1, 1],
450
+ groups=self.output_dim,
451
+ bias=False)
452
+
453
+ mat_th = th.from_numpy(mat1).type(th.FloatTensor)
454
+ mat_th = mat_th.unsqueeze(1)
455
+ mat_th = mat_th.unsqueeze(3)
456
+ self.conv1.weight = th.nn.Parameter(mat_th)
457
+
458
+ output = expect_kaldi_matrix(instr)
459
+ if output is None:
460
+ raise Exception('UniDeepFsmn format error for parsing matrix')
461
+ instr, mat = output
462
+
463
+ self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
464
+ self.linear = nn.Linear(self.input_dim, self.hidden_size)
465
+
466
+ self.project.weight = th.nn.Parameter(
467
+ th.from_numpy(mat).type(th.FloatTensor))
468
+
469
+ output = expect_kaldi_matrix(instr)
470
+ if output is None:
471
+ raise Exception('UniDeepFsmn format error for parsing matrix')
472
+ instr, mat = output
473
+ self.linear.weight = th.nn.Parameter(
474
+ th.from_numpy(mat).type(th.FloatTensor))
475
+
476
+ output = expect_kaldi_matrix(instr)
477
+ if output is None:
478
+ raise Exception('UniDeepFsmn format error for parsing matrix')
479
+ instr, mat = output
480
+ mat = np.squeeze(mat)
481
+ self.linear.bias = th.nn.Parameter(
482
+ th.from_numpy(mat).type(th.FloatTensor))
483
+
484
+ return instr
runtime/Lib/site_packages/models/audio/aec/network/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/aec/network/loss.py ADDED
@@ -0,0 +1,396 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch
4
+ import torch.nn.functional as F
5
+
6
+ from .modulation_loss import (GaborSTRFConv, MelScale,
7
+ ModulationDomainLossModule)
8
+
9
+ EPS = 1e-8
10
+
11
+
12
+ def compute_mask(mixed_spec, clean_spec, mask_type='psmiam', clip=1):
13
+ '''
14
+ stft: (batch, ..., 2) or complex(batch, ...)
15
+ y = x + n
16
+ '''
17
+ if torch.is_complex(mixed_spec):
18
+ yr, yi = mixed_spec.real, mixed_spec.imag
19
+ else:
20
+ yr, yi = mixed_spec[..., 0], mixed_spec[..., 1]
21
+ if torch.is_complex(clean_spec):
22
+ xr, xi = clean_spec.real, clean_spec.imag
23
+ else:
24
+ xr, xi = clean_spec[..., 0], clean_spec[..., 1]
25
+
26
+ if mask_type == 'iam':
27
+ ymag = torch.sqrt(yr**2 + yi**2)
28
+ xmag = torch.sqrt(xr**2 + xi**2)
29
+ iam = xmag / (ymag + EPS)
30
+ return torch.clamp(iam, 0, 1)
31
+
32
+ elif mask_type == 'psm':
33
+ ypow = yr**2 + yi**2
34
+ psm = (xr * yr + xi * yi) / (ypow + EPS)
35
+ return torch.clamp(psm, 0, 1)
36
+
37
+ elif mask_type == 'psmiam':
38
+ ypow = yr**2 + yi**2
39
+ psm = (xr * yr + xi * yi) / (ypow + EPS)
40
+ ymag = torch.sqrt(yr**2 + yi**2)
41
+ xmag = torch.sqrt(xr**2 + xi**2)
42
+ iam = xmag / (ymag + EPS)
43
+ psmiam = psm * iam
44
+ return torch.clamp(psmiam, 0, 1)
45
+
46
+ elif mask_type == 'crm':
47
+ ypow = yr**2 + yi**2
48
+ mr = (xr * yr + xi * yi) / (ypow + EPS)
49
+ mi = (xi * yr - xr * yi) / (ypow + EPS)
50
+ mr = torch.clamp(mr, -clip, clip)
51
+ mi = torch.clamp(mi, -clip, clip)
52
+ return mr, mi
53
+
54
+
55
+ def energy_vad(spec,
56
+ thdhigh=320 * 600 * 600 * 2,
57
+ thdlow=320 * 300 * 300 * 2,
58
+ int16=True):
59
+ '''
60
+ energy based vad should be accurate enough
61
+ spec: (batch, bins, frames, 2)
62
+ returns (batch, frames)
63
+ '''
64
+ energy = torch.sum(spec[..., 0]**2 + spec[..., 1]**2, dim=1)
65
+ vad = energy > thdhigh
66
+ idx = torch.logical_and(vad == 0, energy > thdlow)
67
+ vad[idx] = 0.5
68
+ return vad
69
+
70
+
71
+ def modulation_loss_init(n_fft):
72
+ gabor_strf_parameters = torch.load(
73
+ './network/gabor_strf_parameters.pt')['state_dict']
74
+ gabor_modulation_kernels = GaborSTRFConv(supn=30, supk=30, nkern=60)
75
+ gabor_modulation_kernels.load_state_dict(gabor_strf_parameters)
76
+
77
+ modulation_loss_module = ModulationDomainLossModule(
78
+ gabor_modulation_kernels.eval())
79
+ for param in modulation_loss_module.parameters():
80
+ param.requires_grad = False
81
+
82
+ stft2mel = MelScale(
83
+ n_mels=80, sample_rate=16000, n_stft=n_fft // 2 + 1).cuda()
84
+
85
+ return modulation_loss_module, stft2mel
86
+
87
+
88
+ def mask_loss_function(
89
+ loss_func='psm_loss',
90
+ loss_type='mse', # ['mse', 'mae', 'comb']
91
+ mask_type='psmiam',
92
+ use_mod_loss=False,
93
+ use_wav2vec_loss=False,
94
+ n_fft=640,
95
+ hop_length=320,
96
+ EPS=1e-8,
97
+ weight=None):
98
+ if weight is not None:
99
+ print(f'Use loss weight: {weight}')
100
+ winlen = n_fft
101
+ window = torch.hamming_window(winlen, periodic=False)
102
+
103
+ def stft(x, return_complex=False):
104
+ # returns [batch, bins, frames, 2]
105
+ return torch.stft(
106
+ x,
107
+ n_fft,
108
+ hop_length,
109
+ winlen,
110
+ window=window.to(x.device),
111
+ center=False,
112
+ return_complex=return_complex)
113
+
114
+ def istft(x, slen):
115
+ return torch.istft(
116
+ x,
117
+ n_fft,
118
+ hop_length,
119
+ winlen,
120
+ window=window.to(x.device),
121
+ center=False,
122
+ length=slen)
123
+
124
+ def mask_loss(targets, masks, nframes):
125
+ ''' [Batch, Time, Frequency]
126
+ '''
127
+ with torch.no_grad():
128
+ mask_for_loss = torch.ones_like(targets)
129
+ for idx, num in enumerate(nframes):
130
+ mask_for_loss[idx, num:, :] = 0
131
+ masks = masks * mask_for_loss
132
+ targets = targets * mask_for_loss
133
+
134
+ if weight is None:
135
+ alpha = 1
136
+ else: # for aec ST
137
+ alpha = weight - targets
138
+
139
+ if loss_type == 'mse':
140
+ loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2))
141
+ elif loss_type == 'mae':
142
+ loss = torch.sum(alpha * torch.abs(targets - masks))
143
+ else: # mse(mask), mae(mask) approx 1:2
144
+ loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2)
145
+ + 0.1 * alpha * torch.abs(targets - masks))
146
+ loss /= torch.sum(nframes)
147
+ return loss
148
+
149
+ def spectrum_loss(targets, spec, nframes):
150
+ ''' [Batch, Time, Frequency, 2]
151
+ '''
152
+ with torch.no_grad():
153
+ mask_for_loss = torch.ones_like(targets[..., 0])
154
+ for idx, num in enumerate(nframes):
155
+ mask_for_loss[idx, num:, :] = 0
156
+ xr = spec[..., 0] * mask_for_loss
157
+ xi = spec[..., 1] * mask_for_loss
158
+ yr = targets[..., 0] * mask_for_loss
159
+ yi = targets[..., 1] * mask_for_loss
160
+ xmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2) * mask_for_loss
161
+ ymag = torch.sqrt(targets[..., 0]**2
162
+ + targets[..., 1]**2) * mask_for_loss
163
+
164
+ loss1 = torch.sum(torch.pow(xr - yr, 2) + torch.pow(xi - yi, 2))
165
+ loss2 = torch.sum(torch.pow(xmag - ymag, 2))
166
+
167
+ loss = (loss1 + loss2) / torch.sum(nframes)
168
+ return loss
169
+
170
+ def sa_loss_dlen(mixed, clean, masks, nframes):
171
+ yspec = stft(mixed).permute([0, 2, 1, 3]) / 32768
172
+ xspec = stft(clean).permute([0, 2, 1, 3]) / 32768
173
+ with torch.no_grad():
174
+ mask_for_loss = torch.ones_like(xspec[..., 0])
175
+ for idx, num in enumerate(nframes):
176
+ mask_for_loss[idx, num:, :] = 0
177
+ emag = ((yspec[..., 0]**2 + yspec[..., 1]**2)**0.15) * (masks**0.3)
178
+ xmag = (xspec[..., 0]**2 + xspec[..., 1]**2)**0.15
179
+ emag = emag * mask_for_loss
180
+ xmag = xmag * mask_for_loss
181
+
182
+ loss = torch.sum(torch.pow(emag - xmag, 2)) / torch.sum(nframes)
183
+ return loss
184
+
185
+ def psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask=None):
186
+ mixed_spec = stft(mixed)
187
+ clean_spec = stft(clean)
188
+ targets = compute_mask(mixed_spec, clean_spec, mask_type)
189
+ # [B, T, F]
190
+ targets = targets.permute(0, 2, 1)
191
+
192
+ loss = mask_loss(targets, masks, nframes)
193
+
194
+ if subtask is not None:
195
+ vadtargets = energy_vad(clean_spec)
196
+ with torch.no_grad():
197
+ mask_for_loss = torch.ones_like(targets[:, :, 0])
198
+ for idx, num in enumerate(nframes):
199
+ mask_for_loss[idx, num:] = 0
200
+ subtask = subtask[:, :, 0] * mask_for_loss
201
+ vadtargets = vadtargets * mask_for_loss
202
+
203
+ loss_vad = F.binary_cross_entropy(subtask, vadtargets)
204
+ return loss + loss_vad
205
+ return loss
206
+
207
+ def modulation_loss(mixed, clean, masks, nframes, subtask=None):
208
+ mixed_spec = stft(mixed, True)
209
+ clean_spec = stft(clean, True)
210
+ enhanced_mag = torch.abs(mixed_spec)
211
+ clean_mag = torch.abs(clean_spec)
212
+ with torch.no_grad():
213
+ mask_for_loss = torch.ones_like(clean_mag)
214
+ for idx, num in enumerate(nframes):
215
+ mask_for_loss[idx, :, num:] = 0
216
+ clean_mag = clean_mag * mask_for_loss
217
+ enhanced_mag = enhanced_mag * mask_for_loss * masks.permute([0, 2, 1])
218
+
219
+ # Covert to log-mel representation
220
+ # (B,T,#mel_channels)
221
+ clean_log_mel = torch.log(
222
+ torch.transpose(stft2mel(clean_mag**2), 2, 1) + 1e-8)
223
+ enhanced_log_mel = torch.log(
224
+ torch.transpose(stft2mel(enhanced_mag**2), 2, 1) + 1e-8)
225
+
226
+ alpha = compute_mask(mixed_spec, clean_spec, mask_type)
227
+ alpha = alpha.permute(0, 2, 1)
228
+ loss = 0.05 * modulation_loss_module(enhanced_log_mel, clean_log_mel,
229
+ alpha)
230
+ loss2 = psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask)
231
+ # print(loss.item(), loss2.item()) #approx 1:4
232
+ loss = loss + loss2
233
+ return loss
234
+
235
+ def wav2vec_loss(mixed, clean, masks, nframes, subtask=None):
236
+ mixed /= 32768
237
+ clean /= 32768
238
+ mixed_spec = stft(mixed)
239
+ with torch.no_grad():
240
+ mask_for_loss = torch.ones_like(masks)
241
+ for idx, num in enumerate(nframes):
242
+ mask_for_loss[idx, num:, :] = 0
243
+ masks_est = masks * mask_for_loss
244
+
245
+ estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
246
+ est_clean = istft(estimate, clean.shape[1])
247
+ loss = wav2vec_loss_module(est_clean, clean)
248
+ return loss
249
+
250
+ def sisdr_loss_dlen(mixed,
251
+ clean,
252
+ masks,
253
+ nframes,
254
+ subtask=None,
255
+ zero_mean=True):
256
+ mixed_spec = stft(mixed)
257
+ with torch.no_grad():
258
+ mask_for_loss = torch.ones_like(masks)
259
+ for idx, num in enumerate(nframes):
260
+ mask_for_loss[idx, num:, :] = 0
261
+ masks_est = masks * mask_for_loss
262
+
263
+ estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
264
+ est_clean = istft(estimate, clean.shape[1])
265
+ flen = min(clean.shape[1], est_clean.shape[1])
266
+ clean = clean[:, :flen]
267
+ est_clean = est_clean[:, :flen]
268
+
269
+ # follow asteroid/losses/sdr.py
270
+ if zero_mean:
271
+ clean = clean - torch.mean(clean, dim=1, keepdim=True)
272
+ est_clean = est_clean - torch.mean(est_clean, dim=1, keepdim=True)
273
+
274
+ dot = torch.sum(est_clean * clean, dim=1, keepdim=True)
275
+ s_clean_energy = torch.sum(clean**2, dim=1, keepdim=True) + EPS
276
+ scaled_clean = dot * clean / s_clean_energy
277
+ e_noise = est_clean - scaled_clean
278
+
279
+ # [batch]
280
+ sisdr = torch.sum(
281
+ scaled_clean**2, dim=1) / (
282
+ torch.sum(e_noise**2, dim=1) + EPS)
283
+ sisdr = -10 * torch.log10(sisdr + EPS)
284
+ loss = sisdr.mean()
285
+ return loss
286
+
287
+ def sisdr_freq_loss_dlen(mixed, clean, masks, nframes, subtask=None):
288
+ mixed_spec = stft(mixed)
289
+ clean_spec = stft(clean)
290
+ with torch.no_grad():
291
+ mask_for_loss = torch.ones_like(masks)
292
+ for idx, num in enumerate(nframes):
293
+ mask_for_loss[idx, num:, :] = 0
294
+ masks_est = masks * mask_for_loss
295
+
296
+ estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3)
297
+
298
+ dot_real = estimate[..., 0] * clean_spec[..., 0] + \
299
+ estimate[..., 1] * clean_spec[..., 1]
300
+ dot_imag = estimate[..., 0] * clean_spec[..., 1] - \
301
+ estimate[..., 1] * clean_spec[..., 0]
302
+ dot = torch.cat([dot_real.unsqueeze(3), dot_imag.unsqueeze(3)], dim=-1)
303
+ s_clean_energy = clean_spec[..., 0] ** 2 + \
304
+ clean_spec[..., 1] ** 2 + EPS
305
+ scaled_clean = dot * clean_spec / s_clean_energy.unsqueeze(3)
306
+ e_noise = estimate - scaled_clean
307
+
308
+ # [batch]
309
+ scaled_clean_energy = torch.sum(
310
+ scaled_clean[..., 0]**2 + scaled_clean[..., 1]**2, dim=1)
311
+ e_noise_energy = torch.sum(
312
+ e_noise[..., 0]**2 + e_noise[..., 1]**2, dim=1)
313
+ sisdr = torch.sum(
314
+ scaled_clean_energy, dim=1) / (
315
+ torch.sum(e_noise_energy, dim=1) + EPS)
316
+ sisdr = -10 * torch.log10(sisdr + EPS)
317
+ loss = sisdr.mean()
318
+ return loss
319
+
320
+ def crm_loss_dlen(mixed, clean, masks, nframes, subtask=None):
321
+ mixed_spec = stft(mixed).permute([0, 2, 1, 3])
322
+ clean_spec = stft(clean).permute([0, 2, 1, 3])
323
+ mixed_spec = mixed_spec / 32768
324
+ clean_spec = clean_spec / 32768
325
+ tgt_mr, tgt_mi = compute_mask(mixed_spec, clean_spec, mask_type='crm')
326
+
327
+ D = int(masks.shape[2] / 2)
328
+ with torch.no_grad():
329
+ mask_for_loss = torch.ones_like(clean_spec[..., 0])
330
+ for idx, num in enumerate(nframes):
331
+ mask_for_loss[idx, num:, :] = 0
332
+ mr = masks[..., :D] * mask_for_loss
333
+ mi = masks[..., D:] * mask_for_loss
334
+ tgt_mr = tgt_mr * mask_for_loss
335
+ tgt_mi = tgt_mi * mask_for_loss
336
+
337
+ if weight is None:
338
+ alpha = 1
339
+ else:
340
+ alpha = weight - tgt_mr
341
+ # signal approximation
342
+ yr = mixed_spec[..., 0]
343
+ yi = mixed_spec[..., 1]
344
+ loss1 = torch.sum(alpha * torch.pow((mr * yr - mi * yi) - clean_spec[..., 0], 2)) \
345
+ + torch.sum(alpha * torch.pow((mr * yi + mi * yr) - clean_spec[..., 1], 2))
346
+ # mask approximation
347
+ loss2 = torch.sum(alpha * torch.pow(mr - tgt_mr, 2)) \
348
+ + torch.sum(alpha * torch.pow(mi - tgt_mi, 2))
349
+ loss = 0.5 * (loss1 + loss2) / torch.sum(nframes)
350
+ return loss
351
+
352
+ def crm_miso_loss_dlen(mixed, clean, masks, nframes):
353
+ return crm_loss_dlen(mixed[..., 0], clean[..., 0], masks, nframes)
354
+
355
+ def mimo_loss_dlen(mixed, clean, masks, nframes):
356
+ chs = mixed.shape[-1]
357
+ D = masks.shape[2] // chs
358
+ loss = psm_vad_loss_dlen(mixed[..., 0], clean[..., 0], masks[..., :D],
359
+ nframes)
360
+ for ch in range(1, chs):
361
+ loss1 = psm_vad_loss_dlen(mixed[..., ch], clean[..., ch],
362
+ masks[..., ch * D:ch * D + D], nframes)
363
+ loss = loss + loss1
364
+ return loss / chs
365
+
366
+ def spec_loss_dlen(mixed, clean, spec, nframes):
367
+ clean_spec = stft(clean).permute([0, 2, 1, 3])
368
+ clean_spec = clean_spec / 32768
369
+
370
+ D = spec.shape[2] // 2
371
+ spec_est = torch.cat([spec[..., :D, None], spec[..., D:, None]],
372
+ dim=-1)
373
+ loss = spectrum_loss(clean_spec, spec_est, nframes)
374
+ return loss
375
+
376
+ if loss_func == 'psm_vad_loss_dlen':
377
+ return psm_vad_loss_dlen
378
+ elif loss_func == 'sisdr_loss_dlen':
379
+ return sisdr_loss_dlen
380
+ elif loss_func == 'sisdr_freq_loss_dlen':
381
+ return sisdr_freq_loss_dlen
382
+ elif loss_func == 'crm_loss_dlen':
383
+ return crm_loss_dlen
384
+ elif loss_func == 'modulation_loss':
385
+ return modulation_loss
386
+ elif loss_func == 'wav2vec_loss':
387
+ return wav2vec_loss
388
+ elif loss_func == 'mimo_loss_dlen':
389
+ return mimo_loss_dlen
390
+ elif loss_func == 'spec_loss_dlen':
391
+ return spec_loss_dlen
392
+ elif loss_func == 'sa_loss_dlen':
393
+ return sa_loss_dlen
394
+ else:
395
+ print('error loss func')
396
+ return None
runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import math
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+ from torchaudio.transforms import MelScale
9
+
10
+
11
+ class ModulationDomainLossModule(torch.nn.Module):
12
+ """Modulation-domain loss function developed in [1] for supervised speech enhancement
13
+
14
+ In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram
15
+ as the input spectrogram representation.
16
+ Specific parameter details are in the paper and in the example below
17
+
18
+ Parameters
19
+ ----------
20
+ modulation_kernels: nn.Module
21
+ Differentiable module that transforms a spectrogram representation to the modulation domain
22
+
23
+ modulation_domain = modulation_kernels(input_tf_representation)
24
+ Input Spectrogram representation (B, T, F) ---> |(M) modulation_kernels|--->Modulation Domain(B, M, T', F')
25
+
26
+ norm: boolean
27
+ Normalizes the modulation domain representation to be 0 mean across time
28
+
29
+ [1] T. Vuong, Y. Xia, and R. M. Stern, “A modulation-domain lossfor neural-network-based real-time
30
+ speech enhancement”
31
+ Accepted ICASSP 2021, https://arxiv.org/abs/2102.07330
32
+
33
+
34
+ """
35
+
36
+ def __init__(self, modulation_kernels, norm=True):
37
+ super(ModulationDomainLossModule, self).__init__()
38
+
39
+ self.modulation_kernels = modulation_kernels
40
+ self.mse = nn.MSELoss(reduce=False)
41
+ self.norm = norm
42
+
43
+ def forward(self, enhanced_spect, clean_spect, weight=None):
44
+ """Calculate modulation-domain loss
45
+ Args:
46
+ enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels).
47
+ clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels).
48
+ Returns:
49
+ Tensor: Modulation-domain loss value.
50
+ """
51
+
52
+ clean_mod = self.modulation_kernels(clean_spect)
53
+ enhanced_mod = self.modulation_kernels(enhanced_spect)
54
+
55
+ if self.norm:
56
+ mean_clean_mod = torch.mean(clean_mod, dim=2)
57
+ mean_enhanced_mod = torch.mean(enhanced_mod, dim=2)
58
+
59
+ clean_mod = clean_mod - mean_clean_mod.unsqueeze(2)
60
+ enhanced_mod = enhanced_mod - mean_enhanced_mod.unsqueeze(2)
61
+
62
+ if weight is None:
63
+ alpha = 1
64
+ else: # TF-mask weight
65
+ alpha = 1 + torch.sum(weight, dim=-1, keepdim=True).unsqueeze(1)
66
+ mod_mse_loss = self.mse(enhanced_mod, clean_mod) * alpha
67
+ mod_mse_loss = torch.mean(
68
+ torch.sum(mod_mse_loss, dim=(1, 2, 3))
69
+ / torch.sum(clean_mod**2, dim=(1, 2, 3)))
70
+
71
+ return mod_mse_loss
72
+
73
+
74
+ class ModulationDomainNCCLossModule(torch.nn.Module):
75
+ """Modulation-domain loss function developed in [1] for supervised speech enhancement
76
+
77
+ # Speech Intelligibility Prediction Using Spectro-Temporal Modulation Analysis - based off of this
78
+
79
+ In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram
80
+ as the input spectrogram representation.
81
+ Specific parameter details are in the paper and in the example below
82
+
83
+ Parameters
84
+ ----------
85
+ modulation_kernels: nn.Module
86
+ Differentiable module that transforms a spectrogram representation to the modulation domain
87
+
88
+ modulation_domain = modulation_kernels(input_tf_representation)
89
+ Input Spectrogram representation(B, T, F) --- (M) modulation_kernels---> Modulation Domain(B, M, T', F')
90
+
91
+ [1]
92
+
93
+ """
94
+
95
+ def __init__(self, modulation_kernels):
96
+ super(ModulationDomainNCCLossModule, self).__init__()
97
+
98
+ self.modulation_kernels = modulation_kernels
99
+ self.mse = nn.MSELoss(reduce=False)
100
+
101
+ def forward(self, enhanced_spect, clean_spect):
102
+ """Calculate modulation-domain loss
103
+ Args:
104
+ enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels).
105
+ clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels).
106
+ Returns:
107
+ Tensor: Modulation-domain loss value.
108
+ """
109
+
110
+ clean_mod = self.modulation_kernels(clean_spect)
111
+ enhanced_mod = self.modulation_kernels(enhanced_spect)
112
+ mean_clean_mod = torch.mean(clean_mod, dim=2)
113
+ mean_enhanced_mod = torch.mean(enhanced_mod, dim=2)
114
+
115
+ normalized_clean = clean_mod - mean_clean_mod.unsqueeze(2)
116
+ normalized_enhanced = enhanced_mod - mean_enhanced_mod.unsqueeze(2)
117
+
118
+ inner_product = torch.sum(
119
+ normalized_clean * normalized_enhanced, dim=2)
120
+ normalized_denom = (torch.sum(
121
+ normalized_clean * normalized_clean, dim=2))**.5 * (torch.sum(
122
+ normalized_enhanced * normalized_enhanced, dim=2))**.5
123
+
124
+ ncc = inner_product / normalized_denom
125
+ mod_mse_loss = torch.mean((ncc - 1.0)**2)
126
+
127
+ return mod_mse_loss
128
+
129
+
130
+ class GaborSTRFConv(nn.Module):
131
+ """Gabor-STRF-based cross-correlation kernel."""
132
+
133
+ def __init__(self,
134
+ supn,
135
+ supk,
136
+ nkern,
137
+ rates=None,
138
+ scales=None,
139
+ norm_strf=True,
140
+ real_only=False):
141
+ """Instantiate a Gabor-based STRF convolution layer.
142
+ Parameters
143
+ ----------
144
+ supn: int
145
+ Time support in number of frames. Also the window length.
146
+ supk: int
147
+ Frequency support in number of channels. Also the window length.
148
+ nkern: int
149
+ Number of kernels, each with a learnable rate and scale.
150
+ rates: list of float, None
151
+ Initial values for temporal modulation.
152
+ scales: list of float, None
153
+ Initial values for spectral modulation.
154
+ norm_strf: Boolean
155
+ Normalize STRF kernels to be unit length
156
+ real_only: Boolean
157
+ If True, nkern REAL gabor-STRF kernels
158
+ If False, nkern//2 REAL and nkern//2 IMAGINARY gabor-STRF kernels
159
+ """
160
+ super(GaborSTRFConv, self).__init__()
161
+ self.numN = supn
162
+ self.numK = supk
163
+ self.numKern = nkern
164
+ self.real_only = real_only
165
+ self.norm_strf = norm_strf
166
+
167
+ if not real_only:
168
+ nkern = nkern // 2
169
+
170
+ if supk % 2 == 0: # force odd number
171
+ supk += 1
172
+ self.supk = torch.arange(supk, dtype=torch.float32)
173
+ if supn % 2 == 0: # force odd number
174
+ supn += 1
175
+ self.supn = torch.arange(supn, dtype=self.supk.dtype)
176
+ self.padding = (supn // 2, supk // 2)
177
+ # Set up learnable parameters
178
+ # for param in (rates, scales):
179
+ # assert (not param) or len(param) == nkern
180
+ if not rates:
181
+
182
+ rates = torch.rand(nkern) * math.pi / 2.0
183
+
184
+ if not scales:
185
+
186
+ scales = (torch.rand(nkern) * 2.0 - 1.0) * math.pi / 2.0
187
+
188
+ self.rates_ = nn.Parameter(torch.Tensor(rates))
189
+ self.scales_ = nn.Parameter(torch.Tensor(scales))
190
+
191
+ def strfs(self):
192
+ """Make STRFs using the current parameters."""
193
+
194
+ if self.supn.device != self.rates_.device: # for first run
195
+ self.supn = self.supn.to(self.rates_.device)
196
+ self.supk = self.supk.to(self.rates_.device)
197
+ n0, k0 = self.padding
198
+
199
+ nwind = .5 - .5 * \
200
+ torch.cos(2 * math.pi * (self.supn + 1) / (len(self.supn) + 1))
201
+ kwind = .5 - .5 * \
202
+ torch.cos(2 * math.pi * (self.supk + 1) / (len(self.supk) + 1))
203
+
204
+ new_wind = torch.matmul((nwind).unsqueeze(-1), (kwind).unsqueeze(0))
205
+
206
+ n_n_0 = self.supn - n0
207
+ k_k_0 = self.supk - k0
208
+ n_mult = torch.matmul(
209
+ n_n_0.unsqueeze(1),
210
+ torch.ones((1, len(self.supk))).type(torch.FloatTensor).to(
211
+ self.rates_.device))
212
+ k_mult = torch.matmul(
213
+ torch.ones((len(self.supn),
214
+ 1)).type(torch.FloatTensor).to(self.rates_.device),
215
+ k_k_0.unsqueeze(0))
216
+
217
+ inside = self.rates_.unsqueeze(1).unsqueeze(
218
+ 1) * n_mult + self.scales_.unsqueeze(1).unsqueeze(1) * k_mult
219
+ real_strf = torch.cos(inside) * new_wind.unsqueeze(0)
220
+
221
+ if self.real_only:
222
+ final_strf = real_strf
223
+
224
+ else:
225
+ imag_strf = torch.sin(inside) * new_wind.unsqueeze(0)
226
+ final_strf = torch.cat([real_strf, imag_strf], dim=0)
227
+
228
+ if self.norm_strf:
229
+ final_strf = final_strf / (torch.sum(
230
+ final_strf**2, dim=(1, 2)).unsqueeze(1).unsqueeze(2))**.5
231
+
232
+ return final_strf
233
+
234
+ def forward(self, sigspec):
235
+ """Forward pass a batch of (real) spectra [Batch x Time x Frequency]."""
236
+ if len(sigspec.shape) == 2: # expand batch dimension if single eg
237
+ sigspec = sigspec.unsqueeze(0)
238
+ strfs = self.strfs().unsqueeze(1).type_as(sigspec)
239
+ out = F.conv2d(sigspec.unsqueeze(1), strfs, padding=self.padding)
240
+ return out
241
+
242
+ def __repr__(self):
243
+ """Gabor filter"""
244
+ report = """
245
+ +++++ Gabor Filter Kernels [{}], supn[{}], supk[{}] real only [{}] norm strf [{}] +++++
246
+
247
+ """.format(self.numKern, self.numN, self.numK, self.real_only,
248
+ self.norm_strf)
249
+
250
+ return report
runtime/Lib/site_packages/models/audio/aec/network/se_net.py ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+ from modelscope.models.audio.aec.layers.activations import (RectifiedLinear,
8
+ Sigmoid)
9
+ from modelscope.models.audio.aec.layers.affine_transform import AffineTransform
10
+ from modelscope.models.audio.aec.layers.deep_fsmn import DeepFsmn
11
+ from modelscope.models.audio.aec.layers.uni_deep_fsmn import (Conv2d,
12
+ UniDeepFsmn)
13
+
14
+
15
+ class MaskNet(nn.Module):
16
+
17
+ def __init__(self,
18
+ indim,
19
+ outdim,
20
+ layers=9,
21
+ hidden_dim=128,
22
+ hidden_dim2=None,
23
+ lorder=20,
24
+ rorder=0,
25
+ dilation=1,
26
+ layer_norm=False,
27
+ dropout=0,
28
+ crm=False,
29
+ vad=False,
30
+ linearout=False):
31
+ super(MaskNet, self).__init__()
32
+
33
+ self.linear1 = AffineTransform(indim, hidden_dim)
34
+ self.relu = RectifiedLinear(hidden_dim, hidden_dim)
35
+ if hidden_dim2 is None:
36
+ hidden_dim2 = hidden_dim
37
+
38
+ if rorder == 0:
39
+ repeats = [
40
+ UniDeepFsmn(
41
+ hidden_dim,
42
+ hidden_dim,
43
+ lorder,
44
+ hidden_dim2,
45
+ dilation=dilation,
46
+ layer_norm=layer_norm,
47
+ dropout=dropout) for i in range(layers)
48
+ ]
49
+ else:
50
+ repeats = [
51
+ DeepFsmn(
52
+ hidden_dim,
53
+ hidden_dim,
54
+ lorder,
55
+ rorder,
56
+ hidden_dim2,
57
+ layer_norm=layer_norm,
58
+ dropout=dropout) for i in range(layers)
59
+ ]
60
+ self.deepfsmn = nn.Sequential(*repeats)
61
+
62
+ self.linear2 = AffineTransform(hidden_dim, outdim)
63
+
64
+ self.crm = crm
65
+ if self.crm:
66
+ self.sig = nn.Tanh()
67
+ else:
68
+ self.sig = Sigmoid(outdim, outdim)
69
+
70
+ self.vad = vad
71
+ if self.vad:
72
+ self.linear3 = AffineTransform(hidden_dim, 1)
73
+
74
+ self.layers = layers
75
+ self.linearout = linearout
76
+ if self.linearout and self.vad:
77
+ print('Warning: not supported nnet')
78
+
79
+ def forward(self, feat, ctl=None):
80
+ x1 = self.linear1(feat)
81
+ x2 = self.relu(x1)
82
+ if ctl is not None:
83
+ ctl = min(ctl, self.layers - 1)
84
+ for i in range(ctl):
85
+ x2 = self.deepfsmn[i](x2)
86
+ mask = self.sig(self.linear2(x2))
87
+ if self.vad:
88
+ vad = torch.sigmoid(self.linear3(x2))
89
+ return mask, vad
90
+ else:
91
+ return mask
92
+ x3 = self.deepfsmn(x2)
93
+ if self.linearout:
94
+ return self.linear2(x3)
95
+ mask = self.sig(self.linear2(x3))
96
+ if self.vad:
97
+ vad = torch.sigmoid(self.linear3(x3))
98
+ return mask, vad
99
+ else:
100
+ return mask
101
+
102
+ def to_kaldi_nnet(self):
103
+ re_str = ''
104
+ re_str += '<Nnet>\n'
105
+ re_str += self.linear1.to_kaldi_nnet()
106
+ re_str += self.relu.to_kaldi_nnet()
107
+ for dfsmn in self.deepfsmn:
108
+ re_str += dfsmn.to_kaldi_nnet()
109
+ re_str += self.linear2.to_kaldi_nnet()
110
+ re_str += self.sig.to_kaldi_nnet()
111
+ re_str += '</Nnet>\n'
112
+
113
+ return re_str
114
+
115
+ def to_raw_nnet(self, fid):
116
+ self.linear1.to_raw_nnet(fid)
117
+ for dfsmn in self.deepfsmn:
118
+ dfsmn.to_raw_nnet(fid)
119
+ self.linear2.to_raw_nnet(fid)
120
+
121
+
122
+ class StageNet(nn.Module):
123
+
124
+ def __init__(self,
125
+ indim,
126
+ outdim,
127
+ layers=9,
128
+ layers2=6,
129
+ hidden_dim=128,
130
+ lorder=20,
131
+ rorder=0,
132
+ layer_norm=False,
133
+ dropout=0,
134
+ crm=False,
135
+ vad=False,
136
+ linearout=False):
137
+ super(StageNet, self).__init__()
138
+
139
+ self.stage1 = nn.ModuleList()
140
+ self.stage2 = nn.ModuleList()
141
+ layer = nn.Sequential(nn.Linear(indim, hidden_dim), nn.ReLU())
142
+ self.stage1.append(layer)
143
+ for i in range(layers):
144
+ layer = UniDeepFsmn(
145
+ hidden_dim,
146
+ hidden_dim,
147
+ lorder,
148
+ hidden_dim,
149
+ layer_norm=layer_norm,
150
+ dropout=dropout)
151
+ self.stage1.append(layer)
152
+ layer = nn.Sequential(nn.Linear(hidden_dim, 321), nn.Sigmoid())
153
+ self.stage1.append(layer)
154
+ # stage2
155
+ layer = nn.Sequential(nn.Linear(321 + indim, hidden_dim), nn.ReLU())
156
+ self.stage2.append(layer)
157
+ for i in range(layers2):
158
+ layer = UniDeepFsmn(
159
+ hidden_dim,
160
+ hidden_dim,
161
+ lorder,
162
+ hidden_dim,
163
+ layer_norm=layer_norm,
164
+ dropout=dropout)
165
+ self.stage2.append(layer)
166
+ layer = nn.Sequential(
167
+ nn.Linear(hidden_dim, outdim),
168
+ nn.Sigmoid() if not crm else nn.Tanh())
169
+ self.stage2.append(layer)
170
+ self.crm = crm
171
+ self.vad = vad
172
+ self.linearout = linearout
173
+ self.window = torch.hamming_window(640, periodic=False).cuda()
174
+ self.freezed = False
175
+
176
+ def freeze(self):
177
+ if not self.freezed:
178
+ for param in self.stage1.parameters():
179
+ param.requires_grad = False
180
+ self.freezed = True
181
+ print('freezed stage1')
182
+
183
+ def forward(self, feat, mixture, ctl=None):
184
+ if ctl == 'off':
185
+ x = feat
186
+ for i in range(len(self.stage1)):
187
+ x = self.stage1[i](x)
188
+ return x
189
+ else:
190
+ self.freeze()
191
+ x = feat
192
+ for i in range(len(self.stage1)):
193
+ x = self.stage1[i](x)
194
+
195
+ spec = torch.stft(
196
+ mixture / 32768,
197
+ 640,
198
+ 320,
199
+ 640,
200
+ self.window,
201
+ center=False,
202
+ return_complex=True)
203
+ spec = torch.view_as_real(spec).permute([0, 2, 1, 3])
204
+ specmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2)
205
+ est = x * specmag
206
+ y = torch.cat([est, feat], dim=-1)
207
+ for i in range(len(self.stage2)):
208
+ y = self.stage2[i](y)
209
+ return y
210
+
211
+
212
+ class Unet(nn.Module):
213
+
214
+ def __init__(self,
215
+ indim,
216
+ outdim,
217
+ layers=9,
218
+ dims=[256] * 4,
219
+ lorder=20,
220
+ rorder=0,
221
+ dilation=1,
222
+ layer_norm=False,
223
+ dropout=0,
224
+ crm=False,
225
+ vad=False,
226
+ linearout=False):
227
+ super(Unet, self).__init__()
228
+
229
+ self.linear1 = AffineTransform(indim, dims[0])
230
+ self.relu = RectifiedLinear(dims[0], dims[0])
231
+
232
+ self.encoder = nn.ModuleList()
233
+ self.decoder = nn.ModuleList()
234
+ for i in range(len(dims) - 1):
235
+ layer = nn.Sequential(
236
+ nn.Linear(dims[i], dims[i + 1]), nn.ReLU(),
237
+ nn.Linear(dims[i + 1], dims[i + 1], bias=False),
238
+ Conv2d(
239
+ dims[i + 1],
240
+ dims[i + 1],
241
+ lorder,
242
+ groups=dims[i + 1],
243
+ skip_connect=True))
244
+ self.encoder.append(layer)
245
+ for i in range(len(dims) - 1, 0, -1):
246
+ layer = nn.Sequential(
247
+ nn.Linear(dims[i] * 2, dims[i - 1]), nn.ReLU(),
248
+ nn.Linear(dims[i - 1], dims[i - 1], bias=False),
249
+ Conv2d(
250
+ dims[i - 1],
251
+ dims[i - 1],
252
+ lorder,
253
+ groups=dims[i - 1],
254
+ skip_connect=True))
255
+ self.decoder.append(layer)
256
+ self.tf = nn.ModuleList()
257
+ for i in range(layers - 2 * (len(dims) - 1)):
258
+ layer = nn.Sequential(
259
+ nn.Linear(dims[-1], dims[-1]), nn.ReLU(),
260
+ nn.Linear(dims[-1], dims[-1], bias=False),
261
+ Conv2d(
262
+ dims[-1],
263
+ dims[-1],
264
+ lorder,
265
+ groups=dims[-1],
266
+ skip_connect=True))
267
+ self.tf.append(layer)
268
+
269
+ self.linear2 = AffineTransform(dims[0], outdim)
270
+ self.crm = crm
271
+ self.act = nn.Tanh() if self.crm else nn.Sigmoid()
272
+ self.vad = False
273
+ self.layers = layers
274
+ self.linearout = linearout
275
+
276
+ def forward(self, x, ctl=None):
277
+ x = self.linear1(x)
278
+ x = self.relu(x)
279
+
280
+ encoder_out = []
281
+ for i in range(len(self.encoder)):
282
+ x = self.encoder[i](x)
283
+ encoder_out.append(x)
284
+ for i in range(len(self.tf)):
285
+ x = self.tf[i](x)
286
+ for i in range(len(self.decoder)):
287
+ x = torch.cat([x, encoder_out[-1 - i]], dim=-1)
288
+ x = self.decoder[i](x)
289
+
290
+ x = self.linear2(x)
291
+ if self.linearout:
292
+ return x
293
+ return self.act(x)
294
+
295
+
296
+ class BranchNet(nn.Module):
297
+
298
+ def __init__(self,
299
+ indim,
300
+ outdim,
301
+ layers=9,
302
+ hidden_dim=256,
303
+ lorder=20,
304
+ rorder=0,
305
+ dilation=1,
306
+ layer_norm=False,
307
+ dropout=0,
308
+ crm=False,
309
+ vad=False,
310
+ linearout=False):
311
+ super(BranchNet, self).__init__()
312
+
313
+ self.linear1 = AffineTransform(indim, hidden_dim)
314
+ self.relu = RectifiedLinear(hidden_dim, hidden_dim)
315
+
316
+ self.convs = nn.ModuleList()
317
+ self.deepfsmn = nn.ModuleList()
318
+ self.FREQ = nn.ModuleList()
319
+ self.TIME = nn.ModuleList()
320
+ self.br1 = nn.ModuleList()
321
+ self.br2 = nn.ModuleList()
322
+ for i in range(layers):
323
+ '''
324
+ layer = nn.Sequential(
325
+ nn.Linear(hidden_dim, hidden_dim),
326
+ nn.ReLU(),
327
+ nn.Linear(hidden_dim, hidden_dim, bias=False),
328
+ Conv2d(hidden_dim, hidden_dim, lorder,
329
+ groups=hidden_dim, skip_connect=True)
330
+ )
331
+ self.deepfsmn.append(layer)
332
+ '''
333
+ layer = nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.ReLU())
334
+ self.FREQ.append(layer)
335
+ '''
336
+ layer = nn.GRU(hidden_dim, hidden_dim,
337
+ batch_first=True,
338
+ bidirectional=False)
339
+ self.TIME.append(layer)
340
+
341
+ layer = nn.Sequential(
342
+ nn.Linear(hidden_dim, hidden_dim//2, bias=False),
343
+ Conv2d(hidden_dim//2, hidden_dim//2, lorder,
344
+ groups=hidden_dim//2, skip_connect=True)
345
+ )
346
+ self.br1.append(layer)
347
+ layer = nn.GRU(hidden_dim, hidden_dim//2,
348
+ batch_first=True,
349
+ bidirectional=False)
350
+ self.br2.append(layer)
351
+ '''
352
+
353
+ self.linear2 = AffineTransform(hidden_dim, outdim)
354
+ self.crm = crm
355
+ self.act = nn.Tanh() if self.crm else nn.Sigmoid()
356
+ self.vad = False
357
+ self.layers = layers
358
+ self.linearout = linearout
359
+
360
+ def forward(self, x, ctl=None):
361
+ return self.forward_branch(x)
362
+
363
+ def forward_sepconv(self, x):
364
+ x = torch.unsqueeze(x, 1)
365
+ for i in range(len(self.convs)):
366
+ x = self.convs[i](x)
367
+ x = F.relu(x)
368
+ B, C, H, W = x.shape
369
+ x = x.permute(0, 2, 1, 3)
370
+ x = torch.reshape(x, [B, H, C * W])
371
+ x = self.linear1(x)
372
+ x = self.relu(x)
373
+ for i in range(self.layers):
374
+ x = self.deepfsmn[i](x) + x
375
+ x = self.linear2(x)
376
+ return self.act(x)
377
+
378
+ def forward_branch(self, x):
379
+ x = self.linear1(x)
380
+ x = self.relu(x)
381
+ for i in range(self.layers):
382
+ z = self.FREQ[i](x)
383
+ x = z + x
384
+ x = self.linear2(x)
385
+ if self.linearout:
386
+ return x
387
+ return self.act(x)
388
+
389
+
390
+ class TACNet(nn.Module):
391
+ ''' transform average concatenate for ad hoc dr
392
+ '''
393
+
394
+ def __init__(self,
395
+ indim,
396
+ outdim,
397
+ layers=9,
398
+ hidden_dim=128,
399
+ lorder=20,
400
+ rorder=0,
401
+ crm=False,
402
+ vad=False,
403
+ linearout=False):
404
+ super(TACNet, self).__init__()
405
+
406
+ self.linear1 = AffineTransform(indim, hidden_dim)
407
+ self.relu = RectifiedLinear(hidden_dim, hidden_dim)
408
+
409
+ if rorder == 0:
410
+ repeats = [
411
+ UniDeepFsmn(hidden_dim, hidden_dim, lorder, hidden_dim)
412
+ for i in range(layers)
413
+ ]
414
+ else:
415
+ repeats = [
416
+ DeepFsmn(hidden_dim, hidden_dim, lorder, rorder, hidden_dim)
417
+ for i in range(layers)
418
+ ]
419
+ self.deepfsmn = nn.Sequential(*repeats)
420
+
421
+ self.ch_transform = nn.ModuleList([])
422
+ self.ch_average = nn.ModuleList([])
423
+ self.ch_concat = nn.ModuleList([])
424
+ for i in range(layers):
425
+ self.ch_transform.append(
426
+ nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU()))
427
+ self.ch_average.append(
428
+ nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU()))
429
+ self.ch_concat.append(
430
+ nn.Sequential(
431
+ nn.Linear(hidden_dim * 2, hidden_dim), nn.PReLU()))
432
+
433
+ self.linear2 = AffineTransform(hidden_dim, outdim)
434
+
435
+ self.crm = crm
436
+ if self.crm:
437
+ self.sig = nn.Tanh()
438
+ else:
439
+ self.sig = Sigmoid(outdim, outdim)
440
+
441
+ self.vad = vad
442
+ if self.vad:
443
+ self.linear3 = AffineTransform(hidden_dim, 1)
444
+
445
+ self.layers = layers
446
+ self.linearout = linearout
447
+ if self.linearout and self.vad:
448
+ print('Warning: not supported nnet')
449
+
450
+ def forward(self, feat, ctl=None):
451
+ B, T, F = feat.shape
452
+ # assume 4ch
453
+ ch = 4
454
+ zlist = []
455
+ for c in range(ch):
456
+ z = self.linear1(feat[..., c * (F // 4):(c + 1) * (F // 4)])
457
+ z = self.relu(z)
458
+ zlist.append(z)
459
+ for i in range(self.layers):
460
+ # forward
461
+ for c in range(ch):
462
+ zlist[c] = self.deepfsmn[i](zlist[c])
463
+
464
+ # transform
465
+ olist = []
466
+ for c in range(ch):
467
+ z = self.ch_transform[i](zlist[c])
468
+ olist.append(z)
469
+ # average
470
+ avg = 0
471
+ for c in range(ch):
472
+ avg = avg + olist[c]
473
+ avg = avg / ch
474
+ avg = self.ch_average[i](avg)
475
+ # concate
476
+ for c in range(ch):
477
+ tac = torch.cat([olist[c], avg], dim=-1)
478
+ tac = self.ch_concat[i](tac)
479
+ zlist[c] = zlist[c] + tac
480
+
481
+ for c in range(ch):
482
+ zlist[c] = self.sig(self.linear2(zlist[c]))
483
+ mask = torch.cat(zlist, dim=-1)
484
+ return mask
485
+
486
+ def to_kaldi_nnet(self):
487
+ pass
runtime/Lib/site_packages/models/audio/ans/__init__.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.import_utils import LazyImportModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .frcrn import FRCRNDecorator
8
+
9
+ else:
10
+ _import_structure = {
11
+ 'frcrn': ['FRCRNDecorator'],
12
+ 'dnoise_net': ['DenoiseNet'],
13
+ }
14
+
15
+ import sys
16
+
17
+ sys.modules[__name__] = LazyImportModule(
18
+ __name__,
19
+ globals()['__file__'],
20
+ _import_structure,
21
+ module_spec=__spec__,
22
+ extra_objects={},
23
+ )
runtime/Lib/site_packages/models/audio/ans/complex_nn.py ADDED
@@ -0,0 +1,206 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ #
3
+ # The implementation of class ComplexConv2d, ComplexConvTranspose2d and
4
+ # ComplexBatchNorm2d here is modified based on Jongho Choi(sweetcocoa@snu.ac.kr
5
+ # / Seoul National Univ., ESTsoft ) and publicly available at
6
+ # https://github.com/sweetcocoa/DeepComplexUNetPyTorch
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+
11
+ from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn
12
+
13
+
14
+ class ComplexUniDeepFsmn(nn.Module):
15
+
16
+ def __init__(self, nIn, nHidden=128, nOut=128):
17
+ super(ComplexUniDeepFsmn, self).__init__()
18
+
19
+ self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
20
+ self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
21
+ self.fsmn_re_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden)
22
+ self.fsmn_im_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden)
23
+
24
+ def forward(self, x):
25
+ r"""
26
+
27
+ Args:
28
+ x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2]
29
+
30
+ Returns:
31
+ [batch, feature, sequence, 2], eg: [6, 99, 1024, 2]
32
+ """
33
+ #
34
+ b, c, h, T, d = x.size()
35
+ x = torch.reshape(x, (b, c * h, T, d))
36
+ # x: [b,h,T,2], [6, 256, 106, 2]
37
+ x = torch.transpose(x, 1, 2)
38
+ # x: [b,T,h,2], [6, 106, 256, 2]
39
+
40
+ real_L1 = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1])
41
+ imaginary_L1 = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0])
42
+ # GRU output: [99, 6, 128]
43
+ real = self.fsmn_re_L2(real_L1) - self.fsmn_im_L2(imaginary_L1)
44
+ imaginary = self.fsmn_re_L2(imaginary_L1) + self.fsmn_im_L2(real_L1)
45
+ # output: [b,T,h,2], [99, 6, 1024, 2]
46
+ output = torch.stack((real, imaginary), dim=-1)
47
+
48
+ # output: [b,h,T,2], [6, 99, 1024, 2]
49
+ output = torch.transpose(output, 1, 2)
50
+ output = torch.reshape(output, (b, c, h, T, d))
51
+
52
+ return output
53
+
54
+
55
+ class ComplexUniDeepFsmn_L1(nn.Module):
56
+
57
+ def __init__(self, nIn, nHidden=128, nOut=128):
58
+ super(ComplexUniDeepFsmn_L1, self).__init__()
59
+ self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
60
+ self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden)
61
+
62
+ def forward(self, x):
63
+ r"""
64
+
65
+ Args:
66
+ x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2]
67
+ """
68
+ b, c, h, T, d = x.size()
69
+ # x : [b,T,h,c,2]
70
+ x = torch.transpose(x, 1, 3)
71
+ x = torch.reshape(x, (b * T, h, c, d))
72
+
73
+ real = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1])
74
+ imaginary = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0])
75
+ # output: [b*T,h,c,2], [6*106, h, 256, 2]
76
+ output = torch.stack((real, imaginary), dim=-1)
77
+
78
+ output = torch.reshape(output, (b, T, h, c, d))
79
+ output = torch.transpose(output, 1, 3)
80
+ return output
81
+
82
+
83
+ class ComplexConv2d(nn.Module):
84
+ # https://github.com/litcoderr/ComplexCNN/blob/master/complexcnn/modules.py
85
+ def __init__(self,
86
+ in_channel,
87
+ out_channel,
88
+ kernel_size,
89
+ stride=1,
90
+ padding=0,
91
+ dilation=1,
92
+ groups=1,
93
+ bias=True,
94
+ **kwargs):
95
+ super().__init__()
96
+
97
+ # Model components
98
+ self.conv_re = nn.Conv2d(
99
+ in_channel,
100
+ out_channel,
101
+ kernel_size,
102
+ stride=stride,
103
+ padding=padding,
104
+ dilation=dilation,
105
+ groups=groups,
106
+ bias=bias,
107
+ **kwargs)
108
+ self.conv_im = nn.Conv2d(
109
+ in_channel,
110
+ out_channel,
111
+ kernel_size,
112
+ stride=stride,
113
+ padding=padding,
114
+ dilation=dilation,
115
+ groups=groups,
116
+ bias=bias,
117
+ **kwargs)
118
+
119
+ def forward(self, x):
120
+ r"""
121
+
122
+ Args:
123
+ x: torch with shape: [batch,channel,axis1,axis2,2]
124
+ """
125
+ real = self.conv_re(x[..., 0]) - self.conv_im(x[..., 1])
126
+ imaginary = self.conv_re(x[..., 1]) + self.conv_im(x[..., 0])
127
+ output = torch.stack((real, imaginary), dim=-1)
128
+ return output
129
+
130
+
131
+ class ComplexConvTranspose2d(nn.Module):
132
+
133
+ def __init__(self,
134
+ in_channel,
135
+ out_channel,
136
+ kernel_size,
137
+ stride=1,
138
+ padding=0,
139
+ output_padding=0,
140
+ dilation=1,
141
+ groups=1,
142
+ bias=True,
143
+ **kwargs):
144
+ super().__init__()
145
+
146
+ # Model components
147
+ self.tconv_re = nn.ConvTranspose2d(
148
+ in_channel,
149
+ out_channel,
150
+ kernel_size=kernel_size,
151
+ stride=stride,
152
+ padding=padding,
153
+ output_padding=output_padding,
154
+ groups=groups,
155
+ bias=bias,
156
+ dilation=dilation,
157
+ **kwargs)
158
+ self.tconv_im = nn.ConvTranspose2d(
159
+ in_channel,
160
+ out_channel,
161
+ kernel_size=kernel_size,
162
+ stride=stride,
163
+ padding=padding,
164
+ output_padding=output_padding,
165
+ groups=groups,
166
+ bias=bias,
167
+ dilation=dilation,
168
+ **kwargs)
169
+
170
+ def forward(self, x): # shpae of x : [batch,channel,axis1,axis2,2]
171
+ real = self.tconv_re(x[..., 0]) - self.tconv_im(x[..., 1])
172
+ imaginary = self.tconv_re(x[..., 1]) + self.tconv_im(x[..., 0])
173
+ output = torch.stack((real, imaginary), dim=-1)
174
+ return output
175
+
176
+
177
+ class ComplexBatchNorm2d(nn.Module):
178
+
179
+ def __init__(self,
180
+ num_features,
181
+ eps=1e-5,
182
+ momentum=0.1,
183
+ affine=True,
184
+ track_running_stats=True,
185
+ **kwargs):
186
+ super().__init__()
187
+ self.bn_re = nn.BatchNorm2d(
188
+ num_features=num_features,
189
+ momentum=momentum,
190
+ affine=affine,
191
+ eps=eps,
192
+ track_running_stats=track_running_stats,
193
+ **kwargs)
194
+ self.bn_im = nn.BatchNorm2d(
195
+ num_features=num_features,
196
+ momentum=momentum,
197
+ affine=affine,
198
+ eps=eps,
199
+ track_running_stats=track_running_stats,
200
+ **kwargs)
201
+
202
+ def forward(self, x):
203
+ real = self.bn_re(x[..., 0])
204
+ imag = self.bn_im(x[..., 1])
205
+ output = torch.stack((real, imag), dim=-1)
206
+ return output
runtime/Lib/site_packages/models/audio/ans/conv_stft.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ import numpy as np
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+ from scipy.signal import get_window
7
+
8
+
9
+ def init_kernels(win_len, win_inc, fft_len, win_type=None, invers=False):
10
+ if win_type == 'None' or win_type is None:
11
+ window = np.ones(win_len)
12
+ else:
13
+ window = get_window(win_type, win_len, fftbins=True)**0.5
14
+
15
+ N = fft_len
16
+ fourier_basis = np.fft.rfft(np.eye(N))[:win_len]
17
+ real_kernel = np.real(fourier_basis)
18
+ imag_kernel = np.imag(fourier_basis)
19
+ kernel = np.concatenate([real_kernel, imag_kernel], 1).T
20
+
21
+ if invers:
22
+ kernel = np.linalg.pinv(kernel).T
23
+
24
+ kernel = kernel * window
25
+ kernel = kernel[:, None, :]
26
+ return torch.from_numpy(kernel.astype(np.float32)), torch.from_numpy(
27
+ window[None, :, None].astype(np.float32))
28
+
29
+
30
+ class ConvSTFT(nn.Module):
31
+
32
+ def __init__(self,
33
+ win_len,
34
+ win_inc,
35
+ fft_len=None,
36
+ win_type='hamming',
37
+ feature_type='real',
38
+ fix=True):
39
+ super(ConvSTFT, self).__init__()
40
+
41
+ if fft_len is None:
42
+ self.fft_len = int(2**np.ceil(np.log2(win_len)))
43
+ else:
44
+ self.fft_len = fft_len
45
+
46
+ kernel, _ = init_kernels(win_len, win_inc, self.fft_len, win_type)
47
+ self.weight = nn.Parameter(kernel, requires_grad=(not fix))
48
+ self.feature_type = feature_type
49
+ self.stride = win_inc
50
+ self.win_len = win_len
51
+ self.dim = self.fft_len
52
+
53
+ def forward(self, inputs):
54
+ if inputs.dim() == 2:
55
+ inputs = torch.unsqueeze(inputs, 1)
56
+
57
+ outputs = F.conv1d(inputs, self.weight, stride=self.stride)
58
+
59
+ if self.feature_type == 'complex':
60
+ return outputs
61
+ else:
62
+ dim = self.dim // 2 + 1
63
+ real = outputs[:, :dim, :]
64
+ imag = outputs[:, dim:, :]
65
+ mags = torch.sqrt(real**2 + imag**2)
66
+ phase = torch.atan2(imag, real)
67
+ return mags, phase
68
+
69
+
70
+ class ConviSTFT(nn.Module):
71
+
72
+ def __init__(self,
73
+ win_len,
74
+ win_inc,
75
+ fft_len=None,
76
+ win_type='hamming',
77
+ feature_type='real',
78
+ fix=True):
79
+ super(ConviSTFT, self).__init__()
80
+ if fft_len is None:
81
+ self.fft_len = int(2**np.ceil(np.log2(win_len)))
82
+ else:
83
+ self.fft_len = fft_len
84
+ kernel, window = init_kernels(
85
+ win_len, win_inc, self.fft_len, win_type, invers=True)
86
+ self.weight = nn.Parameter(kernel, requires_grad=(not fix))
87
+ self.feature_type = feature_type
88
+ self.win_type = win_type
89
+ self.win_len = win_len
90
+ self.win_inc = win_inc
91
+ self.stride = win_inc
92
+ self.dim = self.fft_len
93
+ self.register_buffer('window', window)
94
+ self.register_buffer('enframe', torch.eye(win_len)[:, None, :])
95
+
96
+ def forward(self, inputs, phase=None):
97
+ """
98
+ Args:
99
+ inputs : [B, N+2, T] (complex spec) or [B, N//2+1, T] (mags)
100
+ phase: [B, N//2+1, T] (if not none)
101
+ """
102
+
103
+ if phase is not None:
104
+ real = inputs * torch.cos(phase)
105
+ imag = inputs * torch.sin(phase)
106
+ inputs = torch.cat([real, imag], 1)
107
+ outputs = F.conv_transpose1d(inputs, self.weight, stride=self.stride)
108
+
109
+ # this is from torch-stft: https://github.com/pseeth/torch-stft
110
+ t = self.window.repeat(1, 1, inputs.size(-1))**2
111
+ coff = F.conv_transpose1d(t, self.enframe, stride=self.stride)
112
+ outputs = outputs / (coff + 1e-8)
113
+ return outputs
runtime/Lib/site_packages/models/audio/ans/denoise_net.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ # Related papers:
3
+ # Shengkui Zhao, Trung Hieu Nguyen, Bin Ma, “Monaural Speech Enhancement with Complex Convolutional
4
+ # Block Attention Module and Joint Time Frequency Losses”, ICASSP 2021.
5
+ # Shiliang Zhang, Ming Lei, Zhijie Yan, Lirong Dai, “Deep-FSMN for Large Vocabulary Continuous Speech
6
+ # Recognition “, arXiv:1803.05030, 2018.
7
+
8
+ from torch import nn
9
+
10
+ from modelscope.metainfo import Models
11
+ from modelscope.models import MODELS, TorchModel
12
+ from modelscope.models.audio.ans.layers.activations import (RectifiedLinear,
13
+ Sigmoid)
14
+ from modelscope.models.audio.ans.layers.affine_transform import AffineTransform
15
+ from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn
16
+ from modelscope.utils.constant import Tasks
17
+
18
+
19
+ @MODELS.register_module(
20
+ Tasks.acoustic_noise_suppression, module_name=Models.speech_dfsmn_ans)
21
+ class DfsmnAns(TorchModel):
22
+ """Denoise model with DFSMN.
23
+
24
+ Args:
25
+ model_dir (str): the model path.
26
+ fsmn_depth (int): the depth of deepfsmn
27
+ lorder (int):
28
+ """
29
+
30
+ def __init__(self,
31
+ model_dir: str,
32
+ fsmn_depth=9,
33
+ lorder=20,
34
+ *args,
35
+ **kwargs):
36
+ super().__init__(model_dir, *args, **kwargs)
37
+ self.lorder = lorder
38
+ self.linear1 = AffineTransform(120, 256)
39
+ self.relu = RectifiedLinear(256, 256)
40
+ repeats = [
41
+ UniDeepFsmn(256, 256, lorder, 256) for i in range(fsmn_depth)
42
+ ]
43
+ self.deepfsmn = nn.Sequential(*repeats)
44
+ self.linear2 = AffineTransform(256, 961)
45
+ self.sig = Sigmoid(961, 961)
46
+
47
+ def forward(self, input):
48
+ """
49
+ Args:
50
+ input: fbank feature [batch_size,number_of_frame,feature_dimension]
51
+
52
+ Returns:
53
+ mask value [batch_size, number_of_frame, FFT_size/2+1]
54
+ """
55
+ x1 = self.linear1(input)
56
+ x2 = self.relu(x1)
57
+ x3 = self.deepfsmn(x2)
58
+ x4 = self.linear2(x3)
59
+ x5 = self.sig(x4)
60
+ return x5
61
+
62
+ def to_kaldi_nnet(self):
63
+ re_str = ''
64
+ re_str += '<Nnet>\n'
65
+ re_str += self.linear1.to_kaldi_nnet()
66
+ re_str += self.relu.to_kaldi_nnet()
67
+ for dfsmn in self.deepfsmn:
68
+ re_str += dfsmn.to_kaldi_nnet()
69
+ re_str += self.linear2.to_kaldi_nnet()
70
+ re_str += self.sig.to_kaldi_nnet()
71
+ re_str += '</Nnet>\n'
72
+
73
+ return re_str
runtime/Lib/site_packages/models/audio/ans/frcrn.py ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ import os
3
+ from typing import Dict
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.nn.functional as F
8
+
9
+ from modelscope.metainfo import Models
10
+ from modelscope.models import TorchModel
11
+ from modelscope.models.base import Tensor
12
+ from modelscope.models.builder import MODELS
13
+ from modelscope.utils.constant import ModelFile, Tasks
14
+ from .conv_stft import ConviSTFT, ConvSTFT
15
+ from .unet import UNet
16
+
17
+
18
+ @MODELS.register_module(
19
+ Tasks.acoustic_noise_suppression,
20
+ module_name=Models.speech_frcrn_ans_cirm_16k)
21
+ class FRCRNDecorator(TorchModel):
22
+ r""" A decorator of FRCRN for integrating into modelscope framework """
23
+
24
+ def __init__(self, model_dir: str, *args, **kwargs):
25
+ """initialize the frcrn model from the `model_dir` path.
26
+
27
+ Args:
28
+ model_dir (str): the model path.
29
+ """
30
+ super().__init__(model_dir, *args, **kwargs)
31
+ self.model = FRCRN(*args, **kwargs)
32
+ model_bin_file = os.path.join(model_dir,
33
+ ModelFile.TORCH_MODEL_BIN_FILE)
34
+ if os.path.exists(model_bin_file):
35
+ checkpoint = torch.load(
36
+ model_bin_file, map_location=torch.device('cpu'))
37
+ if isinstance(checkpoint, dict) and 'state_dict' in checkpoint:
38
+ # the new trained model by user is based on FRCRNDecorator
39
+ self.load_state_dict(checkpoint['state_dict'])
40
+ else:
41
+ # The released model on Modelscope is based on FRCRN
42
+ self.model.load_state_dict(checkpoint, strict=False)
43
+
44
+ def forward(self, inputs: Dict[str, Tensor]) -> Dict[str, Tensor]:
45
+ result_list = self.model.forward(inputs['noisy'])
46
+ output = {
47
+ 'spec_l1': result_list[0],
48
+ 'wav_l1': result_list[1],
49
+ 'mask_l1': result_list[2],
50
+ 'spec_l2': result_list[3],
51
+ 'wav_l2': result_list[4],
52
+ 'mask_l2': result_list[5]
53
+ }
54
+ if 'clean' in inputs:
55
+ mix_result = self.model.loss(
56
+ inputs['noisy'], inputs['clean'], result_list, mode='Mix')
57
+ output.update(mix_result)
58
+ sisnr_result = self.model.loss(
59
+ inputs['noisy'], inputs['clean'], result_list, mode='SiSNR')
60
+ output.update(sisnr_result)
61
+ # logger hooker will use items under 'log_vars'
62
+ output['log_vars'] = {k: mix_result[k].item() for k in mix_result}
63
+ output['log_vars'].update(
64
+ {k: sisnr_result[k].item()
65
+ for k in sisnr_result})
66
+ return output
67
+
68
+
69
+ class FRCRN(nn.Module):
70
+ r""" Frequency Recurrent CRN """
71
+
72
+ def __init__(self,
73
+ complex,
74
+ model_complexity,
75
+ model_depth,
76
+ log_amp,
77
+ padding_mode,
78
+ win_len=400,
79
+ win_inc=100,
80
+ fft_len=512,
81
+ win_type='hann',
82
+ **kwargs):
83
+ r"""
84
+ Args:
85
+ complex: Whether to use complex networks.
86
+ model_complexity: define the model complexity with the number of layers
87
+ model_depth: Only two options are available : 10, 20
88
+ log_amp: Whether to use log amplitude to estimate signals
89
+ padding_mode: Encoder's convolution filter. 'zeros', 'reflect'
90
+ win_len: length of window used for defining one frame of sample points
91
+ win_inc: length of window shifting (equivalent to hop_size)
92
+ fft_len: number of Short Time Fourier Transform (STFT) points
93
+ win_type: windowing type used in STFT, eg. 'hanning', 'hamming'
94
+ """
95
+ super().__init__()
96
+ self.feat_dim = fft_len // 2 + 1
97
+
98
+ self.win_len = win_len
99
+ self.win_inc = win_inc
100
+ self.fft_len = fft_len
101
+ self.win_type = win_type
102
+
103
+ fix = True
104
+ self.stft = ConvSTFT(
105
+ self.win_len,
106
+ self.win_inc,
107
+ self.fft_len,
108
+ self.win_type,
109
+ feature_type='complex',
110
+ fix=fix)
111
+ self.istft = ConviSTFT(
112
+ self.win_len,
113
+ self.win_inc,
114
+ self.fft_len,
115
+ self.win_type,
116
+ feature_type='complex',
117
+ fix=fix)
118
+ self.unet = UNet(
119
+ 1,
120
+ complex=complex,
121
+ model_complexity=model_complexity,
122
+ model_depth=model_depth,
123
+ padding_mode=padding_mode)
124
+ self.unet2 = UNet(
125
+ 1,
126
+ complex=complex,
127
+ model_complexity=model_complexity,
128
+ model_depth=model_depth,
129
+ padding_mode=padding_mode)
130
+
131
+ def forward(self, inputs):
132
+ out_list = []
133
+ # [B, D*2, T]
134
+ cmp_spec = self.stft(inputs)
135
+ # [B, 1, D*2, T]
136
+ cmp_spec = torch.unsqueeze(cmp_spec, 1)
137
+
138
+ # to [B, 2, D, T] real_part/imag_part
139
+ cmp_spec = torch.cat([
140
+ cmp_spec[:, :, :self.feat_dim, :],
141
+ cmp_spec[:, :, self.feat_dim:, :],
142
+ ], 1)
143
+
144
+ # [B, 2, D, T]
145
+ cmp_spec = torch.unsqueeze(cmp_spec, 4)
146
+ # [B, 1, D, T, 2]
147
+ cmp_spec = torch.transpose(cmp_spec, 1, 4)
148
+ unet1_out = self.unet(cmp_spec)
149
+ cmp_mask1 = torch.tanh(unet1_out)
150
+ unet2_out = self.unet2(unet1_out)
151
+ cmp_mask2 = torch.tanh(unet2_out)
152
+ est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask1)
153
+ out_list.append(est_spec)
154
+ out_list.append(est_wav)
155
+ out_list.append(est_mask)
156
+ cmp_mask2 = cmp_mask2 + cmp_mask1
157
+ est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask2)
158
+ out_list.append(est_spec)
159
+ out_list.append(est_wav)
160
+ out_list.append(est_mask)
161
+ return out_list
162
+
163
+ def apply_mask(self, cmp_spec, cmp_mask):
164
+ est_spec = torch.cat([
165
+ cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 0]
166
+ - cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 1],
167
+ cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 1]
168
+ + cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 0]
169
+ ], 1)
170
+ est_spec = torch.cat([est_spec[:, 0, :, :], est_spec[:, 1, :, :]], 1)
171
+ cmp_mask = torch.squeeze(cmp_mask, 1)
172
+ cmp_mask = torch.cat([cmp_mask[:, :, :, 0], cmp_mask[:, :, :, 1]], 1)
173
+
174
+ est_wav = self.istft(est_spec)
175
+ est_wav = torch.squeeze(est_wav, 1)
176
+ return est_spec, est_wav, cmp_mask
177
+
178
+ def get_params(self, weight_decay=0.0):
179
+ # add L2 penalty
180
+ weights, biases = [], []
181
+ for name, param in self.named_parameters():
182
+ if 'bias' in name:
183
+ biases += [param]
184
+ else:
185
+ weights += [param]
186
+ params = [{
187
+ 'params': weights,
188
+ 'weight_decay': weight_decay,
189
+ }, {
190
+ 'params': biases,
191
+ 'weight_decay': 0.0,
192
+ }]
193
+ return params
194
+
195
+ def loss(self, noisy, labels, out_list, mode='Mix'):
196
+ if mode == 'SiSNR':
197
+ count = 0
198
+ while count < len(out_list):
199
+ est_spec = out_list[count]
200
+ count = count + 1
201
+ est_wav = out_list[count]
202
+ count = count + 1
203
+ est_mask = out_list[count]
204
+ count = count + 1
205
+ if count != 3:
206
+ loss = self.loss_1layer(noisy, est_spec, est_wav, labels,
207
+ est_mask, mode)
208
+ return dict(sisnr=loss)
209
+
210
+ elif mode == 'Mix':
211
+ count = 0
212
+ while count < len(out_list):
213
+ est_spec = out_list[count]
214
+ count = count + 1
215
+ est_wav = out_list[count]
216
+ count = count + 1
217
+ est_mask = out_list[count]
218
+ count = count + 1
219
+ if count != 3:
220
+ amp_loss, phase_loss, SiSNR_loss = self.loss_1layer(
221
+ noisy, est_spec, est_wav, labels, est_mask, mode)
222
+ loss = amp_loss + phase_loss + SiSNR_loss
223
+ return dict(loss=loss, amp_loss=amp_loss, phase_loss=phase_loss)
224
+
225
+ def loss_1layer(self, noisy, est, est_wav, labels, cmp_mask, mode='Mix'):
226
+ r""" Compute the loss by mode
227
+ mode == 'Mix'
228
+ est: [B, F*2, T]
229
+ labels: [B, F*2,T]
230
+ mode == 'SiSNR'
231
+ est: [B, T]
232
+ labels: [B, T]
233
+ """
234
+ if mode == 'SiSNR':
235
+ if labels.dim() == 3:
236
+ labels = torch.squeeze(labels, 1)
237
+ if est_wav.dim() == 3:
238
+ est_wav = torch.squeeze(est_wav, 1)
239
+ return -si_snr(est_wav, labels)
240
+ elif mode == 'Mix':
241
+
242
+ if labels.dim() == 3:
243
+ labels = torch.squeeze(labels, 1)
244
+ if est_wav.dim() == 3:
245
+ est_wav = torch.squeeze(est_wav, 1)
246
+ SiSNR_loss = -si_snr(est_wav, labels)
247
+
248
+ b, d, t = est.size()
249
+ S = self.stft(labels)
250
+ Sr = S[:, :self.feat_dim, :]
251
+ Si = S[:, self.feat_dim:, :]
252
+ Y = self.stft(noisy)
253
+ Yr = Y[:, :self.feat_dim, :]
254
+ Yi = Y[:, self.feat_dim:, :]
255
+ Y_pow = Yr**2 + Yi**2
256
+ gth_mask = torch.cat([(Sr * Yr + Si * Yi) / (Y_pow + 1e-8),
257
+ (Si * Yr - Sr * Yi) / (Y_pow + 1e-8)], 1)
258
+ gth_mask[gth_mask > 2] = 1
259
+ gth_mask[gth_mask < -2] = -1
260
+ amp_loss = F.mse_loss(gth_mask[:, :self.feat_dim, :],
261
+ cmp_mask[:, :self.feat_dim, :]) * d
262
+ phase_loss = F.mse_loss(gth_mask[:, self.feat_dim:, :],
263
+ cmp_mask[:, self.feat_dim:, :]) * d
264
+ return amp_loss, phase_loss, SiSNR_loss
265
+
266
+
267
+ def l2_norm(s1, s2):
268
+ norm = torch.sum(s1 * s2, -1, keepdim=True)
269
+ return norm
270
+
271
+
272
+ def si_snr(s1, s2, eps=1e-8):
273
+ s1_s2_norm = l2_norm(s1, s2)
274
+ s2_s2_norm = l2_norm(s2, s2)
275
+ s_target = s1_s2_norm / (s2_s2_norm + eps) * s2
276
+ e_noise = s1 - s_target
277
+ target_norm = l2_norm(s_target, s_target)
278
+ noise_norm = l2_norm(e_noise, e_noise)
279
+ snr = 10 * torch.log10((target_norm) / (noise_norm + eps) + eps)
280
+ return torch.mean(snr)
runtime/Lib/site_packages/models/audio/ans/layers/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/ans/layers/activations.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch.nn as nn
4
+
5
+ from modelscope.models.audio.ans.layers.layer_base import LayerBase
6
+
7
+
8
+ class RectifiedLinear(LayerBase):
9
+
10
+ def __init__(self, input_dim, output_dim):
11
+ super(RectifiedLinear, self).__init__()
12
+ self.dim = input_dim
13
+ self.relu = nn.ReLU()
14
+
15
+ def forward(self, input):
16
+ return self.relu(input)
17
+
18
+ def to_kaldi_nnet(self):
19
+ re_str = ''
20
+ re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
21
+ return re_str
22
+
23
+ def load_kaldi_nnet(self, instr):
24
+ return instr
25
+
26
+
27
+ class LogSoftmax(LayerBase):
28
+
29
+ def __init__(self, input_dim, output_dim):
30
+ super(LogSoftmax, self).__init__()
31
+ self.dim = input_dim
32
+ self.ls = nn.LogSoftmax()
33
+
34
+ def forward(self, input):
35
+ return self.ls(input)
36
+
37
+ def to_kaldi_nnet(self):
38
+ re_str = ''
39
+ re_str += '<Softmax> %d %d\n' % (self.dim, self.dim)
40
+ return re_str
41
+
42
+ def load_kaldi_nnet(self, instr):
43
+ return instr
44
+
45
+
46
+ class Sigmoid(LayerBase):
47
+
48
+ def __init__(self, input_dim, output_dim):
49
+ super(Sigmoid, self).__init__()
50
+ self.dim = input_dim
51
+ self.sig = nn.Sigmoid()
52
+
53
+ def forward(self, input):
54
+ return self.sig(input)
55
+
56
+ def to_kaldi_nnet(self):
57
+ re_str = ''
58
+ re_str += '<Sigmoid> %d %d\n' % (self.dim, self.dim)
59
+ return re_str
60
+
61
+ def load_kaldi_nnet(self, instr):
62
+ return instr
runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch as th
4
+ import torch.nn as nn
5
+
6
+ from modelscope.models.audio.ans.layers.layer_base import (LayerBase,
7
+ to_kaldi_matrix)
8
+ from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix,
9
+ expect_token_number)
10
+
11
+
12
+ class AffineTransform(LayerBase):
13
+
14
+ def __init__(self, input_dim, output_dim):
15
+ super(AffineTransform, self).__init__()
16
+ self.input_dim = input_dim
17
+ self.output_dim = output_dim
18
+
19
+ self.linear = nn.Linear(input_dim, output_dim)
20
+
21
+ def forward(self, input):
22
+ return self.linear(input)
23
+
24
+ def to_kaldi_nnet(self):
25
+ re_str = ''
26
+
27
+ re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
28
+ self.input_dim)
29
+
30
+ re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
31
+
32
+ linear_weights = self.state_dict()['linear.weight']
33
+
34
+ x = linear_weights.squeeze().numpy()
35
+
36
+ re_str += to_kaldi_matrix(x)
37
+
38
+ linear_bias = self.state_dict()['linear.bias']
39
+
40
+ x = linear_bias.squeeze().numpy()
41
+
42
+ re_str += to_kaldi_matrix(x)
43
+
44
+ return re_str
45
+
46
+ def load_kaldi_nnet(self, instr):
47
+
48
+ output = expect_token_number(
49
+ instr,
50
+ '<LearnRateCoef>',
51
+ )
52
+ if output is None:
53
+ raise Exception('AffineTransform format error')
54
+
55
+ instr, lr = output
56
+
57
+ output = expect_token_number(instr, '<BiasLearnRateCoef>')
58
+ if output is None:
59
+ raise Exception('AffineTransform format error')
60
+
61
+ instr, lr = output
62
+
63
+ output = expect_token_number(instr, '<MaxNorm>')
64
+ if output is None:
65
+ raise Exception('AffineTransform format error')
66
+
67
+ instr, lr = output
68
+
69
+ output = expect_kaldi_matrix(instr)
70
+
71
+ if output is None:
72
+ raise Exception('AffineTransform format error')
73
+
74
+ instr, mat = output
75
+
76
+ self.linear.weight = th.nn.Parameter(
77
+ th.from_numpy(mat).type(th.FloatTensor))
78
+
79
+ output = expect_kaldi_matrix(instr)
80
+ if output is None:
81
+ raise Exception('AffineTransform format error')
82
+
83
+ instr, mat = output
84
+ self.linear.bias = th.nn.Parameter(
85
+ th.from_numpy(mat).type(th.FloatTensor))
86
+ return instr
runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import abc
4
+
5
+ import numpy as np
6
+ import six
7
+ import torch.nn as nn
8
+
9
+
10
+ def to_kaldi_matrix(np_mat):
11
+ """ function that transform as str numpy mat to standard kaldi str matrix
12
+
13
+ Args:
14
+ np_mat: numpy mat
15
+ """
16
+ np.set_printoptions(threshold=np.inf, linewidth=np.nan)
17
+ out_str = str(np_mat)
18
+ out_str = out_str.replace('[', '')
19
+ out_str = out_str.replace(']', '')
20
+ return '[ %s ]\n' % out_str
21
+
22
+
23
+ @six.add_metaclass(abc.ABCMeta)
24
+ class LayerBase(nn.Module):
25
+
26
+ def __init__(self):
27
+ super(LayerBase, self).__init__()
28
+
29
+ @abc.abstractmethod
30
+ def to_kaldi_nnet(self):
31
+ pass
runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+ from modelscope.models.audio.ans.layers.layer_base import (LayerBase,
9
+ to_kaldi_matrix)
10
+ from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix,
11
+ expect_token_number)
12
+
13
+
14
+ class UniDeepFsmn(LayerBase):
15
+
16
+ def __init__(self, input_dim, output_dim, lorder=1, hidden_size=None):
17
+ super(UniDeepFsmn, self).__init__()
18
+
19
+ self.input_dim = input_dim
20
+ self.output_dim = output_dim
21
+ self.lorder = lorder
22
+ self.hidden_size = hidden_size
23
+
24
+ self.linear = nn.Linear(input_dim, hidden_size)
25
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
26
+ self.conv1 = nn.Conv2d(
27
+ output_dim,
28
+ output_dim, (lorder, 1), (1, 1),
29
+ groups=output_dim,
30
+ bias=False)
31
+
32
+ def forward(self, input):
33
+ """
34
+
35
+ Args:
36
+ input: torch with shape: batch (b) x sequence(T) x feature (h)
37
+
38
+ Returns:
39
+ batch (b) x channel (c) x sequence(T) x feature (h)
40
+ """
41
+ f1 = F.relu(self.linear(input))
42
+ p1 = self.project(f1)
43
+ x = torch.unsqueeze(p1, 1)
44
+ # x: batch (b) x channel (c) x sequence(T) x feature (h)
45
+ x_per = x.permute(0, 3, 2, 1)
46
+ # x_per: batch (b) x feature (h) x sequence(T) x channel (c)
47
+ y = F.pad(x_per, [0, 0, self.lorder - 1, 0])
48
+
49
+ out = x_per + self.conv1(y)
50
+ out1 = out.permute(0, 3, 2, 1)
51
+ # out1: batch (b) x channel (c) x sequence(T) x feature (h)
52
+ return input + out1.squeeze()
53
+
54
+ def to_kaldi_nnet(self):
55
+ re_str = ''
56
+ re_str += '<UniDeepFsmn> %d %d\n'\
57
+ % (self.output_dim, self.input_dim)
58
+ re_str += '<LearnRateCoef> %d <HidSize> %d <LOrder> %d <LStride> %d <MaxNorm> 0\n'\
59
+ % (1, self.hidden_size, self.lorder, 1)
60
+
61
+ lfiters = self.state_dict()['conv1.weight']
62
+ x = np.flipud(lfiters.squeeze().numpy().T)
63
+ re_str += to_kaldi_matrix(x)
64
+ proj_weights = self.state_dict()['project.weight']
65
+ x = proj_weights.squeeze().numpy()
66
+ re_str += to_kaldi_matrix(x)
67
+ linear_weights = self.state_dict()['linear.weight']
68
+ x = linear_weights.squeeze().numpy()
69
+ re_str += to_kaldi_matrix(x)
70
+ linear_bias = self.state_dict()['linear.bias']
71
+ x = linear_bias.squeeze().numpy()
72
+ re_str += to_kaldi_matrix(x)
73
+ return re_str
74
+
75
+ def load_kaldi_nnet(self, instr):
76
+ output = expect_token_number(
77
+ instr,
78
+ '<LearnRateCoef>',
79
+ )
80
+ if output is None:
81
+ raise Exception('UniDeepFsmn format error')
82
+ instr, lr = output
83
+
84
+ output = expect_token_number(
85
+ instr,
86
+ '<HidSize>',
87
+ )
88
+ if output is None:
89
+ raise Exception('UniDeepFsmn format error')
90
+ instr, hiddensize = output
91
+ self.hidden_size = int(hiddensize)
92
+
93
+ output = expect_token_number(
94
+ instr,
95
+ '<LOrder>',
96
+ )
97
+ if output is None:
98
+ raise Exception('UniDeepFsmn format error')
99
+ instr, lorder = output
100
+ self.lorder = int(lorder)
101
+
102
+ output = expect_token_number(
103
+ instr,
104
+ '<LStride>',
105
+ )
106
+ if output is None:
107
+ raise Exception('UniDeepFsmn format error')
108
+ instr, lstride = output
109
+ self.lstride = lstride
110
+
111
+ output = expect_token_number(
112
+ instr,
113
+ '<MaxNorm>',
114
+ )
115
+ if output is None:
116
+ raise Exception('UniDeepFsmn format error')
117
+
118
+ output = expect_kaldi_matrix(instr)
119
+ if output is None:
120
+ raise Exception('Fsmn format error')
121
+ instr, mat = output
122
+ mat1 = np.fliplr(mat.T).copy()
123
+ self.conv1 = nn.Conv2d(
124
+ self.output_dim,
125
+ self.output_dim, (self.lorder, 1), (1, 1),
126
+ groups=self.output_dim,
127
+ bias=False)
128
+ mat_th = torch.from_numpy(mat1).type(torch.FloatTensor)
129
+ mat_th = mat_th.unsqueeze(1)
130
+ mat_th = mat_th.unsqueeze(3)
131
+ self.conv1.weight = torch.nn.Parameter(mat_th)
132
+
133
+ output = expect_kaldi_matrix(instr)
134
+ if output is None:
135
+ raise Exception('UniDeepFsmn format error')
136
+ instr, mat = output
137
+
138
+ self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False)
139
+ self.linear = nn.Linear(self.input_dim, self.hidden_size)
140
+ self.project.weight = torch.nn.Parameter(
141
+ torch.from_numpy(mat).type(torch.FloatTensor))
142
+
143
+ output = expect_kaldi_matrix(instr)
144
+ if output is None:
145
+ raise Exception('UniDeepFsmn format error')
146
+ instr, mat = output
147
+ self.linear.weight = torch.nn.Parameter(
148
+ torch.from_numpy(mat).type(torch.FloatTensor))
149
+
150
+ output = expect_kaldi_matrix(instr)
151
+ if output is None:
152
+ raise Exception('UniDeepFsmn format error')
153
+ instr, mat = output
154
+ self.linear.bias = torch.nn.Parameter(
155
+ torch.from_numpy(mat).type(torch.FloatTensor))
156
+ return instr
runtime/Lib/site_packages/models/audio/ans/se_module_complex.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ import torch
3
+ from torch import nn
4
+
5
+
6
+ class SELayer(nn.Module):
7
+
8
+ def __init__(self, channel, reduction=16):
9
+ super(SELayer, self).__init__()
10
+ self.avg_pool = nn.AdaptiveAvgPool2d(1)
11
+ self.fc_r = nn.Sequential(
12
+ nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True),
13
+ nn.Linear(channel // reduction, channel), nn.Sigmoid())
14
+ self.fc_i = nn.Sequential(
15
+ nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True),
16
+ nn.Linear(channel // reduction, channel), nn.Sigmoid())
17
+
18
+ def forward(self, x):
19
+ b, c, _, _, _ = x.size()
20
+ x_r = self.avg_pool(x[:, :, :, :, 0]).view(b, c)
21
+ x_i = self.avg_pool(x[:, :, :, :, 1]).view(b, c)
22
+ y_r = self.fc_r(x_r).view(b, c, 1, 1, 1) - self.fc_i(x_i).view(
23
+ b, c, 1, 1, 1)
24
+ y_i = self.fc_r(x_i).view(b, c, 1, 1, 1) + self.fc_i(x_r).view(
25
+ b, c, 1, 1, 1)
26
+ y = torch.cat([y_r, y_i], 4)
27
+ return x * y
runtime/Lib/site_packages/models/audio/ans/unet.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ #
3
+ # The implementation here is modified based on
4
+ # Jongho Choi(sweetcocoa@snu.ac.kr / Seoul National Univ., ESTsoft )
5
+ # and publicly available at
6
+ # https://github.com/sweetcocoa/DeepComplexUNetPyTorch
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+
11
+ from . import complex_nn
12
+ from .se_module_complex import SELayer
13
+
14
+
15
+ class Encoder(nn.Module):
16
+
17
+ def __init__(self,
18
+ in_channels,
19
+ out_channels,
20
+ kernel_size,
21
+ stride,
22
+ padding=None,
23
+ complex=False,
24
+ padding_mode='zeros'):
25
+ super().__init__()
26
+ if padding is None:
27
+ padding = [(i - 1) // 2 for i in kernel_size] # 'SAME' padding
28
+
29
+ if complex:
30
+ conv = complex_nn.ComplexConv2d
31
+ bn = complex_nn.ComplexBatchNorm2d
32
+ else:
33
+ conv = nn.Conv2d
34
+ bn = nn.BatchNorm2d
35
+
36
+ self.conv = conv(
37
+ in_channels,
38
+ out_channels,
39
+ kernel_size=kernel_size,
40
+ stride=stride,
41
+ padding=padding,
42
+ padding_mode=padding_mode)
43
+ self.bn = bn(out_channels)
44
+ self.relu = nn.LeakyReLU(inplace=True)
45
+
46
+ def forward(self, x):
47
+ x = self.conv(x)
48
+ x = self.bn(x)
49
+ x = self.relu(x)
50
+ return x
51
+
52
+
53
+ class Decoder(nn.Module):
54
+
55
+ def __init__(self,
56
+ in_channels,
57
+ out_channels,
58
+ kernel_size,
59
+ stride,
60
+ padding=(0, 0),
61
+ complex=False):
62
+ super().__init__()
63
+ if complex:
64
+ tconv = complex_nn.ComplexConvTranspose2d
65
+ bn = complex_nn.ComplexBatchNorm2d
66
+ else:
67
+ tconv = nn.ConvTranspose2d
68
+ bn = nn.BatchNorm2d
69
+
70
+ self.transconv = tconv(
71
+ in_channels,
72
+ out_channels,
73
+ kernel_size=kernel_size,
74
+ stride=stride,
75
+ padding=padding)
76
+ self.bn = bn(out_channels)
77
+ self.relu = nn.LeakyReLU(inplace=True)
78
+
79
+ def forward(self, x):
80
+ x = self.transconv(x)
81
+ x = self.bn(x)
82
+ x = self.relu(x)
83
+ return x
84
+
85
+
86
+ class UNet(nn.Module):
87
+
88
+ def __init__(self,
89
+ input_channels=1,
90
+ complex=False,
91
+ model_complexity=45,
92
+ model_depth=20,
93
+ padding_mode='zeros'):
94
+ super().__init__()
95
+
96
+ if complex:
97
+ model_complexity = int(model_complexity // 1.414)
98
+
99
+ self.set_size(
100
+ model_complexity=model_complexity,
101
+ input_channels=input_channels,
102
+ model_depth=model_depth)
103
+ self.encoders = []
104
+ self.model_length = model_depth // 2
105
+ self.fsmn = complex_nn.ComplexUniDeepFsmn(128, 128, 128)
106
+ self.se_layers_enc = []
107
+ self.fsmn_enc = []
108
+ for i in range(self.model_length):
109
+ fsmn_enc = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128)
110
+ self.add_module('fsmn_enc{}'.format(i), fsmn_enc)
111
+ self.fsmn_enc.append(fsmn_enc)
112
+ module = Encoder(
113
+ self.enc_channels[i],
114
+ self.enc_channels[i + 1],
115
+ kernel_size=self.enc_kernel_sizes[i],
116
+ stride=self.enc_strides[i],
117
+ padding=self.enc_paddings[i],
118
+ complex=complex,
119
+ padding_mode=padding_mode)
120
+ self.add_module('encoder{}'.format(i), module)
121
+ self.encoders.append(module)
122
+ se_layer_enc = SELayer(self.enc_channels[i + 1], 8)
123
+ self.add_module('se_layer_enc{}'.format(i), se_layer_enc)
124
+ self.se_layers_enc.append(se_layer_enc)
125
+ self.decoders = []
126
+ self.fsmn_dec = []
127
+ self.se_layers_dec = []
128
+ for i in range(self.model_length):
129
+ fsmn_dec = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128)
130
+ self.add_module('fsmn_dec{}'.format(i), fsmn_dec)
131
+ self.fsmn_dec.append(fsmn_dec)
132
+ module = Decoder(
133
+ self.dec_channels[i] * 2,
134
+ self.dec_channels[i + 1],
135
+ kernel_size=self.dec_kernel_sizes[i],
136
+ stride=self.dec_strides[i],
137
+ padding=self.dec_paddings[i],
138
+ complex=complex)
139
+ self.add_module('decoder{}'.format(i), module)
140
+ self.decoders.append(module)
141
+ if i < self.model_length - 1:
142
+ se_layer_dec = SELayer(self.dec_channels[i + 1], 8)
143
+ self.add_module('se_layer_dec{}'.format(i), se_layer_dec)
144
+ self.se_layers_dec.append(se_layer_dec)
145
+ if complex:
146
+ conv = complex_nn.ComplexConv2d
147
+ else:
148
+ conv = nn.Conv2d
149
+
150
+ linear = conv(self.dec_channels[-1], 1, 1)
151
+
152
+ self.add_module('linear', linear)
153
+ self.complex = complex
154
+ self.padding_mode = padding_mode
155
+
156
+ self.decoders = nn.ModuleList(self.decoders)
157
+ self.encoders = nn.ModuleList(self.encoders)
158
+ self.se_layers_enc = nn.ModuleList(self.se_layers_enc)
159
+ self.se_layers_dec = nn.ModuleList(self.se_layers_dec)
160
+ self.fsmn_enc = nn.ModuleList(self.fsmn_enc)
161
+ self.fsmn_dec = nn.ModuleList(self.fsmn_dec)
162
+
163
+ def forward(self, inputs):
164
+ x = inputs
165
+ # go down
166
+ xs = []
167
+ xs_se = []
168
+ xs_se.append(x)
169
+ for i, encoder in enumerate(self.encoders):
170
+ xs.append(x)
171
+ if i > 0:
172
+ x = self.fsmn_enc[i](x)
173
+ x = encoder(x)
174
+ xs_se.append(self.se_layers_enc[i](x))
175
+ # xs : x0=input x1 ... x9
176
+ x = self.fsmn(x)
177
+
178
+ p = x
179
+ for i, decoder in enumerate(self.decoders):
180
+ p = decoder(p)
181
+ if i < self.model_length - 1:
182
+ p = self.fsmn_dec[i](p)
183
+ if i == self.model_length - 1:
184
+ break
185
+ if i < self.model_length - 2:
186
+ p = self.se_layers_dec[i](p)
187
+ p = torch.cat([p, xs_se[self.model_length - 1 - i]], dim=1)
188
+
189
+ # cmp_spec: [12, 1, 513, 64, 2]
190
+ cmp_spec = self.linear(p)
191
+ return cmp_spec
192
+
193
+ def set_size(self, model_complexity, model_depth=20, input_channels=1):
194
+
195
+ if model_depth == 14:
196
+ self.enc_channels = [
197
+ input_channels, 128, 128, 128, 128, 128, 128, 128
198
+ ]
199
+ self.enc_kernel_sizes = [(5, 2), (5, 2), (5, 2), (5, 2), (5, 2),
200
+ (5, 2), (2, 2)]
201
+ self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1),
202
+ (2, 1)]
203
+ self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1),
204
+ (0, 1), (0, 1)]
205
+ self.dec_channels = [64, 128, 128, 128, 128, 128, 128, 1]
206
+ self.dec_kernel_sizes = [(2, 2), (5, 2), (5, 2), (5, 2), (6, 2),
207
+ (5, 2), (5, 2)]
208
+ self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1),
209
+ (2, 1)]
210
+ self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1),
211
+ (0, 1), (0, 1)]
212
+
213
+ elif model_depth == 10:
214
+ self.enc_channels = [
215
+ input_channels,
216
+ 16,
217
+ 32,
218
+ 64,
219
+ 128,
220
+ 256,
221
+ ]
222
+ self.enc_kernel_sizes = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3)]
223
+ self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)]
224
+ self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)]
225
+ self.dec_channels = [128, 128, 64, 32, 16, 1]
226
+ self.dec_kernel_sizes = [(3, 3), (3, 3), (3, 3), (4, 3), (3, 3)]
227
+ self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)]
228
+ self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)]
229
+
230
+ elif model_depth == 20:
231
+ self.enc_channels = [
232
+ input_channels, model_complexity, model_complexity,
233
+ model_complexity * 2, model_complexity * 2,
234
+ model_complexity * 2, model_complexity * 2,
235
+ model_complexity * 2, model_complexity * 2,
236
+ model_complexity * 2, 128
237
+ ]
238
+
239
+ self.enc_kernel_sizes = [(7, 1), (1, 7), (6, 4), (7, 5), (5, 3),
240
+ (5, 3), (5, 3), (5, 3), (5, 3), (5, 3)]
241
+
242
+ self.enc_strides = [(1, 1), (1, 1), (2, 2), (2, 1), (2, 2), (2, 1),
243
+ (2, 2), (2, 1), (2, 2), (2, 1)]
244
+
245
+ self.enc_paddings = [
246
+ (3, 0),
247
+ (0, 3),
248
+ None, # (0, 2),
249
+ None,
250
+ None, # (3,1),
251
+ None, # (3,1),
252
+ None, # (1,2),
253
+ None,
254
+ None,
255
+ None
256
+ ]
257
+
258
+ self.dec_channels = [
259
+ 0, model_complexity * 2, model_complexity * 2,
260
+ model_complexity * 2, model_complexity * 2,
261
+ model_complexity * 2, model_complexity * 2,
262
+ model_complexity * 2, model_complexity * 2,
263
+ model_complexity * 2, model_complexity * 2,
264
+ model_complexity * 2
265
+ ]
266
+
267
+ self.dec_kernel_sizes = [(4, 3), (4, 2), (4, 3), (4, 2), (4, 3),
268
+ (4, 2), (6, 3), (7, 4), (1, 7), (7, 1)]
269
+
270
+ self.dec_strides = [(2, 1), (2, 2), (2, 1), (2, 2), (2, 1), (2, 2),
271
+ (2, 1), (2, 2), (1, 1), (1, 1)]
272
+
273
+ self.dec_paddings = [(1, 1), (1, 0), (1, 1), (1, 0), (1, 1),
274
+ (1, 0), (2, 1), (2, 1), (0, 3), (3, 0)]
275
+ else:
276
+ raise ValueError('Unknown model depth : {}'.format(model_depth))
runtime/Lib/site_packages/models/audio/asr/__init__.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.import_utils import LazyImportModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .generic_automatic_speech_recognition import GenericAutomaticSpeechRecognition
8
+
9
+ else:
10
+ _import_structure = {
11
+ 'generic_automatic_speech_recognition':
12
+ ['GenericAutomaticSpeechRecognition'],
13
+ }
14
+
15
+ import sys
16
+
17
+ sys.modules[__name__] = LazyImportModule(
18
+ __name__,
19
+ globals()['__file__'],
20
+ _import_structure,
21
+ module_spec=__spec__,
22
+ extra_objects={},
23
+ )
runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ from typing import Any, Dict
5
+
6
+ from modelscope.metainfo import Models
7
+ from modelscope.models.base import Model
8
+ from modelscope.models.builder import MODELS
9
+ from modelscope.utils.constant import Frameworks, Tasks
10
+
11
+ __all__ = ['GenericAutomaticSpeechRecognition']
12
+
13
+
14
+ @MODELS.register_module(
15
+ Tasks.auto_speech_recognition, module_name=Models.generic_asr)
16
+ @MODELS.register_module(
17
+ Tasks.voice_activity_detection, module_name=Models.generic_asr)
18
+ @MODELS.register_module(
19
+ Tasks.speech_separation, module_name=Models.generic_asr)
20
+ @MODELS.register_module(
21
+ Tasks.language_score_prediction, module_name=Models.generic_asr)
22
+ @MODELS.register_module(Tasks.speech_timestamp, module_name=Models.generic_asr)
23
+ class GenericAutomaticSpeechRecognition(Model):
24
+
25
+ def __init__(self, model_dir: str, am_model_name: str,
26
+ model_config: Dict[str, Any], *args, **kwargs):
27
+ """initialize the info of model.
28
+
29
+ Args:
30
+ model_dir (str): the model path.
31
+ am_model_name (str): the am model name from configuration.json
32
+ model_config (Dict[str, Any]): the detail config about model from configuration.json
33
+ """
34
+ super().__init__(model_dir, am_model_name, model_config, *args,
35
+ **kwargs)
36
+ self.model_cfg = {
37
+ # the recognition model dir path
38
+ 'model_workspace': model_dir,
39
+ # the am model name
40
+ 'am_model': am_model_name,
41
+ # the am model file path
42
+ 'am_model_path': os.path.join(model_dir, am_model_name),
43
+ # the recognition model config dict
44
+ 'model_config': model_config
45
+ }
46
+
47
+ def forward(self) -> Dict[str, Any]:
48
+ """preload model and return the info of the model
49
+ """
50
+
51
+ return self.model_cfg
runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ from typing import Any, Dict
5
+
6
+ import json
7
+ import wenetruntime as wenet
8
+
9
+ from modelscope.metainfo import Models
10
+ from modelscope.models.base import Model
11
+ from modelscope.models.builder import MODELS
12
+ from modelscope.utils.constant import Tasks
13
+
14
+ __all__ = ['WeNetAutomaticSpeechRecognition']
15
+
16
+
17
+ @MODELS.register_module(
18
+ Tasks.auto_speech_recognition, module_name=Models.wenet_asr)
19
+ class WeNetAutomaticSpeechRecognition(Model):
20
+
21
+ def __init__(self, model_dir: str, am_model_name: str,
22
+ model_config: Dict[str, Any], *args, **kwargs):
23
+ """initialize the info of model.
24
+
25
+ Args:
26
+ model_dir (str): the model path.
27
+ """
28
+ super().__init__(model_dir, am_model_name, model_config, *args,
29
+ **kwargs)
30
+ self.decoder = wenet.Decoder(model_dir, lang='chs')
31
+
32
+ def forward(self, inputs: Dict[str, Any]) -> str:
33
+ if inputs['audio_format'] == 'wav':
34
+ rst = self.decoder.decode_wav(inputs['audio'])
35
+ else:
36
+ rst = self.decoder.decode(inputs['audio'])
37
+ text = json.loads(rst)['nbest'][0]['sentence']
38
+ return {'text': text}
runtime/Lib/site_packages/models/audio/itn/__init__.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.import_utils import LazyImportModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .generic_inverse_text_processing import GenericInverseTextProcessing
8
+
9
+ else:
10
+ _import_structure = {
11
+ 'generic_inverse_text_processing': ['GenericInverseTextProcessing'],
12
+ }
13
+
14
+ import sys
15
+
16
+ sys.modules[__name__] = LazyImportModule(
17
+ __name__,
18
+ globals()['__file__'],
19
+ _import_structure,
20
+ module_spec=__spec__,
21
+ extra_objects={},
22
+ )
runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ from typing import Any, Dict
5
+
6
+ from modelscope.metainfo import Models
7
+ from modelscope.models.base import Model
8
+ from modelscope.models.builder import MODELS
9
+ from modelscope.utils.constant import Frameworks, Tasks
10
+
11
+
12
+ @MODELS.register_module(
13
+ Tasks.inverse_text_processing, module_name=Models.generic_itn)
14
+ class GenericInverseTextProcessing(Model):
15
+
16
+ def __init__(self, model_dir: str, itn_model_name: str,
17
+ model_config: Dict[str, Any], *args, **kwargs):
18
+ """initialize the info of model.
19
+
20
+ Args:
21
+ model_dir (str): the model path.
22
+ itn_model_name (str): the itn model name from configuration.json
23
+ model_config (Dict[str, Any]): the detail config about model from configuration.json
24
+ """
25
+ super().__init__(model_dir, itn_model_name, model_config, *args,
26
+ **kwargs)
27
+ self.model_cfg = {
28
+ # the recognition model dir path
29
+ 'model_workspace': model_dir,
30
+ # the itn model name
31
+ 'itn_model': itn_model_name,
32
+ # the am model file path
33
+ 'itn_model_path': os.path.join(model_dir, itn_model_name),
34
+ # the recognition model config dict
35
+ 'model_config': model_config
36
+ }
37
+
38
+ def forward(self) -> Dict[str, Any]:
39
+ """
40
+ just return the model config
41
+
42
+ """
43
+
44
+ return self.model_cfg
runtime/Lib/site_packages/models/audio/kws/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.import_utils import LazyImportModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .generic_key_word_spotting import GenericKeyWordSpotting
8
+ from .farfield.model import FSMNSeleNetV2Decorator
9
+ from .nearfield.model import FSMNDecorator
10
+
11
+ else:
12
+ _import_structure = {
13
+ 'generic_key_word_spotting': ['GenericKeyWordSpotting'],
14
+ 'farfield.model': ['FSMNSeleNetV2Decorator'],
15
+ 'nearfield.model': ['FSMNDecorator'],
16
+ }
17
+
18
+ import sys
19
+
20
+ sys.modules[__name__] = LazyImportModule(
21
+ __name__,
22
+ globals()['__file__'],
23
+ _import_structure,
24
+ module_spec=__spec__,
25
+ extra_objects={},
26
+ )
runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py ADDED
@@ -0,0 +1,497 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+ from .model_def import (HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32,
9
+ printNeonMatrix, printNeonVector)
10
+
11
+ DEBUG = False
12
+
13
+
14
+ def to_kaldi_matrix(np_mat):
15
+ """ function that transform as str numpy mat to standard kaldi str matrix
16
+
17
+ Args:
18
+ np_mat: numpy mat
19
+
20
+ Returns: str
21
+ """
22
+ np.set_printoptions(threshold=np.inf, linewidth=np.nan)
23
+ out_str = str(np_mat)
24
+ out_str = out_str.replace('[', '')
25
+ out_str = out_str.replace(']', '')
26
+ return '[ %s ]\n' % out_str
27
+
28
+
29
+ def print_tensor(torch_tensor):
30
+ """ print torch tensor for debug
31
+
32
+ Args:
33
+ torch_tensor: a tensor
34
+ """
35
+ re_str = ''
36
+ x = torch_tensor.detach().squeeze().numpy()
37
+ re_str += to_kaldi_matrix(x)
38
+ re_str += '<!EndOfComponent>\n'
39
+ print(re_str)
40
+
41
+
42
+ class LinearTransform(nn.Module):
43
+
44
+ def __init__(self, input_dim, output_dim):
45
+ super(LinearTransform, self).__init__()
46
+ self.input_dim = input_dim
47
+ self.output_dim = output_dim
48
+ self.linear = nn.Linear(input_dim, output_dim, bias=False)
49
+
50
+ self.debug = False
51
+ self.dataout = None
52
+
53
+ def forward(self, input):
54
+ output = self.linear(input)
55
+
56
+ if self.debug:
57
+ self.dataout = output
58
+
59
+ return output
60
+
61
+ def print_model(self):
62
+ printNeonMatrix(self.linear.weight)
63
+
64
+ def to_kaldi_nnet(self):
65
+ re_str = ''
66
+ re_str += '<LinearTransform> %d %d\n' % (self.output_dim,
67
+ self.input_dim)
68
+ re_str += '<LearnRateCoef> 1\n'
69
+
70
+ linear_weights = self.state_dict()['linear.weight']
71
+ x = linear_weights.squeeze().numpy()
72
+ re_str += to_kaldi_matrix(x)
73
+ re_str += '<!EndOfComponent>\n'
74
+
75
+ return re_str
76
+
77
+
78
+ class AffineTransform(nn.Module):
79
+
80
+ def __init__(self, input_dim, output_dim):
81
+ super(AffineTransform, self).__init__()
82
+ self.input_dim = input_dim
83
+ self.output_dim = output_dim
84
+
85
+ self.linear = nn.Linear(input_dim, output_dim)
86
+
87
+ self.debug = False
88
+ self.dataout = None
89
+
90
+ def forward(self, input):
91
+ output = self.linear(input)
92
+
93
+ if self.debug:
94
+ self.dataout = output
95
+
96
+ return output
97
+
98
+ def print_model(self):
99
+ printNeonMatrix(self.linear.weight)
100
+ printNeonVector(self.linear.bias)
101
+
102
+ def to_kaldi_nnet(self):
103
+ re_str = ''
104
+ re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
105
+ self.input_dim)
106
+ re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
107
+
108
+ linear_weights = self.state_dict()['linear.weight']
109
+ x = linear_weights.squeeze().numpy()
110
+ re_str += to_kaldi_matrix(x)
111
+
112
+ linear_bias = self.state_dict()['linear.bias']
113
+ x = linear_bias.squeeze().numpy()
114
+ re_str += to_kaldi_matrix(x)
115
+ re_str += '<!EndOfComponent>\n'
116
+
117
+ return re_str
118
+
119
+
120
+ class Fsmn(nn.Module):
121
+ """
122
+ FSMN implementation.
123
+ """
124
+
125
+ def __init__(self,
126
+ input_dim,
127
+ output_dim,
128
+ lorder=None,
129
+ rorder=None,
130
+ lstride=None,
131
+ rstride=None):
132
+ super(Fsmn, self).__init__()
133
+
134
+ self.dim = input_dim
135
+
136
+ if lorder is None:
137
+ return
138
+
139
+ self.lorder = lorder
140
+ self.rorder = rorder
141
+ self.lstride = lstride
142
+ self.rstride = rstride
143
+
144
+ self.conv_left = nn.Conv2d(
145
+ self.dim,
146
+ self.dim, (lorder, 1),
147
+ dilation=(lstride, 1),
148
+ groups=self.dim,
149
+ bias=False)
150
+
151
+ if rorder > 0:
152
+ self.conv_right = nn.Conv2d(
153
+ self.dim,
154
+ self.dim, (rorder, 1),
155
+ dilation=(rstride, 1),
156
+ groups=self.dim,
157
+ bias=False)
158
+ else:
159
+ self.conv_right = None
160
+
161
+ self.debug = False
162
+ self.dataout = None
163
+
164
+ def forward(self, input):
165
+ x = torch.unsqueeze(input, 1)
166
+ x_per = x.permute(0, 3, 2, 1)
167
+
168
+ y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
169
+
170
+ if self.conv_right is not None:
171
+ y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
172
+ y_right = y_right[:, :, self.rstride:, :]
173
+ out = x_per + self.conv_left(y_left) + self.conv_right(y_right)
174
+ else:
175
+ out = x_per + self.conv_left(y_left)
176
+
177
+ out1 = out.permute(0, 3, 2, 1)
178
+ output = out1.squeeze(1)
179
+
180
+ if self.debug:
181
+ self.dataout = output
182
+
183
+ return output
184
+
185
+ def print_model(self):
186
+ tmpw = self.conv_left.weight
187
+ tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
188
+ for j in range(tmpw.shape[0]):
189
+ tmpwm[:, j] = tmpw[j, 0, :, 0]
190
+
191
+ printNeonMatrix(tmpwm)
192
+
193
+ if self.conv_right is not None:
194
+ tmpw = self.conv_right.weight
195
+ tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
196
+ for j in range(tmpw.shape[0]):
197
+ tmpwm[:, j] = tmpw[j, 0, :, 0]
198
+
199
+ printNeonMatrix(tmpwm)
200
+
201
+ def to_kaldi_nnet(self):
202
+ re_str = ''
203
+ re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)
204
+ re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (
205
+ 1, self.lorder, self.rorder, self.lstride, self.rstride)
206
+
207
+ lfiters = self.state_dict()['conv_left.weight']
208
+ x = np.flipud(lfiters.squeeze().numpy().T)
209
+ re_str += to_kaldi_matrix(x)
210
+
211
+ if self.conv_right is not None:
212
+ rfiters = self.state_dict()['conv_right.weight']
213
+ x = (rfiters.squeeze().numpy().T)
214
+ re_str += to_kaldi_matrix(x)
215
+ re_str += '<!EndOfComponent>\n'
216
+
217
+ return re_str
218
+
219
+
220
+ class RectifiedLinear(nn.Module):
221
+
222
+ def __init__(self, input_dim, output_dim):
223
+ super(RectifiedLinear, self).__init__()
224
+ self.dim = input_dim
225
+ self.relu = nn.ReLU()
226
+
227
+ def forward(self, input):
228
+ return self.relu(input)
229
+
230
+ def to_kaldi_nnet(self):
231
+ re_str = ''
232
+ re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
233
+ re_str += '<!EndOfComponent>\n'
234
+ return re_str
235
+
236
+
237
+ class FSMNNet(nn.Module):
238
+ """
239
+ FSMN net for keyword spotting
240
+ """
241
+
242
+ def __init__(self,
243
+ input_dim=200,
244
+ linear_dim=128,
245
+ proj_dim=128,
246
+ lorder=10,
247
+ rorder=1,
248
+ num_syn=5,
249
+ fsmn_layers=4):
250
+ """
251
+ Args:
252
+ input_dim: input dimension
253
+ linear_dim: fsmn input dimension
254
+ proj_dim: fsmn projection dimension
255
+ lorder: fsmn left order
256
+ rorder: fsmn right order
257
+ num_syn: output dimension
258
+ fsmn_layers: no. of sequential fsmn layers
259
+ """
260
+ super(FSMNNet, self).__init__()
261
+
262
+ self.input_dim = input_dim
263
+ self.linear_dim = linear_dim
264
+ self.proj_dim = proj_dim
265
+ self.lorder = lorder
266
+ self.rorder = rorder
267
+ self.num_syn = num_syn
268
+ self.fsmn_layers = fsmn_layers
269
+
270
+ self.linear1 = AffineTransform(input_dim, linear_dim)
271
+ self.relu = RectifiedLinear(linear_dim, linear_dim)
272
+
273
+ self.fsmn = self._build_repeats(linear_dim, proj_dim, lorder, rorder,
274
+ fsmn_layers)
275
+
276
+ self.linear2 = AffineTransform(linear_dim, num_syn)
277
+
278
+ @staticmethod
279
+ def _build_repeats(linear_dim=136,
280
+ proj_dim=68,
281
+ lorder=3,
282
+ rorder=2,
283
+ fsmn_layers=5):
284
+ repeats = [
285
+ nn.Sequential(
286
+ LinearTransform(linear_dim, proj_dim),
287
+ Fsmn(proj_dim, proj_dim, lorder, rorder, 1, 1),
288
+ AffineTransform(proj_dim, linear_dim),
289
+ RectifiedLinear(linear_dim, linear_dim))
290
+ for i in range(fsmn_layers)
291
+ ]
292
+
293
+ return nn.Sequential(*repeats)
294
+
295
+ def forward(self, input):
296
+ x1 = self.linear1(input)
297
+ x2 = self.relu(x1)
298
+ x3 = self.fsmn(x2)
299
+ x4 = self.linear2(x3)
300
+ return x4
301
+
302
+ def print_model(self):
303
+ self.linear1.print_model()
304
+
305
+ for layer in self.fsmn:
306
+ layer[0].print_model()
307
+ layer[1].print_model()
308
+ layer[2].print_model()
309
+
310
+ self.linear2.print_model()
311
+
312
+ def print_header(self):
313
+ #
314
+ # write total header
315
+ #
316
+ header = [0.0] * HEADER_BLOCK_SIZE * 4
317
+ # numins
318
+ header[0] = 0.0
319
+ # numouts
320
+ header[1] = 0.0
321
+ # dimins
322
+ header[2] = self.input_dim
323
+ # dimouts
324
+ header[3] = self.num_syn
325
+ # numlayers
326
+ header[4] = 3
327
+
328
+ #
329
+ # write each layer's header
330
+ #
331
+ hidx = 1
332
+
333
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
334
+ LayerType.LAYER_DENSE.value)
335
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
336
+ header[HEADER_BLOCK_SIZE * hidx + 2] = self.input_dim
337
+ header[HEADER_BLOCK_SIZE * hidx + 3] = self.linear_dim
338
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
339
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
340
+ ActivationType.ACTIVATION_RELU.value)
341
+ hidx += 1
342
+
343
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
344
+ LayerType.LAYER_SEQUENTIAL_FSMN.value)
345
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
346
+ header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim
347
+ header[HEADER_BLOCK_SIZE * hidx + 3] = self.proj_dim
348
+ header[HEADER_BLOCK_SIZE * hidx + 4] = self.lorder
349
+ header[HEADER_BLOCK_SIZE * hidx + 5] = self.rorder
350
+ header[HEADER_BLOCK_SIZE * hidx + 6] = self.fsmn_layers
351
+ header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0
352
+ hidx += 1
353
+
354
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
355
+ LayerType.LAYER_DENSE.value)
356
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
357
+ header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim
358
+ header[HEADER_BLOCK_SIZE * hidx + 3] = self.num_syn
359
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
360
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
361
+ ActivationType.ACTIVATION_SOFTMAX.value)
362
+
363
+ for h in header:
364
+ print(f32ToI32(h))
365
+
366
+ def to_kaldi_nnet(self):
367
+ re_str = ''
368
+ re_str += '<Nnet>\n'
369
+ re_str += self.linear1.to_kaldi_nnet()
370
+ re_str += self.relu.to_kaldi_nnet()
371
+
372
+ for fsmn in self.fsmn:
373
+ re_str += fsmn[0].to_kaldi_nnet()
374
+ re_str += fsmn[1].to_kaldi_nnet()
375
+ re_str += fsmn[2].to_kaldi_nnet()
376
+ re_str += fsmn[3].to_kaldi_nnet()
377
+
378
+ re_str += self.linear2.to_kaldi_nnet()
379
+ re_str += '<Softmax> %d %d\n' % (self.num_syn, self.num_syn)
380
+ re_str += '<!EndOfComponent>\n'
381
+ re_str += '</Nnet>\n'
382
+
383
+ return re_str
384
+
385
+
386
+ class DFSMN(nn.Module):
387
+ """
388
+ One deep fsmn layer
389
+ """
390
+
391
+ def __init__(self,
392
+ dimproj=64,
393
+ dimlinear=128,
394
+ lorder=20,
395
+ rorder=1,
396
+ lstride=1,
397
+ rstride=1):
398
+ """
399
+ Args:
400
+ dimproj: projection dimension, input and output dimension of memory blocks
401
+ dimlinear: dimension of mapping layer
402
+ lorder: left order
403
+ rorder: right order
404
+ lstride: left stride
405
+ rstride: right stride
406
+ """
407
+ super(DFSMN, self).__init__()
408
+
409
+ self.lorder = lorder
410
+ self.rorder = rorder
411
+ self.lstride = lstride
412
+ self.rstride = rstride
413
+
414
+ self.expand = AffineTransform(dimproj, dimlinear)
415
+ self.shrink = LinearTransform(dimlinear, dimproj)
416
+
417
+ self.conv_left = nn.Conv2d(
418
+ dimproj,
419
+ dimproj, (lorder, 1),
420
+ dilation=(lstride, 1),
421
+ groups=dimproj,
422
+ bias=False)
423
+
424
+ if rorder > 0:
425
+ self.conv_right = nn.Conv2d(
426
+ dimproj,
427
+ dimproj, (rorder, 1),
428
+ dilation=(rstride, 1),
429
+ groups=dimproj,
430
+ bias=False)
431
+ else:
432
+ self.conv_right = None
433
+
434
+ def forward(self, input):
435
+ f1 = F.relu(self.expand(input))
436
+ p1 = self.shrink(f1)
437
+
438
+ x = torch.unsqueeze(p1, 1)
439
+ x_per = x.permute(0, 3, 2, 1)
440
+
441
+ y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
442
+
443
+ if self.conv_right is not None:
444
+ y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
445
+ y_right = y_right[:, :, self.rstride:, :]
446
+ out = x_per + self.conv_left(y_left) + self.conv_right(y_right)
447
+ else:
448
+ out = x_per + self.conv_left(y_left)
449
+
450
+ out1 = out.permute(0, 3, 2, 1)
451
+ output = input + out1.squeeze(1)
452
+
453
+ return output
454
+
455
+ def print_model(self):
456
+ self.expand.print_model()
457
+ self.shrink.print_model()
458
+
459
+ tmpw = self.conv_left.weight
460
+ tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
461
+ for j in range(tmpw.shape[0]):
462
+ tmpwm[:, j] = tmpw[j, 0, :, 0]
463
+
464
+ printNeonMatrix(tmpwm)
465
+
466
+ if self.conv_right is not None:
467
+ tmpw = self.conv_right.weight
468
+ tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0])
469
+ for j in range(tmpw.shape[0]):
470
+ tmpwm[:, j] = tmpw[j, 0, :, 0]
471
+
472
+ printNeonMatrix(tmpwm)
473
+
474
+
475
+ def build_dfsmn_repeats(linear_dim=128,
476
+ proj_dim=64,
477
+ lorder=20,
478
+ rorder=1,
479
+ fsmn_layers=6):
480
+ """
481
+ build stacked dfsmn layers
482
+ Args:
483
+ linear_dim:
484
+ proj_dim:
485
+ lorder:
486
+ rorder:
487
+ fsmn_layers:
488
+
489
+ Returns:
490
+
491
+ """
492
+ repeats = [
493
+ nn.Sequential(DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1))
494
+ for i in range(fsmn_layers)
495
+ ]
496
+
497
+ return nn.Sequential(*repeats)
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+ from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear
8
+ from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32
9
+
10
+
11
+ class FSMNUnit(nn.Module):
12
+ """ A multi-channel fsmn unit
13
+
14
+ """
15
+
16
+ def __init__(self, dimlinear=128, dimproj=64, lorder=20, rorder=1):
17
+ """
18
+ Args:
19
+ dimlinear: input / output dimension
20
+ dimproj: fsmn input / output dimension
21
+ lorder: left order
22
+ rorder: right order
23
+ """
24
+ super(FSMNUnit, self).__init__()
25
+
26
+ self.shrink = LinearTransform(dimlinear, dimproj)
27
+ self.fsmn = Fsmn(dimproj, dimproj, lorder, rorder, 1, 1)
28
+ self.expand = AffineTransform(dimproj, dimlinear)
29
+
30
+ self.debug = False
31
+ self.dataout = None
32
+
33
+ '''
34
+ batch, time, channel, feature
35
+ '''
36
+
37
+ def forward(self, input):
38
+ if torch.cuda.is_available():
39
+ out = torch.zeros(input.shape).cuda()
40
+ else:
41
+ out = torch.zeros(input.shape)
42
+
43
+ for n in range(input.shape[2]):
44
+ out1 = self.shrink(input[:, :, n, :])
45
+ out2 = self.fsmn(out1)
46
+ out[:, :, n, :] = F.relu(self.expand(out2))
47
+
48
+ if self.debug:
49
+ self.dataout = out
50
+
51
+ return out
52
+
53
+ def print_model(self):
54
+ self.shrink.print_model()
55
+ self.fsmn.print_model()
56
+ self.expand.print_model()
57
+
58
+ def to_kaldi_nnet(self):
59
+ re_str = self.shrink.to_kaldi_nnet()
60
+ re_str += self.fsmn.to_kaldi_nnet()
61
+ re_str += self.expand.to_kaldi_nnet()
62
+
63
+ relu = RectifiedLinear(self.expand.linear.out_features,
64
+ self.expand.linear.out_features)
65
+ re_str += relu.to_kaldi_nnet()
66
+
67
+ return re_str
68
+
69
+
70
+ class FSMNSeleNetV2(nn.Module):
71
+ """ FSMN model with channel selection.
72
+ """
73
+
74
+ def __init__(self,
75
+ input_dim=120,
76
+ linear_dim=128,
77
+ proj_dim=64,
78
+ lorder=20,
79
+ rorder=1,
80
+ num_syn=5,
81
+ fsmn_layers=5,
82
+ sele_layer=0):
83
+ """
84
+ Args:
85
+ input_dim: input dimension
86
+ linear_dim: fsmn input dimension
87
+ proj_dim: fsmn projection dimension
88
+ lorder: fsmn left order
89
+ rorder: fsmn right order
90
+ num_syn: output dimension
91
+ fsmn_layers: no. of fsmn units
92
+ sele_layer: channel selection layer index
93
+ """
94
+ super(FSMNSeleNetV2, self).__init__()
95
+
96
+ self.sele_layer = sele_layer
97
+
98
+ self.featmap = AffineTransform(input_dim, linear_dim)
99
+
100
+ self.mem = []
101
+ for i in range(fsmn_layers):
102
+ unit = FSMNUnit(linear_dim, proj_dim, lorder, rorder)
103
+ self.mem.append(unit)
104
+ self.add_module('mem_{:d}'.format(i), unit)
105
+
106
+ self.decision = AffineTransform(linear_dim, num_syn)
107
+
108
+ def forward(self, input):
109
+ # multi-channel feature mapping
110
+ if torch.cuda.is_available():
111
+ x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
112
+ self.featmap.linear.out_features).cuda()
113
+ else:
114
+ x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
115
+ self.featmap.linear.out_features)
116
+
117
+ for n in range(input.shape[2]):
118
+ x[:, :, n, :] = F.relu(self.featmap(input[:, :, n, :]))
119
+
120
+ for i, unit in enumerate(self.mem):
121
+ y = unit(x)
122
+
123
+ # perform channel selection
124
+ if i == self.sele_layer:
125
+ pool = nn.MaxPool2d((y.shape[2], 1), stride=(y.shape[2], 1))
126
+ y = pool(y)
127
+
128
+ x = y
129
+
130
+ # remove channel dimension
131
+ y = torch.squeeze(y, -2)
132
+ z = self.decision(y)
133
+
134
+ return z
135
+
136
+ def print_model(self):
137
+ self.featmap.print_model()
138
+
139
+ for unit in self.mem:
140
+ unit.print_model()
141
+
142
+ self.decision.print_model()
143
+
144
+ def print_header(self):
145
+ '''
146
+ get FSMN params
147
+ '''
148
+ input_dim = self.featmap.linear.in_features
149
+ linear_dim = self.featmap.linear.out_features
150
+ proj_dim = self.mem[0].shrink.linear.out_features
151
+ lorder = self.mem[0].fsmn.conv_left.kernel_size[0]
152
+ rorder = 0
153
+ if self.mem[0].fsmn.conv_right is not None:
154
+ rorder = self.mem[0].fsmn.conv_right.kernel_size[0]
155
+
156
+ num_syn = self.decision.linear.out_features
157
+ fsmn_layers = len(self.mem)
158
+
159
+ # no. of output channels, 0.0 means the same as numins
160
+ # numouts = 0.0
161
+ numouts = 1.0
162
+
163
+ #
164
+ # write total header
165
+ #
166
+ header = [0.0] * HEADER_BLOCK_SIZE * 4
167
+ # numins
168
+ header[0] = 0.0
169
+ # numouts
170
+ header[1] = numouts
171
+ # dimins
172
+ header[2] = input_dim
173
+ # dimouts
174
+ header[3] = num_syn
175
+ # numlayers
176
+ header[4] = 3
177
+
178
+ #
179
+ # write each layer's header
180
+ #
181
+ hidx = 1
182
+
183
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
184
+ LayerType.LAYER_DENSE.value)
185
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
186
+ header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim
187
+ header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
188
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
189
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
190
+ ActivationType.ACTIVATION_RELU.value)
191
+ hidx += 1
192
+
193
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
194
+ LayerType.LAYER_SEQUENTIAL_FSMN.value)
195
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
196
+ header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
197
+ header[HEADER_BLOCK_SIZE * hidx + 3] = proj_dim
198
+ header[HEADER_BLOCK_SIZE * hidx + 4] = lorder
199
+ header[HEADER_BLOCK_SIZE * hidx + 5] = rorder
200
+ header[HEADER_BLOCK_SIZE * hidx + 6] = fsmn_layers
201
+ if numouts == 1.0:
202
+ header[HEADER_BLOCK_SIZE * hidx + 7] = float(self.sele_layer)
203
+ else:
204
+ header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0
205
+ hidx += 1
206
+
207
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
208
+ LayerType.LAYER_DENSE.value)
209
+ header[HEADER_BLOCK_SIZE * hidx + 1] = numouts
210
+ header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
211
+ header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn
212
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
213
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
214
+ ActivationType.ACTIVATION_SOFTMAX.value)
215
+
216
+ for h in header:
217
+ print(f32ToI32(h))
218
+
219
+ def to_kaldi_nnet(self):
220
+ re_str = '<Nnet>\n'
221
+
222
+ re_str = self.featmap.to_kaldi_nnet()
223
+
224
+ relu = RectifiedLinear(self.featmap.linear.out_features,
225
+ self.featmap.linear.out_features)
226
+ re_str += relu.to_kaldi_nnet()
227
+
228
+ for unit in self.mem:
229
+ re_str += unit.to_kaldi_nnet()
230
+
231
+ re_str += self.decision.to_kaldi_nnet()
232
+
233
+ re_str += '<Softmax> %d %d\n' % (self.decision.linear.out_features,
234
+ self.decision.linear.out_features)
235
+ re_str += '<!EndOfComponent>\n'
236
+ re_str += '</Nnet>\n'
237
+
238
+ return re_str
runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+ from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear
8
+ from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32
9
+
10
+
11
+ class DFSMNUnit(nn.Module):
12
+ """ one multi-channel deep fsmn unit
13
+ Args:
14
+ dimin: input dimension
15
+ dimexpand: feature expansion dimension
16
+ dimout: output dimension
17
+ lorder: left ofder
18
+ rorder: right order
19
+ """
20
+
21
+ def __init__(self,
22
+ dimin=64,
23
+ dimexpand=128,
24
+ dimout=64,
25
+ lorder=10,
26
+ rorder=1):
27
+ super(DFSMNUnit, self).__init__()
28
+
29
+ self.expand = AffineTransform(dimin, dimexpand)
30
+ self.shrink = LinearTransform(dimexpand, dimout)
31
+ self.fsmn = Fsmn(dimout, dimout, lorder, rorder, 1, 1)
32
+
33
+ self.debug = False
34
+ self.dataout = None
35
+
36
+ def forward(self, input):
37
+ """
38
+ Args:
39
+ input: [batch, time, feature]
40
+ """
41
+ out1 = F.relu(self.expand(input))
42
+ out2 = self.shrink(out1)
43
+ out3 = self.fsmn(out2)
44
+
45
+ # add skip connection for matched data
46
+ if input.shape[-1] == out3.shape[-1]:
47
+ out3 = input + out3
48
+ if self.debug:
49
+ self.dataout = out3
50
+ return out3
51
+
52
+ def print_model(self):
53
+ self.expand.printModel()
54
+ self.shrink.printModel()
55
+ self.fsmn.printModel()
56
+
57
+ def to_kaldi_nnet(self):
58
+ re_str = self.expand.toKaldiNNet()
59
+ relu = RectifiedLinear(self.expand.linear.out_features,
60
+ self.expand.linear.out_features)
61
+ re_str += relu.toKaldiNNet()
62
+ re_str = self.shrink.toKaldiNNet()
63
+ re_str += self.fsmn.toKaldiNNet()
64
+ return re_str
65
+
66
+
67
+ class FSMNSeleNetV3(nn.Module):
68
+ """ Deep FSMN model with channel selection performs multi-channel kws.
69
+ Zhang, Shiliang, et al. "Deep-FSMN for large vocabulary continuous speech
70
+ recognition." 2018 IEEE International Conference on Acoustics, Speech and
71
+ Signal Processing (ICASSP). IEEE, 2018.
72
+
73
+ Args:
74
+ input_dim: input dimension
75
+ linear_dim: fsmn input dimension
76
+ proj_dim: fsmn projection dimension
77
+ lorder: fsmn left order
78
+ rorder: fsmn right order
79
+ num_syn: output dimension
80
+ fsmn_layers: no. of fsmn units
81
+ """
82
+
83
+ def __init__(self,
84
+ input_dim=120,
85
+ linear_dim=128,
86
+ proj_dim=64,
87
+ lorder=10,
88
+ rorder=1,
89
+ num_syn=5,
90
+ fsmn_layers=5):
91
+ super(FSMNSeleNetV3, self).__init__()
92
+
93
+ self.mem = []
94
+ # the first unit, mapping input dim to proj dim
95
+ unit = DFSMNUnit(input_dim, linear_dim, proj_dim, lorder, rorder)
96
+ self.mem.append(unit)
97
+ self.add_module('mem_{:d}'.format(0), unit)
98
+
99
+ # deep fsmn layers with skip connection
100
+ for i in range(1, fsmn_layers):
101
+ unit = DFSMNUnit(proj_dim, linear_dim, proj_dim, lorder, rorder)
102
+ self.mem.append(unit)
103
+ self.add_module('mem_{:d}'.format(i), unit)
104
+
105
+ self.expand2 = AffineTransform(proj_dim, linear_dim)
106
+ self.decision = AffineTransform(linear_dim, num_syn)
107
+
108
+ def forward(self, input):
109
+ # multi-channel temp space, [batch, time, channel, feature]
110
+ if torch.cuda.is_available():
111
+ x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
112
+ self.expand2.linear.out_features).cuda()
113
+ else:
114
+ x = torch.zeros(input.shape[0], input.shape[1], input.shape[2],
115
+ self.expand2.linear.out_features)
116
+
117
+ for n in range(input.shape[2]):
118
+ chin = input[:, :, n, :]
119
+
120
+ for unit in self.mem:
121
+ chout = unit(chin)
122
+ chin = chout
123
+
124
+ x[:, :, n, :] = F.relu(self.expand2(chout))
125
+
126
+ # perform max pooling
127
+ pool = nn.MaxPool2d((x.shape[2], 1), stride=(x.shape[2], 1))
128
+ y = pool(x)
129
+
130
+ # remove channel dimension
131
+ y = torch.squeeze(y, -2)
132
+ z = self.decision(y)
133
+
134
+ return z
135
+
136
+ def print_model(self):
137
+ for unit in self.mem:
138
+ unit.print_model()
139
+
140
+ self.expand2.printModel()
141
+ self.decision.printModel()
142
+
143
+ def print_header(self):
144
+ """ get DFSMN params
145
+ """
146
+ input_dim = self.mem[0].expand.linear.in_features
147
+ linear_dim = self.mem[0].expand.linear.out_features
148
+ proj_dim = self.mem[0].shrink.linear.out_features
149
+ lorder = self.mem[0].fsmn.conv_left.kernel_size[0]
150
+ rorder = 0
151
+ if self.mem[0].fsmn.conv_right is not None:
152
+ rorder = self.mem[0].fsmn.conv_right.kernel_size[0]
153
+
154
+ num_syn = self.decision.linear.out_features
155
+ fsmn_layers = len(self.mem)
156
+
157
+ # no. of output channels, 0.0 means the same as numins
158
+ numouts = 1.0
159
+
160
+ #
161
+ # write total header
162
+ #
163
+ header = [0.0] * HEADER_BLOCK_SIZE * 5
164
+ # numins
165
+ header[0] = 0.0
166
+ # numouts
167
+ header[1] = numouts
168
+ # dimins
169
+ header[2] = input_dim
170
+ # dimouts
171
+ header[3] = num_syn
172
+ # numlayers
173
+ header[4] = 4
174
+
175
+ #
176
+ # write each layer's header
177
+ #
178
+ hidx = 1
179
+
180
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
181
+ LayerType.LAYER_DFSMN.value)
182
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
183
+ header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim
184
+ header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
185
+ header[HEADER_BLOCK_SIZE * hidx + 4] = proj_dim
186
+ header[HEADER_BLOCK_SIZE * hidx + 5] = lorder
187
+ header[HEADER_BLOCK_SIZE * hidx + 6] = rorder
188
+ header[HEADER_BLOCK_SIZE * hidx + 7] = fsmn_layers
189
+ hidx += 1
190
+
191
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
192
+ LayerType.LAYER_DENSE.value)
193
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
194
+ header[HEADER_BLOCK_SIZE * hidx + 2] = proj_dim
195
+ header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim
196
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
197
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
198
+ ActivationType.ACTIVATION_RELU.value)
199
+ hidx += 1
200
+
201
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
202
+ LayerType.LAYER_MAX_POOLING.value)
203
+ header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0
204
+ header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
205
+ hidx += 1
206
+
207
+ header[HEADER_BLOCK_SIZE * hidx + 0] = float(
208
+ LayerType.LAYER_DENSE.value)
209
+ header[HEADER_BLOCK_SIZE * hidx + 1] = numouts
210
+ header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim
211
+ header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn
212
+ header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0
213
+ header[HEADER_BLOCK_SIZE * hidx + 5] = float(
214
+ ActivationType.ACTIVATION_SOFTMAX.value)
215
+
216
+ for h in header:
217
+ print(f32ToI32(h))
218
+
219
+ def to_kaldi_nnet(self):
220
+ re_str = '<Nnet>\n'
221
+ for unit in self.mem:
222
+ re_str += unit.to_kaldi_nnet()
223
+ re_str = self.expand2.toKaldiNNet()
224
+ relu = RectifiedLinear(self.expand2.linear.out_features,
225
+ self.expand2.linear.out_features)
226
+ re_str += relu.toKaldiNNet()
227
+ re_str += self.decision.toKaldiNNet()
228
+ re_str += '<Softmax> %d %d\n' % (self.decision.linear.out_features,
229
+ self.decision.linear.out_features)
230
+ re_str += '<!EndOfComponent>\n'
231
+ re_str += '</Nnet>\n'
232
+
233
+ return re_str
runtime/Lib/site_packages/models/audio/kws/farfield/model.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ import tempfile
5
+ from typing import Dict, Optional
6
+
7
+ from modelscope.metainfo import Models
8
+ from modelscope.models import TorchModel
9
+ from modelscope.models.base import Tensor
10
+ from modelscope.models.builder import MODELS
11
+ from modelscope.utils.audio.audio_utils import update_conf
12
+ from modelscope.utils.constant import Tasks
13
+ from .fsmn_sele_v2 import FSMNSeleNetV2
14
+ from .fsmn_sele_v3 import FSMNSeleNetV3
15
+
16
+
17
+ @MODELS.register_module(
18
+ Tasks.keyword_spotting, module_name=Models.speech_dfsmn_kws_char_farfield)
19
+ class FSMNSeleNetV2Decorator(TorchModel):
20
+ r""" A decorator of FSMNSeleNetV2 for integrating into modelscope framework """
21
+
22
+ MODEL_CLASS = FSMNSeleNetV2
23
+ MODEL_TXT = 'model.txt'
24
+ SC_CONFIG = 'sound_connect.conf'
25
+
26
+ def __init__(self,
27
+ model_dir: str,
28
+ training: Optional[bool] = False,
29
+ *args,
30
+ **kwargs):
31
+ """initialize the dfsmn model from the `model_dir` path.
32
+
33
+ Args:
34
+ model_dir (str): the model path.
35
+ """
36
+ super().__init__(model_dir, *args, **kwargs)
37
+ if training:
38
+ self.model = self.MODEL_CLASS(*args, **kwargs)
39
+ else:
40
+ sc_config_file = os.path.join(model_dir, self.SC_CONFIG)
41
+ model_txt_file = os.path.join(model_dir, self.MODEL_TXT)
42
+ self.tmp_dir = tempfile.TemporaryDirectory()
43
+ new_config_file = os.path.join(self.tmp_dir.name, self.SC_CONFIG)
44
+
45
+ self._sc = None
46
+ if os.path.exists(model_txt_file):
47
+ conf_dict = dict(kws_model=model_txt_file)
48
+ update_conf(sc_config_file, new_config_file, conf_dict)
49
+ import py_sound_connect
50
+ self._sc = py_sound_connect.SoundConnect(new_config_file)
51
+ self.size_in = self._sc.bytesPerBlockIn()
52
+ self.size_out = self._sc.bytesPerBlockOut()
53
+ else:
54
+ raise Exception(
55
+ f'Invalid model directory! Failed to load model file:'
56
+ f' {model_txt_file}.')
57
+
58
+ def __del__(self):
59
+ if hasattr(self, 'tmp_dir'):
60
+ self.tmp_dir.cleanup()
61
+
62
+ def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]:
63
+ return self.model.forward(input)
64
+
65
+ def forward_decode(self, data: bytes):
66
+ result = {'pcm': self._sc.process(data, self.size_out)}
67
+ state = self._sc.kwsState()
68
+ if state == 2:
69
+ result['kws'] = {
70
+ 'keyword':
71
+ self._sc.kwsKeyword(self._sc.kwsSpottedKeywordIndex()),
72
+ 'offset': self._sc.kwsKeywordOffset(),
73
+ 'channel': self._sc.kwsBestChannel(),
74
+ 'length': self._sc.kwsKeywordLength(),
75
+ 'confidence': self._sc.kwsConfidence()
76
+ }
77
+ return result
78
+
79
+
80
+ @MODELS.register_module(
81
+ Tasks.keyword_spotting,
82
+ module_name=Models.speech_dfsmn_kws_char_farfield_iot)
83
+ class FSMNSeleNetV3Decorator(FSMNSeleNetV2Decorator):
84
+ r""" A decorator of FSMNSeleNetV3 for integrating into modelscope framework """
85
+
86
+ MODEL_CLASS = FSMNSeleNetV3
87
+
88
+ def __init__(self,
89
+ model_dir: str,
90
+ training: Optional[bool] = False,
91
+ *args,
92
+ **kwargs):
93
+ """initialize the dfsmn model from the `model_dir` path.
94
+
95
+ Args:
96
+ model_dir (str): the model path.
97
+ """
98
+ super().__init__(model_dir, training, *args, **kwargs)
runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import math
4
+ import struct
5
+ from enum import Enum
6
+
7
+ HEADER_BLOCK_SIZE = 10
8
+
9
+
10
+ class LayerType(Enum):
11
+ LAYER_DENSE = 1
12
+ LAYER_GRU = 2
13
+ LAYER_ATTENTION = 3
14
+ LAYER_FSMN = 4
15
+ LAYER_SEQUENTIAL_FSMN = 5
16
+ LAYER_FSMN_SELE = 6
17
+ LAYER_GRU_ATTENTION = 7
18
+ LAYER_DFSMN = 8
19
+
20
+
21
+ class ActivationType(Enum):
22
+ ACTIVATION_NONE = 0
23
+ ACTIVATION_RELU = 1
24
+ ACTIVATION_TANH = 2
25
+ ACTIVATION_SIGMOID = 3
26
+ ACTIVATION_SOFTMAX = 4
27
+ ACTIVATION_LOGSOFTMAX = 5
28
+
29
+
30
+ def f32ToI32(f):
31
+ """
32
+ print layer
33
+ """
34
+ bs = struct.pack('f', f)
35
+
36
+ ba = bytearray()
37
+ ba.append(bs[0])
38
+ ba.append(bs[1])
39
+ ba.append(bs[2])
40
+ ba.append(bs[3])
41
+
42
+ return struct.unpack('i', ba)[0]
43
+
44
+
45
+ def printNeonMatrix(w):
46
+ """
47
+ print matrix with neon padding
48
+ """
49
+ numrows, numcols = w.shape
50
+ numnecols = math.ceil(numcols / 4)
51
+
52
+ for i in range(numrows):
53
+ for j in range(numcols):
54
+ print(f32ToI32(w[i, j]))
55
+
56
+ for j in range(numnecols * 4 - numcols):
57
+ print(0)
58
+
59
+
60
+ def printNeonVector(b):
61
+ """
62
+ print vector with neon padding
63
+ """
64
+ size = b.shape[0]
65
+ nesize = math.ceil(size / 4)
66
+
67
+ for i in range(size):
68
+ print(f32ToI32(b[i]))
69
+
70
+ for i in range(nesize * 4 - size):
71
+ print(0)
72
+
73
+
74
+ def printDense(layer):
75
+ """
76
+ save dense layer
77
+ """
78
+ statedict = layer.state_dict()
79
+ printNeonMatrix(statedict['weight'])
80
+ printNeonVector(statedict['bias'])
81
+
82
+
83
+ def printGRU(layer):
84
+ """
85
+ save gru layer
86
+ """
87
+ statedict = layer.state_dict()
88
+ weight = [statedict['weight_ih_l0'], statedict['weight_hh_l0']]
89
+ bias = [statedict['bias_ih_l0'], statedict['bias_hh_l0']]
90
+ numins, numouts = weight[0].shape
91
+ numins = numins // 3
92
+
93
+ # output input weights
94
+ w_rx = weight[0][:numins, :]
95
+ w_zx = weight[0][numins:numins * 2, :]
96
+ w_x = weight[0][numins * 2:, :]
97
+ printNeonMatrix(w_zx)
98
+ printNeonMatrix(w_rx)
99
+ printNeonMatrix(w_x)
100
+
101
+ # output recurrent weights
102
+ w_rh = weight[1][:numins, :]
103
+ w_zh = weight[1][numins:numins * 2, :]
104
+ w_h = weight[1][numins * 2:, :]
105
+ printNeonMatrix(w_zh)
106
+ printNeonMatrix(w_rh)
107
+ printNeonMatrix(w_h)
108
+
109
+ # output input bias
110
+ b_rx = bias[0][:numins]
111
+ b_zx = bias[0][numins:numins * 2]
112
+ b_x = bias[0][numins * 2:]
113
+ printNeonVector(b_zx)
114
+ printNeonVector(b_rx)
115
+ printNeonVector(b_x)
116
+
117
+ # output recurrent bias
118
+ b_rh = bias[1][:numins]
119
+ b_zh = bias[1][numins:numins * 2]
120
+ b_h = bias[1][numins * 2:]
121
+ printNeonVector(b_zh)
122
+ printNeonVector(b_rh)
123
+ printNeonVector(b_h)
runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ from typing import Any, Dict
5
+
6
+ from modelscope.metainfo import Models
7
+ from modelscope.models.base import Model
8
+ from modelscope.models.builder import MODELS
9
+ from modelscope.utils.constant import Tasks
10
+
11
+ __all__ = ['GenericKeyWordSpotting']
12
+
13
+
14
+ @MODELS.register_module(Tasks.keyword_spotting, module_name=Models.kws_kwsbp)
15
+ class GenericKeyWordSpotting(Model):
16
+
17
+ def __init__(self, model_dir: str, *args, **kwargs):
18
+ """initialize the info of model.
19
+
20
+ Args:
21
+ model_dir (str): the model path.
22
+ """
23
+ super().__init__(model_dir, *args, **kwargs)
24
+ self.model_cfg = {
25
+ 'model_workspace': model_dir,
26
+ 'config_path': os.path.join(model_dir, 'config.yaml')
27
+ }
28
+
29
+ def forward(self) -> Dict[str, Any]:
30
+ """return the info of the model
31
+ """
32
+ return self.model_cfg
runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) 2020 Binbin Zhang
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import re
17
+
18
+ import numpy as np
19
+ import torch
20
+
21
+
22
+ class GlobalCMVN(torch.nn.Module):
23
+
24
+ def __init__(self,
25
+ mean: torch.Tensor,
26
+ istd: torch.Tensor,
27
+ norm_var: bool = True):
28
+ """
29
+ Args:
30
+ mean (torch.Tensor): mean stats
31
+ istd (torch.Tensor): inverse std, std which is 1.0 / std
32
+ """
33
+ super().__init__()
34
+ assert mean.shape == istd.shape
35
+ self.norm_var = norm_var
36
+ # The buffer can be accessed from this module using self.mean
37
+ self.register_buffer('mean', mean)
38
+ self.register_buffer('istd', istd)
39
+
40
+ def forward(self, x: torch.Tensor):
41
+ """
42
+ Args:
43
+ x (torch.Tensor): (batch, max_len, feat_dim)
44
+
45
+ Returns:
46
+ (torch.Tensor): normalized feature
47
+ """
48
+ x = x - self.mean
49
+ if self.norm_var:
50
+ x = x * self.istd
51
+ return x
52
+
53
+
54
+ def load_kaldi_cmvn(cmvn_file):
55
+ """ Load the kaldi format cmvn stats file and no need to calculate
56
+
57
+ Args:
58
+ cmvn_file: cmvn stats file in kaldi format
59
+
60
+ Returns:
61
+ a numpy array of [means, vars]
62
+ """
63
+
64
+ means = None
65
+ variance = None
66
+ with open(cmvn_file) as f:
67
+ all_lines = f.readlines()
68
+ for idx, line in enumerate(all_lines):
69
+ if line.find('AddShift') != -1:
70
+ segs = line.strip().split(' ')
71
+ assert len(segs) == 3
72
+ next_line = all_lines[idx + 1]
73
+ means_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
74
+ means_list = means_str.strip().split(' ')
75
+ means = [0 - float(s) for s in means_list]
76
+ assert len(means) == int(segs[1])
77
+ elif line.find('Rescale') != -1:
78
+ segs = line.strip().split(' ')
79
+ assert len(segs) == 3
80
+ next_line = all_lines[idx + 1]
81
+ vars_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
82
+ vars_list = vars_str.strip().split(' ')
83
+ variance = [float(s) for s in vars_list]
84
+ assert len(variance) == int(segs[1])
85
+ elif line.find('Splice') != -1:
86
+ segs = line.strip().split(' ')
87
+ assert len(segs) == 3
88
+ next_line = all_lines[idx + 1]
89
+ splice_str = re.findall(r'[\[](.*?)[\]]', next_line)[0]
90
+ splice_list = splice_str.strip().split(' ')
91
+ assert len(splice_list) * int(segs[2]) == int(segs[1])
92
+ copy_times = len(splice_list)
93
+ else:
94
+ continue
95
+
96
+ cmvn = np.array([means, variance])
97
+ cmvn = np.tile(cmvn, (1, copy_times))
98
+
99
+ return cmvn
runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py ADDED
@@ -0,0 +1,521 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ '''
2
+ FSMN implementation.
3
+
4
+ Copyright: 2022-03-09 yueyue.nyy
5
+ '''
6
+
7
+ from typing import Tuple
8
+
9
+ import numpy as np
10
+ import torch
11
+ import torch.nn as nn
12
+ import torch.nn.functional as F
13
+
14
+
15
+ def toKaldiMatrix(np_mat):
16
+ np.set_printoptions(threshold=np.inf, linewidth=np.nan)
17
+ out_str = str(np_mat)
18
+ out_str = out_str.replace('[', '')
19
+ out_str = out_str.replace(']', '')
20
+ return '[ %s ]\n' % out_str
21
+
22
+
23
+ def printTensor(torch_tensor):
24
+ re_str = ''
25
+ x = torch_tensor.detach().squeeze().numpy()
26
+ re_str += toKaldiMatrix(x)
27
+ # re_str += '<!EndOfComponent>\n'
28
+ print(re_str)
29
+
30
+
31
+ class LinearTransform(nn.Module):
32
+
33
+ def __init__(self, input_dim, output_dim):
34
+ super(LinearTransform, self).__init__()
35
+ self.input_dim = input_dim
36
+ self.output_dim = output_dim
37
+ self.linear = nn.Linear(input_dim, output_dim, bias=False)
38
+ self.quant = torch.quantization.QuantStub()
39
+ self.dequant = torch.quantization.DeQuantStub()
40
+
41
+ def forward(self, input):
42
+ output = self.quant(input)
43
+ output = self.linear(output)
44
+ output = self.dequant(output)
45
+
46
+ return output
47
+
48
+ def to_kaldi_net(self):
49
+ re_str = ''
50
+ re_str += '<LinearTransform> %d %d\n' % (self.output_dim,
51
+ self.input_dim)
52
+ re_str += '<LearnRateCoef> 1\n'
53
+
54
+ linear_weights = self.state_dict()['linear.weight']
55
+ x = linear_weights.squeeze().numpy()
56
+ re_str += toKaldiMatrix(x)
57
+ # re_str += '<!EndOfComponent>\n'
58
+
59
+ return re_str
60
+
61
+ def to_pytorch_net(self, fread):
62
+ linear_line = fread.readline()
63
+ linear_split = linear_line.strip().split()
64
+ assert len(linear_split) == 3
65
+ assert linear_split[0] == '<LinearTransform>'
66
+ self.output_dim = int(linear_split[1])
67
+ self.input_dim = int(linear_split[2])
68
+
69
+ learn_rate_line = fread.readline()
70
+ assert learn_rate_line.find('LearnRateCoef') != -1
71
+
72
+ self.linear.reset_parameters()
73
+
74
+ # linear_weights = self.state_dict()['linear.weight']
75
+ # print(linear_weights.shape)
76
+ new_weights = torch.zeros((self.output_dim, self.input_dim),
77
+ dtype=torch.float32)
78
+ for i in range(self.output_dim):
79
+ line = fread.readline()
80
+ splits = line.strip().strip('[]').strip().split()
81
+ assert len(splits) == self.input_dim
82
+ cols = torch.tensor([float(item) for item in splits],
83
+ dtype=torch.float32)
84
+ new_weights[i, :] = cols
85
+
86
+ self.linear.weight.data = new_weights
87
+
88
+
89
+ class AffineTransform(nn.Module):
90
+
91
+ def __init__(self, input_dim, output_dim):
92
+ super(AffineTransform, self).__init__()
93
+ self.input_dim = input_dim
94
+ self.output_dim = output_dim
95
+
96
+ self.linear = nn.Linear(input_dim, output_dim)
97
+ self.quant = torch.quantization.QuantStub()
98
+ self.dequant = torch.quantization.DeQuantStub()
99
+
100
+ def forward(self, input):
101
+ output = self.quant(input)
102
+ output = self.linear(output)
103
+ output = self.dequant(output)
104
+
105
+ return output
106
+
107
+ def to_kaldi_net(self):
108
+ re_str = ''
109
+ re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
110
+ self.input_dim)
111
+ re_str += '<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0\n'
112
+
113
+ linear_weights = self.state_dict()['linear.weight']
114
+ x = linear_weights.squeeze().numpy()
115
+ re_str += toKaldiMatrix(x)
116
+
117
+ linear_bias = self.state_dict()['linear.bias']
118
+ x = linear_bias.squeeze().numpy()
119
+ re_str += toKaldiMatrix(x)
120
+ # re_str += '<!EndOfComponent>\n'
121
+
122
+ return re_str
123
+
124
+ def to_pytorch_net(self, fread):
125
+ affine_line = fread.readline()
126
+ affine_split = affine_line.strip().split()
127
+ assert len(affine_split) == 3
128
+ assert affine_split[0] == '<AffineTransform>'
129
+ self.output_dim = int(affine_split[1])
130
+ self.input_dim = int(affine_split[2])
131
+ print('AffineTransform output/input dim: %d %d' %
132
+ (self.output_dim, self.input_dim))
133
+
134
+ learn_rate_line = fread.readline()
135
+ assert learn_rate_line.find('LearnRateCoef') != -1
136
+
137
+ # linear_weights = self.state_dict()['linear.weight']
138
+ # print(linear_weights.shape)
139
+ self.linear.reset_parameters()
140
+
141
+ new_weights = torch.zeros((self.output_dim, self.input_dim),
142
+ dtype=torch.float32)
143
+ for i in range(self.output_dim):
144
+ line = fread.readline()
145
+ splits = line.strip().strip('[]').strip().split()
146
+ assert len(splits) == self.input_dim
147
+ cols = torch.tensor([float(item) for item in splits],
148
+ dtype=torch.float32)
149
+ new_weights[i, :] = cols
150
+
151
+ self.linear.weight.data = new_weights
152
+
153
+ # linear_bias = self.state_dict()['linear.bias']
154
+ # print(linear_bias.shape)
155
+ bias_line = fread.readline()
156
+ splits = bias_line.strip().strip('[]').strip().split()
157
+ assert len(splits) == self.output_dim
158
+ new_bias = torch.tensor([float(item) for item in splits],
159
+ dtype=torch.float32)
160
+
161
+ self.linear.bias.data = new_bias
162
+
163
+
164
+ class FSMNBlock(nn.Module):
165
+
166
+ def __init__(
167
+ self,
168
+ input_dim: int,
169
+ output_dim: int,
170
+ lorder=None,
171
+ rorder=None,
172
+ lstride=1,
173
+ rstride=1,
174
+ ):
175
+ super(FSMNBlock, self).__init__()
176
+
177
+ self.dim = input_dim
178
+
179
+ if lorder is None:
180
+ return
181
+
182
+ self.lorder = lorder
183
+ self.rorder = rorder
184
+ self.lstride = lstride
185
+ self.rstride = rstride
186
+
187
+ self.conv_left = nn.Conv2d(
188
+ self.dim,
189
+ self.dim, [lorder, 1],
190
+ dilation=[lstride, 1],
191
+ groups=self.dim,
192
+ bias=False)
193
+
194
+ if rorder > 0:
195
+ self.conv_right = nn.Conv2d(
196
+ self.dim,
197
+ self.dim, [rorder, 1],
198
+ dilation=[rstride, 1],
199
+ groups=self.dim,
200
+ bias=False)
201
+ else:
202
+ self.conv_right = None
203
+
204
+ self.quant = torch.quantization.QuantStub()
205
+ self.dequant = torch.quantization.DeQuantStub()
206
+
207
+ def forward(self, input):
208
+ x = torch.unsqueeze(input, 1)
209
+ x_per = x.permute(0, 3, 2, 1)
210
+
211
+ y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
212
+ y_left = self.quant(y_left)
213
+ y_left = self.conv_left(y_left)
214
+ y_left = self.dequant(y_left)
215
+ out = x_per + y_left
216
+
217
+ if self.conv_right is not None:
218
+ y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
219
+ y_right = y_right[:, :, self.rstride:, :]
220
+ y_right = self.quant(y_right)
221
+ y_right = self.conv_right(y_right)
222
+ y_right = self.dequant(y_right)
223
+ out += y_right
224
+
225
+ out_per = out.permute(0, 3, 2, 1)
226
+ output = out_per.squeeze(1)
227
+
228
+ return output
229
+
230
+ def to_kaldi_net(self):
231
+ re_str = ''
232
+ re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)
233
+ re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (
234
+ 1, self.lorder, self.rorder, self.lstride, self.rstride)
235
+
236
+ # print(self.conv_left.weight,self.conv_right.weight)
237
+ lfiters = self.state_dict()['conv_left.weight']
238
+ x = np.flipud(lfiters.squeeze().numpy().T)
239
+ re_str += toKaldiMatrix(x)
240
+
241
+ if self.conv_right is not None:
242
+ rfiters = self.state_dict()['conv_right.weight']
243
+ x = (rfiters.squeeze().numpy().T)
244
+ re_str += toKaldiMatrix(x)
245
+ # re_str += '<!EndOfComponent>\n'
246
+
247
+ return re_str
248
+
249
+ def to_pytorch_net(self, fread):
250
+ fsmn_line = fread.readline()
251
+ fsmn_split = fsmn_line.strip().split()
252
+ assert len(fsmn_split) == 3
253
+ assert fsmn_split[0] == '<Fsmn>'
254
+ self.dim = int(fsmn_split[1])
255
+
256
+ params_line = fread.readline()
257
+ params_split = params_line.strip().strip('[]').strip().split()
258
+ assert len(params_split) == 12
259
+ assert params_split[0] == '<LearnRateCoef>'
260
+ assert params_split[2] == '<LOrder>'
261
+ self.lorder = int(params_split[3])
262
+ assert params_split[4] == '<ROrder>'
263
+ self.rorder = int(params_split[5])
264
+ assert params_split[6] == '<LStride>'
265
+ self.lstride = int(params_split[7])
266
+ assert params_split[8] == '<RStride>'
267
+ self.rstride = int(params_split[9])
268
+ assert params_split[10] == '<MaxNorm>'
269
+
270
+ # lfilters = self.state_dict()['conv_left.weight']
271
+ # print(lfilters.shape)
272
+ print('read conv_left weight')
273
+ new_lfilters = torch.zeros((self.lorder, 1, self.dim, 1),
274
+ dtype=torch.float32)
275
+ for i in range(self.lorder):
276
+ print('read conv_left weight -- %d' % i)
277
+ line = fread.readline()
278
+ splits = line.strip().strip('[]').strip().split()
279
+ assert len(splits) == self.dim
280
+ cols = torch.tensor([float(item) for item in splits],
281
+ dtype=torch.float32)
282
+ new_lfilters[self.lorder - 1 - i, 0, :, 0] = cols
283
+
284
+ new_lfilters = torch.transpose(new_lfilters, 0, 2)
285
+ # print(new_lfilters.shape)
286
+
287
+ self.conv_left.reset_parameters()
288
+ self.conv_left.weight.data = new_lfilters
289
+ # print(self.conv_left.weight.shape)
290
+
291
+ if self.rorder > 0:
292
+ # rfilters = self.state_dict()['conv_right.weight']
293
+ # print(rfilters.shape)
294
+ print('read conv_right weight')
295
+ new_rfilters = torch.zeros((self.rorder, 1, self.dim, 1),
296
+ dtype=torch.float32)
297
+ line = fread.readline()
298
+ for i in range(self.rorder):
299
+ print('read conv_right weight -- %d' % i)
300
+ line = fread.readline()
301
+ splits = line.strip().strip('[]').strip().split()
302
+ assert len(splits) == self.dim
303
+ cols = torch.tensor([float(item) for item in splits],
304
+ dtype=torch.float32)
305
+ new_rfilters[i, 0, :, 0] = cols
306
+
307
+ new_rfilters = torch.transpose(new_rfilters, 0, 2)
308
+ # print(new_rfilters.shape)
309
+ self.conv_right.reset_parameters()
310
+ self.conv_right.weight.data = new_rfilters
311
+ # print(self.conv_right.weight.shape)
312
+
313
+
314
+ class RectifiedLinear(nn.Module):
315
+
316
+ def __init__(self, input_dim, output_dim):
317
+ super(RectifiedLinear, self).__init__()
318
+ self.dim = input_dim
319
+ self.relu = nn.ReLU()
320
+ self.dropout = nn.Dropout(0.1)
321
+
322
+ def forward(self, input):
323
+ out = self.relu(input)
324
+ # out = self.dropout(out)
325
+ return out
326
+
327
+ def to_kaldi_net(self):
328
+ re_str = ''
329
+ re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
330
+ # re_str += '<!EndOfComponent>\n'
331
+ return re_str
332
+
333
+ # re_str = ''
334
+ # re_str += '<ParametricRelu> %d %d\n' % (self.dim, self.dim)
335
+ # re_str += '<AlphaLearnRateCoef> 0 <BetaLearnRateCoef> 0\n'
336
+ # re_str += toKaldiMatrix(np.ones((self.dim), dtype = 'int32'))
337
+ # re_str += toKaldiMatrix(np.zeros((self.dim), dtype = 'int32'))
338
+ # re_str += '<!EndOfComponent>\n'
339
+ # return re_str
340
+
341
+ def to_pytorch_net(self, fread):
342
+ line = fread.readline()
343
+ splits = line.strip().split()
344
+ assert len(splits) == 3
345
+ assert splits[0] == '<RectifiedLinear>'
346
+ assert int(splits[1]) == int(splits[2])
347
+ assert int(splits[1]) == self.dim
348
+ self.dim = int(splits[1])
349
+
350
+
351
+ def _build_repeats(
352
+ fsmn_layers: int,
353
+ linear_dim: int,
354
+ proj_dim: int,
355
+ lorder: int,
356
+ rorder: int,
357
+ lstride=1,
358
+ rstride=1,
359
+ ):
360
+ repeats = [
361
+ nn.Sequential(
362
+ LinearTransform(linear_dim, proj_dim),
363
+ FSMNBlock(proj_dim, proj_dim, lorder, rorder, 1, 1),
364
+ AffineTransform(proj_dim, linear_dim),
365
+ RectifiedLinear(linear_dim, linear_dim))
366
+ for i in range(fsmn_layers)
367
+ ]
368
+
369
+ return nn.Sequential(*repeats)
370
+
371
+
372
+ class FSMN(nn.Module):
373
+
374
+ def __init__(
375
+ self,
376
+ input_dim: int,
377
+ input_affine_dim: int,
378
+ fsmn_layers: int,
379
+ linear_dim: int,
380
+ proj_dim: int,
381
+ lorder: int,
382
+ rorder: int,
383
+ lstride: int,
384
+ rstride: int,
385
+ output_affine_dim: int,
386
+ output_dim: int,
387
+ ):
388
+ """
389
+ Args:
390
+ input_dim: input dimension
391
+ input_affine_dim: input affine layer dimension
392
+ fsmn_layers: no. of fsmn units
393
+ linear_dim: fsmn input dimension
394
+ proj_dim: fsmn projection dimension
395
+ lorder: fsmn left order
396
+ rorder: fsmn right order
397
+ lstride: fsmn left stride
398
+ rstride: fsmn right stride
399
+ output_affine_dim: output affine layer dimension
400
+ output_dim: output dimension
401
+ """
402
+ super(FSMN, self).__init__()
403
+
404
+ self.input_dim = input_dim
405
+ self.input_affine_dim = input_affine_dim
406
+ self.fsmn_layers = fsmn_layers
407
+ self.linear_dim = linear_dim
408
+ self.proj_dim = proj_dim
409
+ self.lorder = lorder
410
+ self.rorder = rorder
411
+ self.lstride = lstride
412
+ self.rstride = rstride
413
+ self.output_affine_dim = output_affine_dim
414
+ self.output_dim = output_dim
415
+
416
+ self.in_linear1 = AffineTransform(input_dim, input_affine_dim)
417
+ self.in_linear2 = AffineTransform(input_affine_dim, linear_dim)
418
+ self.relu = RectifiedLinear(linear_dim, linear_dim)
419
+
420
+ self.fsmn = _build_repeats(fsmn_layers, linear_dim, proj_dim, lorder,
421
+ rorder, lstride, rstride)
422
+
423
+ self.out_linear1 = AffineTransform(linear_dim, output_affine_dim)
424
+ self.out_linear2 = AffineTransform(output_affine_dim, output_dim)
425
+ # self.softmax = nn.Softmax(dim = -1)
426
+
427
+ def fuse_modules(self):
428
+ pass
429
+
430
+ def forward(
431
+ self,
432
+ input: torch.Tensor,
433
+ in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float)
434
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
435
+ """
436
+ Args:
437
+ input (torch.Tensor): Input tensor (B, T, D)
438
+ in_cache(torch.Tensor): (B, D, C), C is the accumulated cache size
439
+ """
440
+
441
+ # print("FSMN forward!!!!")
442
+ # print(input.shape)
443
+ # print(input)
444
+ # print(self.in_linear1.input_dim)
445
+ # print(self.in_linear1.output_dim)
446
+
447
+ x1 = self.in_linear1(input)
448
+ x2 = self.in_linear2(x1)
449
+ x3 = self.relu(x2)
450
+ x4 = self.fsmn(x3)
451
+ x5 = self.out_linear1(x4)
452
+ x6 = self.out_linear2(x5)
453
+ # x7 = self.softmax(x6)
454
+
455
+ # return x7, None
456
+ return x6, in_cache
457
+
458
+ def to_kaldi_net(self):
459
+ re_str = ''
460
+ re_str += '<Nnet>\n'
461
+ re_str += self.in_linear1.to_kaldi_net()
462
+ re_str += self.in_linear2.to_kaldi_net()
463
+ re_str += self.relu.to_kaldi_net()
464
+
465
+ for fsmn in self.fsmn:
466
+ re_str += fsmn[0].to_kaldi_net()
467
+ re_str += fsmn[1].to_kaldi_net()
468
+ re_str += fsmn[2].to_kaldi_net()
469
+ re_str += fsmn[3].to_kaldi_net()
470
+
471
+ re_str += self.out_linear1.to_kaldi_net()
472
+ re_str += self.out_linear2.to_kaldi_net()
473
+ re_str += '<Softmax> %d %d\n' % (self.output_dim, self.output_dim)
474
+ # re_str += '<!EndOfComponent>\n'
475
+ re_str += '</Nnet>\n'
476
+
477
+ return re_str
478
+
479
+ def to_pytorch_net(self, kaldi_file):
480
+ with open(kaldi_file, 'r', encoding='utf8') as fread:
481
+ fread = open(kaldi_file, 'r')
482
+ nnet_start_line = fread.readline()
483
+ assert nnet_start_line.strip() == '<Nnet>'
484
+
485
+ self.in_linear1.to_pytorch_net(fread)
486
+ self.in_linear2.to_pytorch_net(fread)
487
+ self.relu.to_pytorch_net(fread)
488
+
489
+ for fsmn in self.fsmn:
490
+ fsmn[0].to_pytorch_net(fread)
491
+ fsmn[1].to_pytorch_net(fread)
492
+ fsmn[2].to_pytorch_net(fread)
493
+ fsmn[3].to_pytorch_net(fread)
494
+
495
+ self.out_linear1.to_pytorch_net(fread)
496
+ self.out_linear2.to_pytorch_net(fread)
497
+
498
+ softmax_line = fread.readline()
499
+ softmax_split = softmax_line.strip().split()
500
+ assert softmax_split[0].strip() == '<Softmax>'
501
+ assert int(softmax_split[1]) == self.output_dim
502
+ assert int(softmax_split[2]) == self.output_dim
503
+ # '<!EndOfComponent>\n'
504
+
505
+ nnet_end_line = fread.readline()
506
+ assert nnet_end_line.strip() == '</Nnet>'
507
+ fread.close()
508
+
509
+
510
+ if __name__ == '__main__':
511
+ fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599)
512
+ print(fsmn)
513
+
514
+ num_params = sum(p.numel() for p in fsmn.parameters())
515
+ print('the number of model params: {}'.format(num_params))
516
+ x = torch.zeros(128, 200, 400) # batch-size * time * dim
517
+ y, _ = fsmn(x) # batch-size * time * dim
518
+ print('input shape: {}'.format(x.shape))
519
+ print('output shape: {}'.format(y.shape))
520
+
521
+ print(fsmn.to_kaldi_net())
runtime/Lib/site_packages/models/audio/kws/nearfield/model.py ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ import sys
5
+ import tempfile
6
+ from typing import Dict, Optional, Tuple
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+
11
+ from modelscope.metainfo import Models
12
+ from modelscope.models import TorchModel
13
+ from modelscope.models.base import Tensor
14
+ from modelscope.models.builder import MODELS
15
+ from modelscope.utils.audio.audio_utils import update_conf
16
+ from modelscope.utils.constant import Tasks
17
+ from .cmvn import GlobalCMVN, load_kaldi_cmvn
18
+ from .fsmn import FSMN
19
+
20
+
21
+ @MODELS.register_module(
22
+ Tasks.keyword_spotting,
23
+ module_name=Models.speech_kws_fsmn_char_ctc_nearfield)
24
+ class FSMNDecorator(TorchModel):
25
+ r""" A decorator of FSMN for integrating into modelscope framework """
26
+
27
+ def __init__(self,
28
+ model_dir: str,
29
+ cmvn_file: str = None,
30
+ backbone: dict = None,
31
+ input_dim: int = 400,
32
+ output_dim: int = 2599,
33
+ training: Optional[bool] = False,
34
+ *args,
35
+ **kwargs):
36
+ """initialize the fsmn model from the `model_dir` path.
37
+
38
+ Args:
39
+ model_dir (str): the model path.
40
+ cmvn_file (str): cmvn file
41
+ backbone (dict): params related to backbone
42
+ input_dim (int): input dimension of network
43
+ output_dim (int): output dimension of network
44
+ training (bool): training or inference mode
45
+ """
46
+ super().__init__(model_dir, *args, **kwargs)
47
+
48
+ self.model = None
49
+ self.model_cfg = None
50
+
51
+ if training:
52
+ self.model = self.init_model(cmvn_file, backbone, input_dim,
53
+ output_dim)
54
+ else:
55
+ self.model_cfg = {
56
+ 'model_workspace': model_dir,
57
+ 'config_path': os.path.join(model_dir, 'config.yaml')
58
+ }
59
+
60
+ def __del__(self):
61
+ if hasattr(self, 'tmp_dir'):
62
+ self.tmp_dir.cleanup()
63
+
64
+ def forward(self, input) -> Dict[str, Tensor]:
65
+ """
66
+ Args:
67
+ input (torch.Tensor): Input tensor (B, T, D)
68
+ """
69
+ if self.model is not None and input is not None:
70
+ return self.model.forward(input)
71
+ else:
72
+ return self.model_cfg
73
+
74
+ def init_model(self, cmvn_file, backbone, input_dim, output_dim):
75
+ if cmvn_file is not None:
76
+ mean, istd = load_kaldi_cmvn(cmvn_file)
77
+ global_cmvn = GlobalCMVN(
78
+ torch.from_numpy(mean).float(),
79
+ torch.from_numpy(istd).float(),
80
+ )
81
+ else:
82
+ global_cmvn = None
83
+
84
+ hidden_dim = 128
85
+ preprocessing = None
86
+
87
+ input_affine_dim = backbone['input_affine_dim']
88
+ num_layers = backbone['num_layers']
89
+ linear_dim = backbone['linear_dim']
90
+ proj_dim = backbone['proj_dim']
91
+ left_order = backbone['left_order']
92
+ right_order = backbone['right_order']
93
+ left_stride = backbone['left_stride']
94
+ right_stride = backbone['right_stride']
95
+ output_affine_dim = backbone['output_affine_dim']
96
+ backbone = FSMN(input_dim, input_affine_dim, num_layers, linear_dim,
97
+ proj_dim, left_order, right_order, left_stride,
98
+ right_stride, output_affine_dim, output_dim)
99
+
100
+ classifier = None
101
+ activation = None
102
+
103
+ kws_model = KWSModel(input_dim, output_dim, hidden_dim, global_cmvn,
104
+ preprocessing, backbone, classifier, activation)
105
+ return kws_model
106
+
107
+
108
+ class KWSModel(nn.Module):
109
+ """Our model consists of four parts:
110
+ 1. global_cmvn: Optional, (idim, idim)
111
+ 2. preprocessing: feature dimension projection, (idim, hdim)
112
+ 3. backbone: backbone or feature extractor of the whole network, (hdim, hdim)
113
+ 4. classifier: output layer or classifier of KWS model, (hdim, odim)
114
+ 5. activation:
115
+ nn.Sigmoid for wakeup word
116
+ nn.Identity for speech command dataset
117
+ """
118
+
119
+ def __init__(
120
+ self,
121
+ idim: int,
122
+ odim: int,
123
+ hdim: int,
124
+ global_cmvn: Optional[nn.Module],
125
+ preprocessing: Optional[nn.Module],
126
+ backbone: nn.Module,
127
+ classifier: nn.Module,
128
+ activation: nn.Module,
129
+ ):
130
+ """
131
+ Args:
132
+ idim (int): input dimension of network
133
+ odim (int): output dimension of network
134
+ hdim (int): hidden dimension of network
135
+ global_cmvn (nn.Module): cmvn for input feature, (idim, idim)
136
+ preprocessing (nn.Module): feature dimension projection, (idim, hdim)
137
+ backbone (nn.Module): backbone or feature extractor of the whole network, (hdim, hdim)
138
+ classifier (nn.Module): output layer or classifier of KWS model, (hdim, odim)
139
+ activation (nn.Module): nn.Identity for training, nn.Sigmoid for inference
140
+ """
141
+ super().__init__()
142
+ self.idim = idim
143
+ self.odim = odim
144
+ self.hdim = hdim
145
+ self.global_cmvn = global_cmvn
146
+ self.preprocessing = preprocessing
147
+ self.backbone = backbone
148
+ self.classifier = classifier
149
+ self.activation = activation
150
+
151
+ def to_kaldi_net(self):
152
+ return self.backbone.to_kaldi_net()
153
+
154
+ def to_pytorch_net(self, kaldi_file):
155
+ return self.backbone.to_pytorch_net(kaldi_file)
156
+
157
+ def forward(
158
+ self,
159
+ x: torch.Tensor,
160
+ in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float)
161
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
162
+ if self.global_cmvn is not None:
163
+ x = self.global_cmvn(x)
164
+ if self.preprocessing is not None:
165
+ x = self.preprocessing(x)
166
+
167
+ x, out_cache = self.backbone(x, in_cache)
168
+
169
+ if self.classifier is not None:
170
+ x = self.classifier(x)
171
+ if self.activation is not None:
172
+ x = self.activation(x)
173
+ return x, out_cache
174
+
175
+ def fuse_modules(self):
176
+ if self.preprocessing is not None:
177
+ self.preprocessing.fuse_modules()
178
+ self.backbone.fuse_modules()
runtime/Lib/site_packages/models/audio/punc/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import os
4
+ from typing import Any, Dict
5
+
6
+ from modelscope.metainfo import Models
7
+ from modelscope.models.base import Model
8
+ from modelscope.models.builder import MODELS
9
+ from modelscope.utils.constant import Frameworks, Tasks
10
+
11
+
12
+ @MODELS.register_module(Tasks.punctuation, module_name=Models.generic_punc)
13
+ class PunctuationProcessing(Model):
14
+
15
+ def __init__(self, model_dir: str, punc_model_name: str,
16
+ punc_model_config: Dict[str, Any], *args, **kwargs):
17
+ """initialize the info of model.
18
+
19
+ Args:
20
+ model_dir (str): the model path.
21
+ punc_model_name (str): the itn model name from configuration.json
22
+ punc_model_config (Dict[str, Any]): the detail config about model from configuration.json
23
+ """
24
+ super().__init__(model_dir, punc_model_name, punc_model_config, *args,
25
+ **kwargs)
26
+ self.model_cfg = {
27
+ # the recognition model dir path
28
+ 'model_workspace': model_dir,
29
+ # the itn model name
30
+ 'punc_model': punc_model_name,
31
+ # the am model file path
32
+ 'punc_model_path': os.path.join(model_dir, punc_model_name),
33
+ # the recognition model config dict
34
+ 'model_config': punc_model_config
35
+ }
36
+
37
+ def forward(self) -> Dict[str, Any]:
38
+ """
39
+ just return the model config
40
+
41
+ """
42
+
43
+ return self.model_cfg
runtime/Lib/site_packages/models/audio/separation/__init__.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+ from typing import TYPE_CHECKING
3
+
4
+ from modelscope.utils.import_utils import LazyImportModule
5
+
6
+ if TYPE_CHECKING:
7
+ from .mossformer import MossFormer
8
+ from .m2.mossformer import MossFormer2
9
+
10
+ else:
11
+ _import_structure = {
12
+ 'mossformer': ['MossFormer'],
13
+ 'm2.mossformer': ['MossFormer2'],
14
+ }
15
+
16
+ import sys
17
+
18
+ sys.modules[__name__] = LazyImportModule(
19
+ __name__,
20
+ globals()['__file__'],
21
+ _import_structure,
22
+ module_spec=__spec__,
23
+ extra_objects={},
24
+ )
runtime/Lib/site_packages/models/audio/separation/layer_norm.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2018 Northwestern Polytechnical University (author: Ke Wang)
2
+ # made publicly available under the MIT License
3
+ # at https://github.com/wangkenpu/Conv-TasNet-PyTorch/blob/64188ffa48971218fdd68b66906970f215d7eca2/model/layer_norm.py
4
+
5
+ from __future__ import absolute_import, division, print_function
6
+
7
+ import torch
8
+ import torch.nn as nn
9
+
10
+
11
+ class CLayerNorm(nn.LayerNorm):
12
+ """Channel-wise layer normalization."""
13
+
14
+ def __init__(self, *args, **kwargs):
15
+ super(CLayerNorm, self).__init__(*args, **kwargs)
16
+
17
+ def forward(self, sample):
18
+ """Forward function.
19
+
20
+ Args:
21
+ sample: [batch_size, channels, length]
22
+ """
23
+ if sample.dim() != 3:
24
+ raise RuntimeError('{} only accept 3-D tensor as input'.format(
25
+ self.__name__))
26
+ # [N, C, T] -> [N, T, C]
27
+ sample = torch.transpose(sample, 1, 2)
28
+ # LayerNorm
29
+ sample = super().forward(sample)
30
+ # [N, T, C] -> [N, C, T]
31
+ sample = torch.transpose(sample, 1, 2)
32
+ return sample
33
+
34
+
35
+ class GLayerNorm(nn.Module):
36
+ """Global Layer Normalization for TasNet."""
37
+
38
+ def __init__(self, channels, eps=1e-5):
39
+ super(GLayerNorm, self).__init__()
40
+ self.eps = eps
41
+ self.norm_dim = channels
42
+ self.gamma = nn.Parameter(torch.Tensor(channels))
43
+ self.beta = nn.Parameter(torch.Tensor(channels))
44
+ self.reset_parameters()
45
+
46
+ def reset_parameters(self):
47
+ nn.init.ones_(self.gamma)
48
+ nn.init.zeros_(self.beta)
49
+
50
+ def forward(self, sample):
51
+ """Forward function.
52
+
53
+ Args:
54
+ sample: [batch_size, channels, length]
55
+ """
56
+ if sample.dim() != 3:
57
+ raise RuntimeError('{} only accept 3-D tensor as input'.format(
58
+ self.__name__))
59
+ # [N, C, T] -> [N, T, C]
60
+ sample = torch.transpose(sample, 1, 2)
61
+ # Mean and variance [N, 1, 1]
62
+ mean = torch.mean(sample, (1, 2), keepdim=True)
63
+ var = torch.mean((sample - mean)**2, (1, 2), keepdim=True)
64
+ sample = (sample
65
+ - mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta
66
+ # [N, T, C] -> [N, C, T]
67
+ sample = torch.transpose(sample, 1, 2)
68
+ return sample
runtime/Lib/site_packages/models/audio/separation/m2/__init__.py ADDED
File without changes
runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py ADDED
@@ -0,0 +1,278 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ import torch.nn.init as init
6
+ from torch import Tensor
7
+
8
+ EPS = 1e-8
9
+
10
+
11
+ class GlobalLayerNorm(nn.Module):
12
+ """Calculate Global Layer Normalization.
13
+
14
+ Args:
15
+ dim : (int or list or torch.Size)
16
+ Input shape from an expected input of size.
17
+ eps : float
18
+ A value added to the denominator for numerical stability.
19
+ elementwise_affine : bool
20
+ A boolean value that when set to True,
21
+ this module has learnable per-element affine parameters
22
+ initialized to ones (for weights) and zeros (for biases).
23
+
24
+ Example:
25
+ -------
26
+ >>> x = torch.randn(5, 10, 20)
27
+ >>> GLN = GlobalLayerNorm(10, 3)
28
+ >>> x_norm = GLN(x)
29
+ """
30
+
31
+ def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True):
32
+ super(GlobalLayerNorm, self).__init__()
33
+ self.dim = dim
34
+ self.eps = eps
35
+ self.elementwise_affine = elementwise_affine
36
+
37
+ if self.elementwise_affine:
38
+ if shape == 3:
39
+ self.weight = nn.Parameter(torch.ones(self.dim, 1))
40
+ self.bias = nn.Parameter(torch.zeros(self.dim, 1))
41
+ if shape == 4:
42
+ self.weight = nn.Parameter(torch.ones(self.dim, 1, 1))
43
+ self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1))
44
+ else:
45
+ self.register_parameter('weight', None)
46
+ self.register_parameter('bias', None)
47
+
48
+ def forward(self, x):
49
+ """Returns the normalized tensor.
50
+
51
+ Args:
52
+ x : torch.Tensor
53
+ Tensor of size [N, C, K, S] or [N, C, L].
54
+ """
55
+ # x = N x C x K x S or N x C x L
56
+ # N x 1 x 1
57
+ # cln: mean,var N x 1 x K x S
58
+ # gln: mean,var N x 1 x 1
59
+ if x.dim() == 3:
60
+ mean = torch.mean(x, (1, 2), keepdim=True)
61
+ var = torch.mean((x - mean)**2, (1, 2), keepdim=True)
62
+ if self.elementwise_affine:
63
+ # yapf: disable
64
+ x = (self.weight * (x - mean) / torch.sqrt(var + self.eps)
65
+ + self.bias)
66
+ # yapf: enable
67
+ else:
68
+ x = (x - mean) / torch.sqrt(var + self.eps)
69
+
70
+ if x.dim() == 4:
71
+ mean = torch.mean(x, (1, 2, 3), keepdim=True)
72
+ var = torch.mean((x - mean)**2, (1, 2, 3), keepdim=True)
73
+ if self.elementwise_affine:
74
+ # yapf: disable
75
+ x = (self.weight * (x - mean) / torch.sqrt(var + self.eps)
76
+ + self.bias)
77
+ # yapf: enable
78
+ else:
79
+ x = (x - mean) / torch.sqrt(var + self.eps)
80
+ return x
81
+
82
+
83
+ class CumulativeLayerNorm(nn.LayerNorm):
84
+ """Calculate Cumulative Layer Normalization.
85
+
86
+ Args:
87
+ dim : int
88
+ Dimension that you want to normalize.
89
+ elementwise_affine : True
90
+ Learnable per-element affine parameters.
91
+
92
+ Example:
93
+ -------
94
+ >>> x = torch.randn(5, 10, 20)
95
+ >>> CLN = CumulativeLayerNorm(10)
96
+ >>> x_norm = CLN(x)
97
+ """
98
+
99
+ def __init__(self, dim, elementwise_affine=True):
100
+ super(CumulativeLayerNorm, self).__init__(
101
+ dim, elementwise_affine=elementwise_affine, eps=1e-8)
102
+
103
+ def forward(self, x):
104
+ """Returns the normalized tensor.
105
+
106
+ Args:
107
+ x : torch.Tensor
108
+ Tensor size [N, C, K, S] or [N, C, L]
109
+ """
110
+ # x: N x C x K x S or N x C x L
111
+ # N x K x S x C
112
+ if x.dim() == 4:
113
+ x = x.permute(0, 2, 3, 1).contiguous()
114
+ # N x K x S x C == only channel norm
115
+ x = super().forward(x)
116
+ # N x C x K x S
117
+ x = x.permute(0, 3, 1, 2).contiguous()
118
+ if x.dim() == 3:
119
+ x = torch.transpose(x, 1, 2)
120
+ # N x L x C == only channel norm
121
+ x = super().forward(x)
122
+ # N x C x L
123
+ x = torch.transpose(x, 1, 2)
124
+ return x
125
+
126
+
127
+ class Transpose(nn.Module):
128
+ """ Wrapper class of torch.transpose() for Sequential module. """
129
+
130
+ def __init__(self, shape: tuple):
131
+ super(Transpose, self).__init__()
132
+ self.shape = shape
133
+
134
+ def forward(self, x: Tensor) -> Tensor:
135
+ return x.transpose(*self.shape)
136
+
137
+
138
+ class DepthwiseConv1d(nn.Module):
139
+ """When groups == in_channels and out_channels == K * in_channels, where K is a positive integer,
140
+ this operation is termed in literature as depthwise convolution.
141
+
142
+ Args:
143
+ in_channels (int): Number of channels in the input
144
+ out_channels (int): Number of channels produced by the convolution
145
+ kernel_size (int or tuple): Size of the convolving kernel
146
+ stride (int, optional): Stride of the convolution. Default: 1
147
+ padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0
148
+ bias (bool, optional): If True, adds a learnable bias to the output. Default: True
149
+ Inputs: inputs
150
+ - **inputs** (batch, in_channels, time): Tensor containing input vector
151
+ Returns: outputs
152
+ - **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution.
153
+ """
154
+
155
+ def __init__(
156
+ self,
157
+ in_channels: int,
158
+ out_channels: int,
159
+ kernel_size: int,
160
+ stride: int = 1,
161
+ padding: int = 0,
162
+ bias: bool = False,
163
+ ) -> None:
164
+ super(DepthwiseConv1d, self).__init__()
165
+ assert out_channels % in_channels == 0, 'out_channels should be constant multiple of in_channels'
166
+ self.conv = nn.Conv1d(
167
+ in_channels=in_channels,
168
+ out_channels=out_channels,
169
+ kernel_size=kernel_size,
170
+ groups=in_channels,
171
+ stride=stride,
172
+ padding=padding,
173
+ bias=bias,
174
+ )
175
+
176
+ def forward(self, inputs: Tensor) -> Tensor:
177
+ return self.conv(inputs)
178
+
179
+
180
+ class ConvModule(nn.Module):
181
+ """
182
+ Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU).
183
+ This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution
184
+ to aid training deep models.
185
+
186
+ Args:
187
+ in_channels (int): Number of channels in the input
188
+ kernel_size (int or tuple, optional): Size of the convolving kernel Default: 17
189
+ dropout_p (float, optional): probability of dropout
190
+ Inputs: inputs
191
+ inputs (batch, time, dim): Tensor contains input sequences
192
+ Outputs: outputs
193
+ outputs (batch, time, dim): Tensor produces by conformer convolution module.
194
+ """
195
+
196
+ def __init__(
197
+ self,
198
+ in_channels: int,
199
+ kernel_size: int = 17,
200
+ expansion_factor: int = 2,
201
+ dropout_p: float = 0.1,
202
+ ) -> None:
203
+ super(ConvModule, self).__init__()
204
+ assert (
205
+ kernel_size - 1
206
+ ) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding"
207
+ assert expansion_factor == 2, 'Currently, Only Supports expansion_factor 2'
208
+
209
+ self.sequential = nn.Sequential(
210
+ Transpose(shape=(1, 2)),
211
+ DepthwiseConv1d(
212
+ in_channels,
213
+ in_channels,
214
+ kernel_size,
215
+ stride=1,
216
+ padding=(kernel_size - 1) // 2),
217
+ )
218
+
219
+ def forward(self, inputs: Tensor) -> Tensor:
220
+ return inputs + self.sequential(inputs).transpose(1, 2)
221
+
222
+
223
+ class DilatedDenseNet(nn.Module):
224
+
225
+ def __init__(self, depth=4, lorder=20, in_channels=64):
226
+ super(DilatedDenseNet, self).__init__()
227
+ self.depth = depth
228
+ self.in_channels = in_channels
229
+ self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.)
230
+ self.twidth = lorder * 2 - 1
231
+ self.kernel_size = (self.twidth, 1)
232
+ for i in range(self.depth):
233
+ dil = 2**i
234
+ pad_length = lorder + (dil - 1) * (lorder - 1) - 1
235
+ setattr(self, 'pad{}'.format(i + 1),
236
+ nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.))
237
+ setattr(
238
+ self, 'conv{}'.format(i + 1),
239
+ nn.Conv2d(
240
+ self.in_channels * (i + 1),
241
+ self.in_channels,
242
+ kernel_size=self.kernel_size,
243
+ dilation=(dil, 1),
244
+ groups=self.in_channels,
245
+ bias=False))
246
+ setattr(self, 'norm{}'.format(i + 1),
247
+ nn.InstanceNorm2d(in_channels, affine=True))
248
+ setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels))
249
+
250
+ def forward(self, x):
251
+ x = torch.unsqueeze(x, 1)
252
+ x_per = x.permute(0, 3, 2, 1)
253
+ skip = x_per
254
+ for i in range(self.depth):
255
+ out = getattr(self, 'pad{}'.format(i + 1))(skip)
256
+ out = getattr(self, 'conv{}'.format(i + 1))(out)
257
+ out = getattr(self, 'norm{}'.format(i + 1))(out)
258
+ out = getattr(self, 'prelu{}'.format(i + 1))(out)
259
+ skip = torch.cat([out, skip], dim=1)
260
+ out1 = out.permute(0, 3, 2, 1)
261
+ return out1.squeeze(1)
262
+
263
+
264
+ class FFConvMDilated(nn.Module):
265
+
266
+ def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1):
267
+ super().__init__()
268
+ self.mdl = nn.Sequential(
269
+ norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(),
270
+ DilatedDenseNet(depth=2, lorder=17, in_channels=dim_out),
271
+ nn.Dropout(dropout))
272
+
273
+ def forward(
274
+ self,
275
+ x,
276
+ ):
277
+ output = self.mdl(x)
278
+ return output
runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba, Inc. and its affiliates.
2
+
3
+ import torch as th
4
+ import torch.nn as nn
5
+ import torch.nn.functional as F
6
+
7
+
8
+ class UniDeepFsmn(nn.Module):
9
+
10
+ def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
11
+ super(UniDeepFsmn, self).__init__()
12
+
13
+ self.input_dim = input_dim
14
+ self.output_dim = output_dim
15
+ if lorder is None:
16
+ return
17
+ self.lorder = lorder
18
+ self.hidden_size = hidden_size
19
+ self.linear = nn.Linear(input_dim, hidden_size)
20
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
21
+ self.conv1 = nn.Conv2d(
22
+ output_dim,
23
+ output_dim, [lorder + lorder - 1, 1], [1, 1],
24
+ groups=output_dim,
25
+ bias=False)
26
+
27
+ def forward(self, input):
28
+ f1 = F.relu(self.linear(input))
29
+ p1 = self.project(f1)
30
+ x = th.unsqueeze(p1, 1)
31
+ x_per = x.permute(0, 3, 2, 1)
32
+ y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1])
33
+ out = x_per + self.conv1(y)
34
+ out1 = out.permute(0, 3, 2, 1)
35
+ return input + out1.squeeze()
36
+
37
+
38
+ class UniDeepFsmnDual(nn.Module):
39
+
40
+ def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None):
41
+ super(UniDeepFsmnDual, self).__init__()
42
+
43
+ self.input_dim = input_dim
44
+ self.output_dim = output_dim
45
+ if lorder is None:
46
+ return
47
+ self.lorder = lorder
48
+ self.hidden_size = hidden_size
49
+ self.linear = nn.Linear(input_dim, hidden_size)
50
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
51
+ self.conv1 = nn.Conv2d(
52
+ output_dim,
53
+ output_dim, [lorder + lorder - 1, 1], [1, 1],
54
+ groups=output_dim,
55
+ bias=False)
56
+ self.conv2 = nn.Conv2d(
57
+ output_dim,
58
+ output_dim, [lorder + lorder - 1, 1], [1, 1],
59
+ groups=output_dim // 4,
60
+ bias=False)
61
+
62
+ def forward(self, input):
63
+
64
+ f1 = F.relu(self.linear(input))
65
+ p1 = self.project(f1)
66
+ x = th.unsqueeze(p1, 1)
67
+ x_per = x.permute(0, 3, 2, 1)
68
+ y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1])
69
+ conv1_out = x_per + self.conv1(y)
70
+ z = F.pad(conv1_out, [0, 0, self.lorder - 1, self.lorder - 1])
71
+ out = conv1_out + self.conv2(z)
72
+ out1 = out.permute(0, 3, 2, 1)
73
+ return input + out1.squeeze()
74
+
75
+
76
+ class DilatedDenseNet(nn.Module):
77
+
78
+ def __init__(self, depth=4, lorder=20, in_channels=64):
79
+ super(DilatedDenseNet, self).__init__()
80
+ self.depth = depth
81
+ self.in_channels = in_channels
82
+ self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.)
83
+ self.twidth = lorder * 2 - 1
84
+ self.kernel_size = (self.twidth, 1)
85
+ for i in range(self.depth):
86
+ dil = 2**i
87
+ pad_length = lorder + (dil - 1) * (lorder - 1) - 1
88
+ setattr(self, 'pad{}'.format(i + 1),
89
+ nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.))
90
+ setattr(
91
+ self, 'conv{}'.format(i + 1),
92
+ nn.Conv2d(
93
+ self.in_channels * (i + 1),
94
+ self.in_channels,
95
+ kernel_size=self.kernel_size,
96
+ dilation=(dil, 1),
97
+ groups=self.in_channels,
98
+ bias=False))
99
+ setattr(self, 'norm{}'.format(i + 1),
100
+ nn.InstanceNorm2d(in_channels, affine=True))
101
+ setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels))
102
+
103
+ def forward(self, x):
104
+ skip = x
105
+ for i in range(self.depth):
106
+ out = getattr(self, 'pad{}'.format(i + 1))(skip)
107
+ out = getattr(self, 'conv{}'.format(i + 1))(out)
108
+ out = getattr(self, 'norm{}'.format(i + 1))(out)
109
+ out = getattr(self, 'prelu{}'.format(i + 1))(out)
110
+ skip = th.cat([out, skip], dim=1)
111
+ return out
112
+
113
+
114
+ class UniDeepFsmnDilated(nn.Module):
115
+
116
+ def __init__(self,
117
+ input_dim,
118
+ output_dim,
119
+ lorder=None,
120
+ hidden_size=None,
121
+ depth=2):
122
+ super(UniDeepFsmnDilated, self).__init__()
123
+
124
+ self.input_dim = input_dim
125
+ self.output_dim = output_dim
126
+ self.depth = depth
127
+ if lorder is None:
128
+ return
129
+ self.lorder = lorder
130
+ self.hidden_size = hidden_size
131
+ self.linear = nn.Linear(input_dim, hidden_size)
132
+ self.project = nn.Linear(hidden_size, output_dim, bias=False)
133
+ self.conv = DilatedDenseNet(
134
+ depth=self.depth, lorder=lorder, in_channels=output_dim)
135
+
136
+ def forward(self, input):
137
+ f1 = F.relu(self.linear(input))
138
+ p1 = self.project(f1)
139
+ x = th.unsqueeze(p1, 1)
140
+ x_per = x.permute(0, 3, 2, 1)
141
+ out = self.conv(x_per)
142
+ out1 = out.permute(0, 3, 2, 1)
143
+
144
+ return input + out1.squeeze()
runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2018 Northwestern Polytechnical University (author: Ke Wang)
2
+
3
+ from __future__ import absolute_import, division, print_function
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+
8
+
9
+ class CLayerNorm(nn.LayerNorm):
10
+ """Channel-wise layer normalization."""
11
+
12
+ def __init__(self, *args, **kwargs):
13
+ super(CLayerNorm, self).__init__(*args, **kwargs)
14
+
15
+ def forward(self, sample):
16
+ """Forward function.
17
+
18
+ Args:
19
+ sample: [batch_size, channels, length]
20
+ """
21
+ if sample.dim() != 3:
22
+ raise RuntimeError('{} only accept 3-D tensor as input'.format(
23
+ self.__name__))
24
+ # [N, C, T] -> [N, T, C]
25
+ sample = torch.transpose(sample, 1, 2)
26
+ # LayerNorm
27
+ sample = super().forward(sample)
28
+ # [N, T, C] -> [N, C, T]
29
+ sample = torch.transpose(sample, 1, 2)
30
+ return sample
31
+
32
+
33
+ class ILayerNorm(nn.InstanceNorm1d):
34
+ """Channel-wise layer normalization."""
35
+
36
+ def __init__(self, *args, **kwargs):
37
+ super(ILayerNorm, self).__init__(*args, **kwargs)
38
+
39
+ def forward(self, sample):
40
+ """Forward function.
41
+
42
+ Args:
43
+ sample: [batch_size, channels, length]
44
+ """
45
+ if sample.dim() != 3:
46
+ raise RuntimeError('{} only accept 3-D tensor as input'.format(
47
+ self.__name__))
48
+ # [N, C, T] -> [N, T, C]
49
+ sample = torch.transpose(sample, 1, 2)
50
+ # LayerNorm
51
+ sample = super().forward(sample)
52
+ # [N, T, C] -> [N, C, T]
53
+ sample = torch.transpose(sample, 1, 2)
54
+ return sample
55
+
56
+
57
+ class GLayerNorm(nn.Module):
58
+ """Global Layer Normalization for TasNet."""
59
+
60
+ def __init__(self, channels, eps=1e-5):
61
+ super(GLayerNorm, self).__init__()
62
+ self.eps = eps
63
+ self.norm_dim = channels
64
+ self.gamma = nn.Parameter(torch.Tensor(channels))
65
+ self.beta = nn.Parameter(torch.Tensor(channels))
66
+ # self.register_parameter('weight', self.gamma)
67
+ # self.register_parameter('bias', self.beta)
68
+ self.reset_parameters()
69
+
70
+ def reset_parameters(self):
71
+ nn.init.ones_(self.gamma)
72
+ nn.init.zeros_(self.beta)
73
+
74
+ def forward(self, sample):
75
+ """Forward function.
76
+
77
+ Args:
78
+ sample: [batch_size, channels, length]
79
+ """
80
+ if sample.dim() != 3:
81
+ raise RuntimeError('{} only accept 3-D tensor as input'.format(
82
+ self.__name__))
83
+ # [N, C, T] -> [N, T, C]
84
+ sample = torch.transpose(sample, 1, 2)
85
+ # Mean and variance [N, 1, 1]
86
+ mean = torch.mean(sample, (1, 2), keepdim=True)
87
+ var = torch.mean((sample - mean)**2, (1, 2), keepdim=True)
88
+ sample = (sample
89
+ - mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta
90
+ # [N, T, C] -> [N, C, T]
91
+ sample = torch.transpose(sample, 1, 2)
92
+ return sample
93
+
94
+
95
+ class _LayerNorm(nn.Module):
96
+ """Layer Normalization base class."""
97
+
98
+ def __init__(self, channel_size):
99
+ super(_LayerNorm, self).__init__()
100
+ self.channel_size = channel_size
101
+ self.gamma = nn.Parameter(torch.ones(channel_size), requires_grad=True)
102
+ self.beta = nn.Parameter(torch.zeros(channel_size), requires_grad=True)
103
+
104
+ def apply_gain_and_bias(self, normed_x):
105
+ """ Assumes input of size `[batch, chanel, *]`. """
106
+ return (self.gamma * normed_x.transpose(1, -1) + self.beta).transpose(
107
+ 1, -1)
108
+
109
+
110
+ class GlobLayerNorm(_LayerNorm):
111
+ """Global Layer Normalization (globLN)."""
112
+
113
+ def forward(self, x):
114
+ """ Applies forward pass.
115
+ Works for any input size > 2D.
116
+
117
+ Args:
118
+ x (:class:`torch.Tensor`): Shape `[batch, chan, *]`
119
+ Returns:
120
+ :class:`torch.Tensor`: gLN_x `[batch, chan, *]`
121
+ """
122
+ dims = list(range(1, len(x.shape)))
123
+ mean = x.mean(dim=dims, keepdim=True)
124
+ var = torch.pow(x - mean, 2).mean(dim=dims, keepdim=True)
125
+ return self.apply_gain_and_bias((x - mean) / (var + 1e-8).sqrt())