jennyxue commited on
Commit
b62a9ef
·
verified ·
1 Parent(s): 6b256c9

bf16 build of MossFormer2

Browse files
Files changed (3) hide show
  1. bf16/config.json +19 -0
  2. bf16/mlx_model.py +156 -0
  3. bf16/model.safetensors +3 -0
bf16/config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "sample_rate": 48000,
3
+ "win_len": 1920,
4
+ "win_inc": 384,
5
+ "fft_len": 1920,
6
+ "win_type": "hamming",
7
+ "num_mels": 60,
8
+ "preemphasis": 0.97,
9
+ "one_time_decode_length": 20,
10
+ "decode_window": 4,
11
+ "chunk_seconds": 4.0,
12
+ "chunk_overlap": 0.25,
13
+ "auto_chunk_threshold": 60.0,
14
+ "in_channels": 180,
15
+ "out_channels": 512,
16
+ "out_channels_final": 961,
17
+ "num_blocks": 24,
18
+ "model_type": "mossformer2_se"
19
+ }
bf16/mlx_model.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MossFormer2 SE 48K in MLX: speech enhancement (denoising) at 48 kHz, as ClearerVoice-Studio runs it.
2
+
3
+ The network is mlx-audio's port (mlx_audio.sts.models.mossformer2_se, mlx-audio 0.6.5) with two corrections that
4
+ make it match the original PyTorch model:
5
+
6
+ - rotary embeddings on interleaved pairs (rotary_embedding_torch, which the original uses); mlx-audio builds
7
+ nn.RoPE(traditional=False), which rotates the two halves instead
8
+ - Kaldi filterbanks computed exactly as torchaudio.compliance.kaldi.fbank: pre-emphasis of each frame's first
9
+ sample against itself (x0 - 0.97 x0), log floor at float32 epsilon, dither as an explicit argument (the original
10
+ uses dither=1.0; 0 makes the output deterministic)
11
+
12
+ import mlx_model
13
+ model = mlx_model.load("8bit") # a folder: config.json + model.safetensors
14
+ clean = mlx_model.enhance(model, "noisy.wav") # float32 numpy, 48 kHz (input resampled to 48 kHz)
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from pathlib import Path
20
+
21
+ import mlx.core as mx
22
+ import mlx.nn as nn
23
+ import numpy as np
24
+ from mlx.utils import tree_unflatten
25
+
26
+ MAX_WAV_VALUE = 32768.0
27
+ SAMPLE_RATE = 48000
28
+
29
+
30
+ def load(folder) -> "nn.Module":
31
+ from mlx_audio.sts.models.mossformer2_se.config import MossFormer2SEConfig
32
+ from mlx_audio.sts.models.mossformer2_se.mossformer2_se_wrapper import MossFormer2SE
33
+ folder = Path(folder)
34
+ conf = json.loads((folder / "config.json").read_text())
35
+ q = conf.pop("quantization_config", None)
36
+ conf.pop("model_type", None)
37
+ cfg = MossFormer2SEConfig.from_dict(conf)
38
+ model = MossFormer2SE(cfg)
39
+ for _, m in model.named_modules():
40
+ if isinstance(getattr(m, "rotary_pos_emb", None), nn.RoPE):
41
+ r = m.rotary_pos_emb
42
+ m.rotary_pos_emb = nn.RoPE(r.dims, traditional=True, base=r.base)
43
+ if q:
44
+ nn.quantize(model, group_size=q["group_size"], bits=q["bits"])
45
+ model.update(tree_unflatten(list(mx.load(str(folder / "model.safetensors")).items())))
46
+ mx.eval(model.parameters())
47
+ model.eval()
48
+ model.config = cfg
49
+ return model
50
+
51
+
52
+ def _hamming(n: int) -> mx.array:
53
+ k = mx.arange(n, dtype=mx.float32)
54
+ return 0.54 - 0.46 * mx.cos(2 * np.pi * k / (n - 1))
55
+
56
+
57
+ def fbank(x: mx.array, cfg, dither: float = 1.0) -> mx.array:
58
+ """torchaudio.compliance.kaldi.fbank(x, dither, frame_length=win_len, frame_shift=win_inc, num_mel_bins=num_mels,
59
+ sample_frequency=48000, window_type='hamming') with Kaldi's other defaults (snip_edges, remove_dc_offset,
60
+ preemphasis 0.97, power spectrum, low_freq 20, high_freq Nyquist)."""
61
+ from mlx_audio.dsp import get_mel_banks_kaldi
62
+ size, shift = cfg.win_len, cfg.win_inc
63
+ n = 1 + (x.shape[0] - size) // shift
64
+ idx = mx.arange(n)[:, None] * shift + mx.arange(size)[None, :]
65
+ frames = x[idx]
66
+ if dither:
67
+ frames = frames + mx.random.normal(frames.shape) * dither
68
+ frames = frames - mx.mean(frames, axis=1, keepdims=True)
69
+ prev = mx.concatenate([frames[:, :1], frames[:, :-1]], axis=1)
70
+ frames = (frames - 0.97 * prev) * _hamming(size)
71
+ pad = 1 << (size - 1).bit_length()
72
+ spec = mx.abs(mx.fft.rfft(frames, n=pad, axis=1)) ** 2
73
+ banks, _ = get_mel_banks_kaldi(cfg.num_mels, pad, float(SAMPLE_RATE), 20.0, 0.0)
74
+ banks = mx.pad(banks, [(0, 0), (0, 1)])
75
+ return mx.log(mx.maximum(spec @ banks.T, np.finfo(np.float32).eps))
76
+
77
+
78
+ def _deltas(x: mx.array) -> mx.array:
79
+ """torchaudio.functional.compute_deltas(win_length=5) over the last axis (replicate padding)."""
80
+ p = mx.concatenate([x[..., :1], x[..., :1], x, x[..., -1:], x[..., -1:]], axis=-1)
81
+ t = x.shape[-1]
82
+ return (-2 * p[..., 0:t] - p[..., 1:t + 1] + p[..., 3:t + 3] + 2 * p[..., 4:t + 4]) / 10.0
83
+
84
+
85
+ def _stft(x: mx.array, cfg) -> mx.array:
86
+ """The original's stft (torch.stft, center=False, hamming window of win_len, n_fft = fft_len): (frames, bins)."""
87
+ size, shift = cfg.win_len, cfg.win_inc
88
+ n = 1 + (x.shape[0] - size) // shift
89
+ idx = mx.arange(n)[:, None] * shift + mx.arange(size)[None, :]
90
+ return mx.fft.rfft(x[idx] * _hamming(size), n=cfg.fft_len, axis=1)
91
+
92
+
93
+ def _istft(spec: np.ndarray, cfg, length: int) -> np.ndarray:
94
+ """torch.istft(center=False, hamming window): overlap-add with window-square normalisation."""
95
+ size, shift = cfg.win_len, cfg.win_inc
96
+ w = np.array(_hamming(size))
97
+ frames = np.fft.irfft(spec, n=cfg.fft_len, axis=1)[:, :size] * w
98
+ n = frames.shape[0]
99
+ out = np.zeros(size + shift * (n - 1)); norm = np.zeros_like(out)
100
+ for i in range(n):
101
+ out[i * shift:i * shift + size] += frames[i]
102
+ norm[i * shift:i * shift + size] += w ** 2
103
+ nz = norm > 1e-11
104
+ out[nz] /= norm[nz]
105
+ return np.pad(out, (0, max(0, length - len(out))))[:length]
106
+
107
+
108
+ def _segment(model, audio: np.ndarray, dither: float) -> np.ndarray:
109
+ cfg = model.config
110
+ x = mx.array(audio.astype(np.float32))
111
+ f = fbank(x, cfg, dither)
112
+ d1 = _deltas(f.T); d2 = _deltas(d1)
113
+ feats = mx.concatenate([f, d1.T, d2.T], axis=1)[None]
114
+ mask = model(feats)[-1][0] # (frames, bins)
115
+ spec = _stft(x, cfg)
116
+ n = min(spec.shape[0], mask.shape[0])
117
+ masked = np.array(spec[:n] * mask[:n].astype(mx.float32))
118
+ return _istft(masked, cfg, len(audio))
119
+
120
+
121
+ def enhance(model, audio, sr: int | None = None, dither: float = 1.0) -> np.ndarray:
122
+ """Denoise `audio` (a path or a float waveform at `sr`); returns float32 at 48 kHz, the input's length. Inputs over
123
+ 20 s go through the original's 4 s windows with 3 s stride (the edges of each window dropped)."""
124
+ if isinstance(audio, (str, Path)):
125
+ import soundfile as sf
126
+ audio, sr = sf.read(str(audio), dtype="float32")
127
+ audio = np.asarray(audio, dtype=np.float32)
128
+ audio = audio.mean(1) if audio.ndim > 1 else audio
129
+ if sr and sr != SAMPLE_RATE:
130
+ import librosa
131
+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
132
+ cfg = model.config
133
+ length = len(audio)
134
+ x = audio * MAX_WAV_VALUE
135
+ if length <= SAMPLE_RATE * cfg.one_time_decode_length:
136
+ return (_segment(model, x, dither) / MAX_WAV_VALUE).astype(np.float32)
137
+ window = int(SAMPLE_RATE * cfg.decode_window)
138
+ stride = int(window * 0.75)
139
+ t = len(x)
140
+ if t < window:
141
+ x = np.concatenate([x, np.zeros(window - t)])
142
+ elif t < window + stride:
143
+ x = np.concatenate([x, np.zeros(window + stride - t)])
144
+ elif (t - window) % stride:
145
+ x = np.concatenate([x, np.zeros(t - (t - window) // stride * stride)])
146
+ out = np.zeros(len(x))
147
+ give = (window - stride) // 2
148
+ i = 0
149
+ while i + window <= len(x):
150
+ seg = _segment(model, x[i:i + window], dither)
151
+ if i == 0:
152
+ out[:window - give] = seg[:-give]
153
+ else:
154
+ out[i + give:i + window - give] = seg[give:-give]
155
+ i += stride
156
+ return (out[:length] / MAX_WAV_VALUE).astype(np.float32)
bf16/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24ec32b445b3c7e4134e9b3b2c4d19c146b14f35d45f4d576f0b55787a641947
3
+ size 110654001