Audio-to-Audio
MLX
Safetensors
moonbox
mlx-audio
mossformer2
clearervoice
speech-enhancement
denoising
48khz
Instructions to use moonbox-dev/MossFormer2_SE_48K-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use moonbox-dev/MossFormer2_SE_48K-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download moonbox-dev/MossFormer2_SE_48K-MLX --local-dir MossFormer2_SE_48K-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
6bit build of MossFormer2
Browse files- 6bit/config.json +23 -0
- 6bit/mlx_model.py +156 -0
- 6bit/model.safetensors +3 -0
6bit/config.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"sample_rate": 48000,
|
| 3 |
+
"win_len": 1920,
|
| 4 |
+
"win_inc": 384,
|
| 5 |
+
"fft_len": 1920,
|
| 6 |
+
"win_type": "hamming",
|
| 7 |
+
"num_mels": 60,
|
| 8 |
+
"preemphasis": 0.97,
|
| 9 |
+
"one_time_decode_length": 20,
|
| 10 |
+
"decode_window": 4,
|
| 11 |
+
"chunk_seconds": 4.0,
|
| 12 |
+
"chunk_overlap": 0.25,
|
| 13 |
+
"auto_chunk_threshold": 60.0,
|
| 14 |
+
"in_channels": 180,
|
| 15 |
+
"out_channels": 512,
|
| 16 |
+
"out_channels_final": 961,
|
| 17 |
+
"num_blocks": 24,
|
| 18 |
+
"model_type": "mossformer2_se",
|
| 19 |
+
"quantization_config": {
|
| 20 |
+
"group_size": 64,
|
| 21 |
+
"bits": 6
|
| 22 |
+
}
|
| 23 |
+
}
|
6bit/mlx_model.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MossFormer2 SE 48K in MLX: speech enhancement (denoising) at 48 kHz, as ClearerVoice-Studio runs it.
|
| 2 |
+
|
| 3 |
+
The network is mlx-audio's port (mlx_audio.sts.models.mossformer2_se, mlx-audio 0.6.5) with two corrections that
|
| 4 |
+
make it match the original PyTorch model:
|
| 5 |
+
|
| 6 |
+
- rotary embeddings on interleaved pairs (rotary_embedding_torch, which the original uses); mlx-audio builds
|
| 7 |
+
nn.RoPE(traditional=False), which rotates the two halves instead
|
| 8 |
+
- Kaldi filterbanks computed exactly as torchaudio.compliance.kaldi.fbank: pre-emphasis of each frame's first
|
| 9 |
+
sample against itself (x0 - 0.97 x0), log floor at float32 epsilon, dither as an explicit argument (the original
|
| 10 |
+
uses dither=1.0; 0 makes the output deterministic)
|
| 11 |
+
|
| 12 |
+
import mlx_model
|
| 13 |
+
model = mlx_model.load("8bit") # a folder: config.json + model.safetensors
|
| 14 |
+
clean = mlx_model.enhance(model, "noisy.wav") # float32 numpy, 48 kHz (input resampled to 48 kHz)
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import json
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import mlx.core as mx
|
| 22 |
+
import mlx.nn as nn
|
| 23 |
+
import numpy as np
|
| 24 |
+
from mlx.utils import tree_unflatten
|
| 25 |
+
|
| 26 |
+
MAX_WAV_VALUE = 32768.0
|
| 27 |
+
SAMPLE_RATE = 48000
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def load(folder) -> "nn.Module":
|
| 31 |
+
from mlx_audio.sts.models.mossformer2_se.config import MossFormer2SEConfig
|
| 32 |
+
from mlx_audio.sts.models.mossformer2_se.mossformer2_se_wrapper import MossFormer2SE
|
| 33 |
+
folder = Path(folder)
|
| 34 |
+
conf = json.loads((folder / "config.json").read_text())
|
| 35 |
+
q = conf.pop("quantization_config", None)
|
| 36 |
+
conf.pop("model_type", None)
|
| 37 |
+
cfg = MossFormer2SEConfig.from_dict(conf)
|
| 38 |
+
model = MossFormer2SE(cfg)
|
| 39 |
+
for _, m in model.named_modules():
|
| 40 |
+
if isinstance(getattr(m, "rotary_pos_emb", None), nn.RoPE):
|
| 41 |
+
r = m.rotary_pos_emb
|
| 42 |
+
m.rotary_pos_emb = nn.RoPE(r.dims, traditional=True, base=r.base)
|
| 43 |
+
if q:
|
| 44 |
+
nn.quantize(model, group_size=q["group_size"], bits=q["bits"])
|
| 45 |
+
model.update(tree_unflatten(list(mx.load(str(folder / "model.safetensors")).items())))
|
| 46 |
+
mx.eval(model.parameters())
|
| 47 |
+
model.eval()
|
| 48 |
+
model.config = cfg
|
| 49 |
+
return model
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _hamming(n: int) -> mx.array:
|
| 53 |
+
k = mx.arange(n, dtype=mx.float32)
|
| 54 |
+
return 0.54 - 0.46 * mx.cos(2 * np.pi * k / (n - 1))
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def fbank(x: mx.array, cfg, dither: float = 1.0) -> mx.array:
|
| 58 |
+
"""torchaudio.compliance.kaldi.fbank(x, dither, frame_length=win_len, frame_shift=win_inc, num_mel_bins=num_mels,
|
| 59 |
+
sample_frequency=48000, window_type='hamming') with Kaldi's other defaults (snip_edges, remove_dc_offset,
|
| 60 |
+
preemphasis 0.97, power spectrum, low_freq 20, high_freq Nyquist)."""
|
| 61 |
+
from mlx_audio.dsp import get_mel_banks_kaldi
|
| 62 |
+
size, shift = cfg.win_len, cfg.win_inc
|
| 63 |
+
n = 1 + (x.shape[0] - size) // shift
|
| 64 |
+
idx = mx.arange(n)[:, None] * shift + mx.arange(size)[None, :]
|
| 65 |
+
frames = x[idx]
|
| 66 |
+
if dither:
|
| 67 |
+
frames = frames + mx.random.normal(frames.shape) * dither
|
| 68 |
+
frames = frames - mx.mean(frames, axis=1, keepdims=True)
|
| 69 |
+
prev = mx.concatenate([frames[:, :1], frames[:, :-1]], axis=1)
|
| 70 |
+
frames = (frames - 0.97 * prev) * _hamming(size)
|
| 71 |
+
pad = 1 << (size - 1).bit_length()
|
| 72 |
+
spec = mx.abs(mx.fft.rfft(frames, n=pad, axis=1)) ** 2
|
| 73 |
+
banks, _ = get_mel_banks_kaldi(cfg.num_mels, pad, float(SAMPLE_RATE), 20.0, 0.0)
|
| 74 |
+
banks = mx.pad(banks, [(0, 0), (0, 1)])
|
| 75 |
+
return mx.log(mx.maximum(spec @ banks.T, np.finfo(np.float32).eps))
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _deltas(x: mx.array) -> mx.array:
|
| 79 |
+
"""torchaudio.functional.compute_deltas(win_length=5) over the last axis (replicate padding)."""
|
| 80 |
+
p = mx.concatenate([x[..., :1], x[..., :1], x, x[..., -1:], x[..., -1:]], axis=-1)
|
| 81 |
+
t = x.shape[-1]
|
| 82 |
+
return (-2 * p[..., 0:t] - p[..., 1:t + 1] + p[..., 3:t + 3] + 2 * p[..., 4:t + 4]) / 10.0
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _stft(x: mx.array, cfg) -> mx.array:
|
| 86 |
+
"""The original's stft (torch.stft, center=False, hamming window of win_len, n_fft = fft_len): (frames, bins)."""
|
| 87 |
+
size, shift = cfg.win_len, cfg.win_inc
|
| 88 |
+
n = 1 + (x.shape[0] - size) // shift
|
| 89 |
+
idx = mx.arange(n)[:, None] * shift + mx.arange(size)[None, :]
|
| 90 |
+
return mx.fft.rfft(x[idx] * _hamming(size), n=cfg.fft_len, axis=1)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _istft(spec: np.ndarray, cfg, length: int) -> np.ndarray:
|
| 94 |
+
"""torch.istft(center=False, hamming window): overlap-add with window-square normalisation."""
|
| 95 |
+
size, shift = cfg.win_len, cfg.win_inc
|
| 96 |
+
w = np.array(_hamming(size))
|
| 97 |
+
frames = np.fft.irfft(spec, n=cfg.fft_len, axis=1)[:, :size] * w
|
| 98 |
+
n = frames.shape[0]
|
| 99 |
+
out = np.zeros(size + shift * (n - 1)); norm = np.zeros_like(out)
|
| 100 |
+
for i in range(n):
|
| 101 |
+
out[i * shift:i * shift + size] += frames[i]
|
| 102 |
+
norm[i * shift:i * shift + size] += w ** 2
|
| 103 |
+
nz = norm > 1e-11
|
| 104 |
+
out[nz] /= norm[nz]
|
| 105 |
+
return np.pad(out, (0, max(0, length - len(out))))[:length]
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _segment(model, audio: np.ndarray, dither: float) -> np.ndarray:
|
| 109 |
+
cfg = model.config
|
| 110 |
+
x = mx.array(audio.astype(np.float32))
|
| 111 |
+
f = fbank(x, cfg, dither)
|
| 112 |
+
d1 = _deltas(f.T); d2 = _deltas(d1)
|
| 113 |
+
feats = mx.concatenate([f, d1.T, d2.T], axis=1)[None]
|
| 114 |
+
mask = model(feats)[-1][0] # (frames, bins)
|
| 115 |
+
spec = _stft(x, cfg)
|
| 116 |
+
n = min(spec.shape[0], mask.shape[0])
|
| 117 |
+
masked = np.array(spec[:n] * mask[:n].astype(mx.float32))
|
| 118 |
+
return _istft(masked, cfg, len(audio))
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def enhance(model, audio, sr: int | None = None, dither: float = 1.0) -> np.ndarray:
|
| 122 |
+
"""Denoise `audio` (a path or a float waveform at `sr`); returns float32 at 48 kHz, the input's length. Inputs over
|
| 123 |
+
20 s go through the original's 4 s windows with 3 s stride (the edges of each window dropped)."""
|
| 124 |
+
if isinstance(audio, (str, Path)):
|
| 125 |
+
import soundfile as sf
|
| 126 |
+
audio, sr = sf.read(str(audio), dtype="float32")
|
| 127 |
+
audio = np.asarray(audio, dtype=np.float32)
|
| 128 |
+
audio = audio.mean(1) if audio.ndim > 1 else audio
|
| 129 |
+
if sr and sr != SAMPLE_RATE:
|
| 130 |
+
import librosa
|
| 131 |
+
audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
|
| 132 |
+
cfg = model.config
|
| 133 |
+
length = len(audio)
|
| 134 |
+
x = audio * MAX_WAV_VALUE
|
| 135 |
+
if length <= SAMPLE_RATE * cfg.one_time_decode_length:
|
| 136 |
+
return (_segment(model, x, dither) / MAX_WAV_VALUE).astype(np.float32)
|
| 137 |
+
window = int(SAMPLE_RATE * cfg.decode_window)
|
| 138 |
+
stride = int(window * 0.75)
|
| 139 |
+
t = len(x)
|
| 140 |
+
if t < window:
|
| 141 |
+
x = np.concatenate([x, np.zeros(window - t)])
|
| 142 |
+
elif t < window + stride:
|
| 143 |
+
x = np.concatenate([x, np.zeros(window + stride - t)])
|
| 144 |
+
elif (t - window) % stride:
|
| 145 |
+
x = np.concatenate([x, np.zeros(t - (t - window) // stride * stride)])
|
| 146 |
+
out = np.zeros(len(x))
|
| 147 |
+
give = (window - stride) // 2
|
| 148 |
+
i = 0
|
| 149 |
+
while i + window <= len(x):
|
| 150 |
+
seg = _segment(model, x[i:i + window], dither)
|
| 151 |
+
if i == 0:
|
| 152 |
+
out[:window - give] = seg[:-give]
|
| 153 |
+
else:
|
| 154 |
+
out[i + give:i + window - give] = seg[give:-give]
|
| 155 |
+
i += stride
|
| 156 |
+
return (out[:length] / MAX_WAV_VALUE).astype(np.float32)
|
6bit/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6f3698c77ecb141c6f0e74eb65ac4bc861619ca0bbfd36e9503303700486b178
|
| 3 |
+
size 56537536
|