NINA / enhance.py
Malina
Claude Opus 4.8
NINA Phase 2: vocal isolation + speaker diarization + name attribution
e2c96b8
Raw History Blame
3.38 kB
# ─────────────────────────────────────────────
# NINA β€” vocal isolation (torchaudio Hybrid Demucs)
# ─────────────────────────────────────────────
# Separates the singing/speaking voice from music + noise so Whisper can read
# speech that's otherwise masked (e.g. a concert recording). Uses torchaudio's
# bundled Hybrid Demucs so we don't pull the standalone `demucs` package (which
# pins older torch). Everything is best-effort: any failure returns None and the
# caller keeps the original audio.
import numpy as np
from config import (
DEMUCS_BUNDLE, ISOLATION_CHUNK_SEC, ISOLATION_MAX_SECONDS, SAMPLE_RATE,
)
_model = None
_model_sr = None
_failed = False
def _get_model():
global _model, _model_sr, _failed
if _failed:
return None
if _model is None:
try:
import torch # noqa: F401
import torchaudio
bundle = getattr(torchaudio.pipelines, DEMUCS_BUNDLE)
_model = bundle.get_model().to("cpu").eval()
_model_sr = bundle.sample_rate
print(f"[NINA] Vocal isolation ready ({DEMUCS_BUNDLE}, sr={_model_sr}).")
except Exception as e:
_failed = True
print(f"[NINA] Vocal isolation unavailable: {e} β€” keeping original audio.")
return None
return _model
def isolate_vocals(audio: np.ndarray, sr: int = SAMPLE_RATE):
"""Return a vocals-only mono waveform at `sr`, or None if unavailable/failed.
Timing is preserved (same length/timeline as the input), so transcript
timestamps and diarization stay aligned.
"""
try:
if audio is None or len(audio) == 0:
return None
if len(audio) / sr > ISOLATION_MAX_SECONDS:
print("[NINA] Audio too long for vocal isolation β€” skipping.")
return None
model = _get_model()
if model is None:
return None
import torch
import torchaudio
with torch.no_grad():
wav = torch.from_numpy(np.ascontiguousarray(audio, dtype=np.float32)).unsqueeze(0)
wav = torchaudio.functional.resample(wav, sr, _model_sr)
wav = wav.repeat(2, 1) # Hybrid Demucs expects stereo
ref = wav.mean(0)
mean, std = ref.mean(), ref.std() + 1e-8
wav = (wav - mean) / std # normalize, per Demucs
voc_idx = model.sources.index("vocals")
chunk = max(1, int(ISOLATION_CHUNK_SEC * _model_sr))
pieces = []
for s in range(0, wav.shape[1], chunk): # chunk to bound memory
seg = wav[:, s:s + chunk]
out = model(seg.unsqueeze(0))[0] # (sources, channels, length)
pieces.append(out[voc_idx]) # (channels, length)
vocals = torch.cat(pieces, dim=1) * std + mean
vocals = vocals.mean(0, keepdim=True) # β†’ mono
vocals = torchaudio.functional.resample(vocals, _model_sr, sr)
return vocals.squeeze(0).cpu().numpy().astype(np.float32)
except Exception as e:
print(f"[NINA] Vocal isolation failed: {e}")
return None