""" BS-RoFormer: separates a mixture into six stems with the Band-Split RoPE Transformer. The SW checkpoint estimates vocals, drums, bass, guitar, piano, and other in a single pass, so every stem is returned at once rather than one chosen ahead of time. """ import spaces # imported before torch, as ZeroGPU requires import tempfile from pathlib import Path import gradio as gr import numpy as np import soundfile as sf import torch import torchaudio import yaml from bs_roformer.inference import SafeLoaderWithTuple from bs_roformer.utils import demix_track, get_model_from_config from huggingface_hub import hf_hub_download from ml_collections import ConfigDict from pyharp import ModelCard, build_endpoint from pyharp.tags import Channels, SampleRate, Subcategory model_card = ModelCard( name="BS-RoFormer", description=( "Separates a mixture into vocals, drums, bass, guitar, piano, and other with the " "Band-Split RoPE Transformer, returning every stem along with an instrumental, which is " "the mixture without its vocals. Separation runs at 44.1 kHz in stereo, and the stems " "come back at the rate of the input, in one channel for a mono input. Repository: " "https://github.com/lucidrains/BS-RoFormer " "Paper: https://arxiv.org/abs/2309.02612" ), author="Wei-Tsung Lu, Ju-Chiang Wang, Qiuqiang Kong, Yun-Ning Hung (ByteDance)", tags=[Subcategory.STEM_SEPARATION, SampleRate(44100), Channels(2), "stems", "vocals", "drums", "bass", 'domain:music', ], ) # The checkpoint bs_roformer would fetch for its "roformer-model-bs-roformer-sw-by-jarredou" entry, # from the main branch of its repository, read here at a pinned revision instead MODEL_REPO = "enerjazzer/BS-ROFO-SW-Fixed" MODEL_REVISION = "a443a2985534b3bc815ef54a5d446c6a0390f974" # The checkpoint was trained on 44.1 kHz stereo, whatever rate the input arrives at MODEL_SR = 44100 # Presentation order. The checkpoint emits them as bass, drums, other, vocals, guitar, piano. STEMS = ["vocals", "drums", "bass", "guitar", "piano", "other"] device = "cuda" if torch.cuda.is_available() else "cpu" # A lossless input keeps its container. A lossy one comes back as WAV, since encoding the stems # lossily again would expose the artifacts of the first encode and add a second set. LOSSLESS = {".wav", ".flac"} # pyharp runs process_fn in a worker, which imports this file as __mp_main__ and starts with the # server, so the model is downloaded, built, and placed there alone, on its main thread, where # ZeroGPU's emulation answers the device queries its attention layers make if __name__ != "__main__": with open(hf_hub_download(MODEL_REPO, "BS-Rofo-SW-Fixed.yaml", revision=MODEL_REVISION)) as f: config = ConfigDict(yaml.load(f, Loader=SafeLoaderWithTuple)) model = get_model_from_config("bs_roformer", config) model.load_state_dict(torch.load( hf_hub_download(MODEL_REPO, "BS-Rofo-SW-Fixed.ckpt", revision=MODEL_REVISION), map_location="cpu", )) model.eval() model.to(device) # PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose # reservation would exceed that is refused rather than started. A shorter one is left to # ZeroGPU, which admits it when the caller's own allowance covers it. MAX_GPU_SECONDS = 900 def gpu_seconds(input_audio_path: str) -> float: # Measured at about 9 s for 30 s of audio, padded return 15 + 0.35 * sf.info(input_audio_path).duration def process_fn(input_audio_path: str): # Checked before the GPU is reserved, so a recording the model cannot take spends none of # the caller's ZeroGPU allowance if not input_audio_path: raise gr.Error("Provide the mixture to separate.") try: info = sf.info(input_audio_path) except Exception as exc: raise gr.Error(f"Could not read the audio file: {exc}") from exc if info.frames == 0: raise gr.Error("The audio file is empty.") num_channels = info.channels if num_channels > 2: raise gr.Error(f"Expected mono or stereo audio, but the input has {num_channels} channels.") if gpu_seconds(input_audio_path) > MAX_GPU_SECONDS: limit = (MAX_GPU_SECONDS - 15) / 0.35 / 60 raise gr.Error(f"Trim the recording to {int(limit)} minutes or less.") return separate(input_audio_path) @spaces.GPU(duration=gpu_seconds) @torch.inference_mode() def separate(input_audio_path: str): mix, sr = sf.read(input_audio_path, always_2d=True) num_samples, num_channels = mix.shape # The model only accepts stereo, so a mono input is duplicated and the stems folded back stereo = np.repeat(mix, 2, axis=1) if num_channels == 1 else mix mixture = torch.from_numpy(np.ascontiguousarray(stereo.T, dtype=np.float32)) if sr != MODEL_SR: mixture = torchaudio.functional.resample(mixture, sr, MODEL_SR) estimates, _ = demix_track(config, model, mixture, device) # The library derives its own instrumental from the first instrument, which for this # checkpoint is the bass rather than the vocals, so it is computed here instead estimates["instrumental"] = mixture.numpy() - estimates["vocals"] out_dir = Path(tempfile.mkdtemp()) stem = Path(input_audio_path).stem input_ext = Path(input_audio_path).suffix.lower() lossless = input_ext in LOSSLESS and input_ext != ".wav" output_paths = [] for name in STEMS + ["instrumental"]: audio = torch.from_numpy(np.ascontiguousarray(estimates[name], dtype=np.float32)) if sr != MODEL_SR: audio = torchaudio.functional.resample(audio, MODEL_SR, sr) # Resampling can shift the length by a sample or two audio = audio[:, :num_samples] if audio.shape[1] < num_samples: audio = torch.nn.functional.pad(audio, (0, num_samples - audio.shape[1])) audio = audio.mean(dim=0, keepdim=True) if num_channels == 1 else audio # Floating point in WAV, where FLAC takes 24 bits path = out_dir / f"{stem}_{name}{input_ext if lossless else '.wav'}" sf.write(path, audio.numpy().T, sr, subtype="PCM_24" if lossless else "FLOAT") output_paths.append(str(path)) return tuple(output_paths) # The processing worker imports this file, so the app must not be built there if __name__ == "__main__": with gr.Blocks() as demo: input_components = [ gr.Audio(type="filepath", label="Input Audio").set_info( "The mixture to separate, in mono or stereo." ), ] output_components = [ gr.Audio(type="filepath", label=f"{name.capitalize()}").set_info(info) for name, info in [ ("vocals", "The vocals."), ("drums", "The drums and percussion."), ("bass", "The bass."), ("guitar", "The guitar."), ("piano", "The piano."), ("other", "Everything not assigned to another stem."), ("instrumental", "The mixture without its vocals."), ] ] app = build_endpoint( model_card=model_card, input_components=input_components, output_components=output_components, process_fn=process_fn, ) demo.queue().launch(show_error=True, pwa=True, ssr_mode=False)