Spaces:
Running on Zero
Running on Zero
Download app.py from teamup-tech/BS-RoFormer: direct link, hf CLI and curl.
- Browser
- Download file 7.43 kB
-
https://huggingface.co/spaces/teamup-tech/BS-RoFormer/resolve/main/app.py
- Command line
-
hf download hf://spaces/teamup-tech/BS-RoFormer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/teamup-tech/BS-RoFormer/resolve/main/app.py
7.43 kB
| """ | |
| BS-RoFormer: separates a mixture into six stems with the Band-Split RoPE Transformer. | |
| The SW checkpoint estimates vocals, drums, bass, guitar, piano, and other in a single pass, so | |
| every stem is returned at once rather than one chosen ahead of time. | |
| """ | |
| import spaces # imported before torch, as ZeroGPU requires | |
| import tempfile | |
| from pathlib import Path | |
| import gradio as gr | |
| import numpy as np | |
| import soundfile as sf | |
| import torch | |
| import torchaudio | |
| import yaml | |
| from bs_roformer.inference import SafeLoaderWithTuple | |
| from bs_roformer.utils import demix_track, get_model_from_config | |
| from huggingface_hub import hf_hub_download | |
| from ml_collections import ConfigDict | |
| from pyharp import ModelCard, build_endpoint | |
| from pyharp.tags import Channels, SampleRate, Subcategory | |
| model_card = ModelCard( | |
| name="BS-RoFormer", | |
| description=( | |
| "Separates a mixture into vocals, drums, bass, guitar, piano, and other with the " | |
| "Band-Split RoPE Transformer, returning every stem along with an instrumental, which is " | |
| "the mixture without its vocals. Separation runs at 44.1 kHz in stereo, and the stems " | |
| "come back at the rate of the input, in one channel for a mono input. Repository: " | |
| "https://github.com/lucidrains/BS-RoFormer " | |
| "Paper: https://arxiv.org/abs/2309.02612" | |
| ), | |
| author="Wei-Tsung Lu, Ju-Chiang Wang, Qiuqiang Kong, Yun-Ning Hung (ByteDance)", | |
| tags=[Subcategory.STEM_SEPARATION, SampleRate(44100), Channels(2), "stems", "vocals", "drums", | |
| "bass", | |
| 'domain:music', | |
| ], | |
| ) | |
| # The checkpoint bs_roformer would fetch for its "roformer-model-bs-roformer-sw-by-jarredou" entry, | |
| # from the main branch of its repository, read here at a pinned revision instead | |
| MODEL_REPO = "enerjazzer/BS-ROFO-SW-Fixed" | |
| MODEL_REVISION = "a443a2985534b3bc815ef54a5d446c6a0390f974" | |
| # The checkpoint was trained on 44.1 kHz stereo, whatever rate the input arrives at | |
| MODEL_SR = 44100 | |
| # Presentation order. The checkpoint emits them as bass, drums, other, vocals, guitar, piano. | |
| STEMS = ["vocals", "drums", "bass", "guitar", "piano", "other"] | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| # A lossless input keeps its container. A lossy one comes back as WAV, since encoding the stems | |
| # lossily again would expose the artifacts of the first encode and add a second set. | |
| LOSSLESS = {".wav", ".flac"} | |
| # pyharp runs process_fn in a worker, which imports this file as __mp_main__ and starts with the | |
| # server, so the model is downloaded, built, and placed there alone, on its main thread, where | |
| # ZeroGPU's emulation answers the device queries its attention layers make | |
| if __name__ != "__main__": | |
| with open(hf_hub_download(MODEL_REPO, "BS-Rofo-SW-Fixed.yaml", revision=MODEL_REVISION)) as f: | |
| config = ConfigDict(yaml.load(f, Loader=SafeLoaderWithTuple)) | |
| model = get_model_from_config("bs_roformer", config) | |
| model.load_state_dict(torch.load( | |
| hf_hub_download(MODEL_REPO, "BS-Rofo-SW-Fixed.ckpt", revision=MODEL_REVISION), | |
| map_location="cpu", | |
| )) | |
| model.eval() | |
| model.to(device) | |
| # PyHARP stops a job that runs past its timeout_s, 15 minutes by default, so a request whose | |
| # reservation would exceed that is refused rather than started. A shorter one is left to | |
| # ZeroGPU, which admits it when the caller's own allowance covers it. | |
| MAX_GPU_SECONDS = 900 | |
| def gpu_seconds(input_audio_path: str) -> float: | |
| # Measured at about 9 s for 30 s of audio, padded | |
| return 15 + 0.35 * sf.info(input_audio_path).duration | |
| def process_fn(input_audio_path: str): | |
| # Checked before the GPU is reserved, so a recording the model cannot take spends none of | |
| # the caller's ZeroGPU allowance | |
| if not input_audio_path: | |
| raise gr.Error("Provide the mixture to separate.") | |
| try: | |
| info = sf.info(input_audio_path) | |
| except Exception as exc: | |
| raise gr.Error(f"Could not read the audio file: {exc}") from exc | |
| if info.frames == 0: | |
| raise gr.Error("The audio file is empty.") | |
| num_channels = info.channels | |
| if num_channels > 2: | |
| raise gr.Error(f"Expected mono or stereo audio, but the input has {num_channels} channels.") | |
| if gpu_seconds(input_audio_path) > MAX_GPU_SECONDS: | |
| limit = (MAX_GPU_SECONDS - 15) / 0.35 / 60 | |
| raise gr.Error(f"Trim the recording to {int(limit)} minutes or less.") | |
| return separate(input_audio_path) | |
| def separate(input_audio_path: str): | |
| mix, sr = sf.read(input_audio_path, always_2d=True) | |
| num_samples, num_channels = mix.shape | |
| # The model only accepts stereo, so a mono input is duplicated and the stems folded back | |
| stereo = np.repeat(mix, 2, axis=1) if num_channels == 1 else mix | |
| mixture = torch.from_numpy(np.ascontiguousarray(stereo.T, dtype=np.float32)) | |
| if sr != MODEL_SR: | |
| mixture = torchaudio.functional.resample(mixture, sr, MODEL_SR) | |
| estimates, _ = demix_track(config, model, mixture, device) | |
| # The library derives its own instrumental from the first instrument, which for this | |
| # checkpoint is the bass rather than the vocals, so it is computed here instead | |
| estimates["instrumental"] = mixture.numpy() - estimates["vocals"] | |
| out_dir = Path(tempfile.mkdtemp()) | |
| stem = Path(input_audio_path).stem | |
| input_ext = Path(input_audio_path).suffix.lower() | |
| lossless = input_ext in LOSSLESS and input_ext != ".wav" | |
| output_paths = [] | |
| for name in STEMS + ["instrumental"]: | |
| audio = torch.from_numpy(np.ascontiguousarray(estimates[name], dtype=np.float32)) | |
| if sr != MODEL_SR: | |
| audio = torchaudio.functional.resample(audio, MODEL_SR, sr) | |
| # Resampling can shift the length by a sample or two | |
| audio = audio[:, :num_samples] | |
| if audio.shape[1] < num_samples: | |
| audio = torch.nn.functional.pad(audio, (0, num_samples - audio.shape[1])) | |
| audio = audio.mean(dim=0, keepdim=True) if num_channels == 1 else audio | |
| # Floating point in WAV, where FLAC takes 24 bits | |
| path = out_dir / f"{stem}_{name}{input_ext if lossless else '.wav'}" | |
| sf.write(path, audio.numpy().T, sr, subtype="PCM_24" if lossless else "FLOAT") | |
| output_paths.append(str(path)) | |
| return tuple(output_paths) | |
| # The processing worker imports this file, so the app must not be built there | |
| if __name__ == "__main__": | |
| with gr.Blocks() as demo: | |
| input_components = [ | |
| gr.Audio(type="filepath", label="Input Audio").set_info( | |
| "The mixture to separate, in mono or stereo." | |
| ), | |
| ] | |
| output_components = [ | |
| gr.Audio(type="filepath", label=f"{name.capitalize()}").set_info(info) | |
| for name, info in [ | |
| ("vocals", "The vocals."), | |
| ("drums", "The drums and percussion."), | |
| ("bass", "The bass."), | |
| ("guitar", "The guitar."), | |
| ("piano", "The piano."), | |
| ("other", "Everything not assigned to another stem."), | |
| ("instrumental", "The mixture without its vocals."), | |
| ] | |
| ] | |
| app = build_endpoint( | |
| model_card=model_card, | |
| input_components=input_components, | |
| output_components=output_components, | |
| process_fn=process_fn, | |
| ) | |
| demo.queue().launch(show_error=True, pwa=True, ssr_mode=False) | |