LING commited on
Commit
c5c143f
·
1 Parent(s): 8726604

Deploy Stable Audio 3 editing endpoint

Browse files
Files changed (8) hide show
  1. .gitignore +4 -0
  2. README.md +11 -3
  3. SOURCES.md +21 -0
  4. THIRD_PARTY_NOTICES.md +27 -0
  5. app.py +158 -0
  6. model.json +29 -0
  7. requirements.txt +25 -0
  8. stable_audio_runtime.py +164 -0
.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ __pycache__/
2
+ *.py[cod]
3
+ .DS_Store
4
+ .ruff_cache/
README.md CHANGED
@@ -1,15 +1,23 @@
1
  ---
2
  title: Stable Audio 3
3
- emoji: 👁
4
  colorFrom: indigo
5
  colorTo: gray
6
  sdk: gradio
7
- sdk_version: 6.20.0
8
  python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
  license: other
12
  short_description: Edit, inpaint, and continue music audio from text prompts
 
 
13
  ---
14
 
15
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
1
  ---
2
  title: Stable Audio 3
3
+ emoji: 🎛️
4
  colorFrom: indigo
5
  colorTo: gray
6
  sdk: gradio
7
+ sdk_version: 5.28.0
8
  python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
  license: other
12
  short_description: Edit, inpaint, and continue music audio from text prompts
13
+ models:
14
+ - stabilityai/stable-audio-3-small-music
15
  ---
16
 
17
+ HARP endpoint for editing music with Stable Audio 3 Small-Music. The endpoint
18
+ supports prompt-guided restyling, region inpainting, and continuation of an
19
+ uploaded music clip.
20
+
21
+ Model access is governed by the Stability AI Community License and the Gemma
22
+ Terms of Use. Source and license details are recorded in `SOURCES.md` and
23
+ `THIRD_PARTY_NOTICES.md`.
SOURCES.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Sources
2
+
3
+ ## Stable Audio 3
4
+
5
+ - Source: https://github.com/Stability-AI/stable-audio-3
6
+ - Source revision: `124e8a799f57a1f665495ecb72e547d0a62867f1`
7
+ - Source license: MIT
8
+ - Model: https://huggingface.co/stabilityai/stable-audio-3-small-music
9
+ - Model revision: `0fef1392cd842149a2b6d445e181c97608faac06`
10
+ - Model license: Stability AI Community License
11
+ - Paper: https://arxiv.org/abs/2605.17991
12
+
13
+ ## Runtime
14
+
15
+ - Package: `stable-audio-tools==0.0.20`
16
+ - Source: https://github.com/Stability-AI/stable-audio-tools
17
+
18
+ The Space installs the runtime package without dependencies because its
19
+ published metadata fixes an older Torch version. Runtime dependencies are
20
+ declared separately in `requirements.txt` to remain compatible with
21
+ Hugging Face ZeroGPU and pyHARP.
THIRD_PARTY_NOTICES.md ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Third-Party Notices
2
+
3
+ ## Stable Audio 3 Source Code
4
+
5
+ The Stable Audio 3 source code is provided by Stability AI under the MIT
6
+ License:
7
+
8
+ https://github.com/Stability-AI/stable-audio-3
9
+
10
+ ## Stable Audio 3 Small-Music Weights
11
+
12
+ The model weights are governed by the Stability AI Community License:
13
+
14
+ https://huggingface.co/stabilityai/stable-audio-3-small-music
15
+
16
+ https://stability.ai/license
17
+
18
+ Access to the model repository requires accepting its terms on Hugging Face.
19
+
20
+ ## T5Gemma
21
+
22
+ Stable Audio 3 Small-Music includes a T5Gemma text-conditioning component
23
+ redistributed under the Gemma Terms of Use:
24
+
25
+ https://huggingface.co/google/t5gemma-b-b-ul2
26
+
27
+ https://ai.google.dev/gemma/terms
app.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import gradio as gr
4
+ import soundfile as sf
5
+
6
+ try:
7
+ import spaces
8
+ except ImportError:
9
+ class spaces:
10
+ class GPU:
11
+ def __init__(self, func=None, duration=180):
12
+ self.func = func
13
+
14
+ def __call__(self, *args, **kwargs):
15
+ if self.func is not None:
16
+ return self.func(*args, **kwargs)
17
+ return args[0]
18
+
19
+ from pyharp import ModelCard, build_endpoint
20
+
21
+ from stable_audio_runtime import edit_audio
22
+
23
+ MIN_AUDIO_SECONDS = 5
24
+ MAX_AUDIO_SECONDS = 30
25
+
26
+ model_card = ModelCard(
27
+ name="Stable Audio 3",
28
+ description=(
29
+ "Restyle, inpaint, or continue music audio using a text prompt."
30
+ ),
31
+ author="Stability AI",
32
+ tags=[
33
+ "audio-generation",
34
+ "music-editing",
35
+ "audio-inpainting",
36
+ "audio-continuation",
37
+ ],
38
+ )
39
+
40
+
41
+ def _audio_duration(path: str | None) -> float:
42
+ if not path:
43
+ raise gr.Error("Please upload a music clip.")
44
+ try:
45
+ duration = float(sf.info(path).duration)
46
+ except Exception as exc:
47
+ raise gr.Error(f"Could not read the audio file: {exc}") from exc
48
+ if duration < MIN_AUDIO_SECONDS:
49
+ raise gr.Error(f"Audio must be at least {MIN_AUDIO_SECONDS} seconds.")
50
+ if duration > MAX_AUDIO_SECONDS:
51
+ raise gr.Error(f"Audio must be no longer than {MAX_AUDIO_SECONDS} seconds.")
52
+ return duration
53
+
54
+
55
+ @spaces.GPU(duration=180)
56
+ def process_fn(
57
+ audio_path: str | None,
58
+ prompt: str,
59
+ mode: str,
60
+ edit_start: float,
61
+ edit_end: float,
62
+ continuation_length: float,
63
+ strength: float,
64
+ seed: int,
65
+ ) -> str:
66
+ duration = _audio_duration(audio_path)
67
+ prompt = (prompt or "").strip()
68
+ if not prompt:
69
+ raise gr.Error("Please enter an editing prompt.")
70
+
71
+ if mode == "Inpaint" and (
72
+ edit_start < 0 or edit_end > duration or edit_end <= edit_start
73
+ ):
74
+ raise gr.Error(
75
+ "The inpaint region must be inside the uploaded clip, "
76
+ "with the end after the start."
77
+ )
78
+
79
+ try:
80
+ return edit_audio(
81
+ audio_path=audio_path,
82
+ prompt=prompt,
83
+ mode=mode,
84
+ edit_start=float(edit_start),
85
+ edit_end=float(edit_end),
86
+ continuation_length=float(continuation_length),
87
+ strength=float(strength),
88
+ seed=int(seed),
89
+ )
90
+ except gr.Error:
91
+ raise
92
+ except Exception as exc:
93
+ raise gr.Error(f"Stable Audio 3 inference failed: {exc}") from exc
94
+
95
+
96
+ with gr.Blocks(title="Stable Audio 3") as demo:
97
+ input_components = [
98
+ gr.Audio(type="filepath", label="Music Audio")
99
+ .harp_required(True)
100
+ .set_info("Music clip between 5 and 30 seconds."),
101
+ gr.Textbox(
102
+ label="Prompt",
103
+ placeholder="A warm synthwave groove with punchy drums",
104
+ ).harp_required(True),
105
+ gr.Dropdown(
106
+ choices=["Restyle", "Inpaint", "Continue"],
107
+ value="Restyle",
108
+ label="Edit Mode",
109
+ ),
110
+ gr.Slider(
111
+ minimum=0,
112
+ maximum=30,
113
+ value=4,
114
+ step=0.1,
115
+ label="Edit Start (seconds)",
116
+ ).set_info("Used by Inpaint mode."),
117
+ gr.Slider(
118
+ minimum=0,
119
+ maximum=30,
120
+ value=8,
121
+ step=0.1,
122
+ label="Edit End (seconds)",
123
+ ).set_info("Used by Inpaint mode."),
124
+ gr.Slider(
125
+ minimum=1,
126
+ maximum=15,
127
+ value=8,
128
+ step=1,
129
+ label="Continuation Length (seconds)",
130
+ ).set_info("Used by Continue mode."),
131
+ gr.Slider(
132
+ minimum=0.1,
133
+ maximum=1.0,
134
+ value=0.75,
135
+ step=0.05,
136
+ label="Transformation Strength",
137
+ ).set_info("Used by Restyle mode."),
138
+ gr.Slider(
139
+ minimum=0,
140
+ maximum=99999,
141
+ value=0,
142
+ step=1,
143
+ label="Seed",
144
+ ).set_info("Use 0 for a random seed."),
145
+ ]
146
+ output_components = [
147
+ gr.Audio(type="filepath", label="Edited Audio"),
148
+ ]
149
+ build_endpoint(
150
+ model_card=model_card,
151
+ input_components=input_components,
152
+ output_components=output_components,
153
+ process_fn=process_fn,
154
+ )
155
+
156
+
157
+ if __name__ == "__main__":
158
+ demo.queue(default_concurrency_limit=1).launch(show_error=True, pwa=True)
model.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "Stable Audio 3 Small-Music",
3
+ "task": [
4
+ "audio-to-audio editing",
5
+ "audio inpainting",
6
+ "audio continuation"
7
+ ],
8
+ "source": {
9
+ "repo": "Stability-AI/stable-audio-3",
10
+ "revision": "124e8a799f57a1f665495ecb72e547d0a62867f1",
11
+ "license": "MIT"
12
+ },
13
+ "runtime": {
14
+ "package": "stable-audio-tools",
15
+ "version": "0.0.20"
16
+ },
17
+ "model": {
18
+ "repo": "stabilityai/stable-audio-3-small-music",
19
+ "revision": "0fef1392cd842149a2b6d445e181c97608faac06",
20
+ "license": "Stability AI Community License"
21
+ },
22
+ "limits": {
23
+ "input_seconds": {
24
+ "minimum": 5,
25
+ "maximum": 30
26
+ },
27
+ "maximum_continuation_seconds": 15
28
+ }
29
+ }
requirements.txt ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ gradio==5.28.0
2
+ torch==2.8.0
3
+ torchaudio==2.8.0
4
+ numpy<2
5
+ soundfile>=0.13.1
6
+ einops>=0.8.2
7
+ einops-exts>=0.0.4
8
+ alias-free-torch==0.0.6
9
+ dill>=0.4
10
+ huggingface-hub>=1.7.1,<2
11
+ importlib-resources>=5.12
12
+ nnAudio
13
+ PyWavelets>=1.6,<2
14
+ pytorch-lightning==2.5.5
15
+ safetensors>=0.7.0
16
+ scipy
17
+ sentencepiece>=0.2,<1
18
+ setuptools<81
19
+ soxr
20
+ torchsde
21
+ tqdm
22
+ transformers>=5.8.0,<6
23
+ v-diffusion-pytorch==0.0.2
24
+ vector-quantize-pytorch==1.14.41
25
+ git+https://github.com/TEAMuP-dev/pyharp.git@d65c4f7d0264dcdb3024a6c5466cddd7b2defdca
stable_audio_runtime.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import subprocess
4
+ import sys
5
+ import tempfile
6
+ import threading
7
+ from pathlib import Path
8
+
9
+ MODEL_REPO = "stabilityai/stable-audio-3-small-music"
10
+ MODEL_REVISION = "0fef1392cd842149a2b6d445e181c97608faac06"
11
+ STABLE_AUDIO_TOOLS_VERSION = "0.0.20"
12
+ OUTPUT_ROOT = Path(tempfile.gettempdir()) / "stable_audio_3_outputs"
13
+
14
+ _MODEL = None
15
+ _MODEL_CONFIG = None
16
+ _MODEL_LOCK = threading.Lock()
17
+
18
+
19
+ def _ensure_stable_audio_tools() -> None:
20
+ try:
21
+ import stable_audio_tools # noqa: F401
22
+
23
+ return
24
+ except ImportError:
25
+ pass
26
+
27
+ subprocess.check_call(
28
+ [
29
+ sys.executable,
30
+ "-m",
31
+ "pip",
32
+ "install",
33
+ "--quiet",
34
+ "--no-deps",
35
+ f"stable-audio-tools=={STABLE_AUDIO_TOOLS_VERSION}",
36
+ ]
37
+ )
38
+
39
+
40
+ def _load_model():
41
+ global _MODEL, _MODEL_CONFIG
42
+ if _MODEL is not None:
43
+ return _MODEL, _MODEL_CONFIG
44
+
45
+ with _MODEL_LOCK:
46
+ if _MODEL is not None:
47
+ return _MODEL, _MODEL_CONFIG
48
+
49
+ _ensure_stable_audio_tools()
50
+
51
+ import torch
52
+ from stable_audio_tools.models import pretrained
53
+
54
+ original_download = pretrained.hf_hub_download
55
+
56
+ def pinned_download(repo_id, *args, **kwargs):
57
+ if repo_id == MODEL_REPO:
58
+ kwargs.setdefault("revision", MODEL_REVISION)
59
+ return original_download(repo_id, *args, **kwargs)
60
+
61
+ pretrained.hf_hub_download = pinned_download
62
+ try:
63
+ model, config = pretrained.get_pretrained_model(MODEL_REPO)
64
+ finally:
65
+ pretrained.hf_hub_download = original_download
66
+
67
+ model = model.to("cuda").to(torch.float16)
68
+ model.eval().requires_grad_(False)
69
+ _MODEL = model
70
+ _MODEL_CONFIG = config
71
+ return _MODEL, _MODEL_CONFIG
72
+
73
+
74
+ def _load_audio(path: str):
75
+ import torch
76
+ import torchaudio
77
+
78
+ audio, sample_rate = torchaudio.load(path)
79
+ if audio.shape[0] > 2:
80
+ audio = audio[:2]
81
+ return int(sample_rate), audio.to(torch.float32)
82
+
83
+
84
+ def _save_audio(output, sample_rate: int) -> str:
85
+ import soundfile as sf
86
+ import torch
87
+
88
+ output = output.permute(1, 0, 2).reshape(output.shape[1], -1)
89
+ output = output.to(torch.float32)
90
+ peak = output.abs().max().clamp(min=1e-9)
91
+ output = output.div(peak).clamp(-1, 1).cpu().numpy().T
92
+
93
+ OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
94
+ output_dir = Path(tempfile.mkdtemp(dir=OUTPUT_ROOT))
95
+ output_path = output_dir / "stable_audio_3_edit.wav"
96
+ sf.write(output_path, output, sample_rate, subtype="PCM_16")
97
+ return str(output_path)
98
+
99
+
100
+ def edit_audio(
101
+ audio_path: str,
102
+ prompt: str,
103
+ mode: str,
104
+ edit_start: float,
105
+ edit_end: float,
106
+ continuation_length: float,
107
+ strength: float,
108
+ seed: int,
109
+ ) -> str:
110
+ import torch
111
+ import torchaudio
112
+
113
+ model, config = _load_model()
114
+ from stable_audio_tools.inference.generation import (
115
+ generate_diffusion_cond_inpaint,
116
+ )
117
+ sample_rate = int(config["sample_rate"])
118
+ sample_size = int(config["sample_size"])
119
+ source_rate, source = _load_audio(audio_path)
120
+ source_duration = source.shape[-1] / source_rate
121
+
122
+ if source_rate != sample_rate:
123
+ source = torchaudio.functional.resample(source, source_rate, sample_rate)
124
+ model_dtype = next(model.parameters()).dtype
125
+ source_tuple = (sample_rate, source.to(model_dtype))
126
+
127
+ if mode == "Continue":
128
+ output_duration = source_duration + continuation_length
129
+ else:
130
+ output_duration = source_duration
131
+
132
+ conditioning = [{"prompt": prompt, "seconds_total": output_duration}]
133
+ kwargs = {
134
+ "steps": 8,
135
+ "cfg_scale": 1.0,
136
+ "conditioning": conditioning,
137
+ "sample_size": sample_size,
138
+ "sampler_type": "pingpong",
139
+ "seed": int(seed) if seed > 0 else -1,
140
+ "device": "cuda",
141
+ "sigma_max": 1.0,
142
+ "apg_scale": 1.0,
143
+ "duration_padding_sec": 6.0,
144
+ }
145
+
146
+ if mode == "Restyle":
147
+ kwargs["init_audio"] = source_tuple
148
+ kwargs["init_noise_level"] = float(strength)
149
+ elif mode == "Inpaint":
150
+ kwargs["inpaint_audio"] = source_tuple
151
+ kwargs["inpaint_mask_start_seconds"] = float(edit_start)
152
+ kwargs["inpaint_mask_end_seconds"] = float(edit_end)
153
+ elif mode == "Continue":
154
+ kwargs["inpaint_audio"] = source_tuple
155
+ kwargs["inpaint_mask_start_seconds"] = float(source_duration)
156
+ kwargs["inpaint_mask_end_seconds"] = float(output_duration)
157
+ else:
158
+ raise ValueError(f"Unknown edit mode: {mode}")
159
+
160
+ with torch.inference_mode():
161
+ output = generate_diffusion_cond_inpaint(model, **kwargs)
162
+
163
+ output = output[..., : int(output_duration * sample_rate)]
164
+ return _save_audio(output, sample_rate)