Spaces:
Running on Zero
Running on Zero
Fetch X-Codec's HuBERT at startup too, pinned, and reserve the GPU time each mode takes.
Browse files
app.py
CHANGED
|
@@ -49,8 +49,11 @@ CHECKPOINT_REVISIONS = {
|
|
| 49 |
"Mixed": "06b68ce17425f90b7217d00eed7a46de9cfdb57c",
|
| 50 |
"Dual-track (vocals + accompaniment)": "1f020d290238d7d7b0ceac8071ca873d99d20e54",
|
| 51 |
}
|
| 52 |
-
#
|
| 53 |
MERT_REPO, MERT_REVISION = "m-a-p/MERT-v1-330M", "5240c2708a5acaee1007f43fb9735c7dcd0b78c9"
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
state = {name: {"model": None, "processor": None} for name in MODES}
|
| 56 |
checkpoint_paths = {}
|
|
@@ -75,7 +78,8 @@ def fetch_xcodec_assets():
|
|
| 75 |
|
| 76 |
def fetch_assets():
|
| 77 |
"""Fetch every file the two modes load, so that no GPU call downloads anything: the X-Codec
|
| 78 |
-
assets, each mode's SongGen checkpoint (5.4 GB), MERT (1.3 GB) and
|
|
|
|
| 79 |
global assets_error
|
| 80 |
try:
|
| 81 |
fetch_xcodec_assets()
|
|
@@ -83,6 +87,7 @@ def fetch_assets():
|
|
| 83 |
checkpoint_paths[mode] = snapshot_download(repo, revision=CHECKPOINT_REVISIONS[mode])
|
| 84 |
snapshot_download(MERT_REPO, revision=MERT_REVISION,
|
| 85 |
allow_patterns=["*.json", "*.py", "pytorch_model.bin"])
|
|
|
|
| 86 |
pretrained.get_model("htdemucs") # caches its checkpoint for the processor
|
| 87 |
except Exception as e:
|
| 88 |
assets_error = str(e)
|
|
@@ -146,8 +151,13 @@ def process_fn(mode_name, description, lyrics, ref_voice_path, extract_vocals, t
|
|
| 146 |
return sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature)
|
| 147 |
|
| 148 |
|
| 149 |
-
|
| 150 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
@torch.inference_mode()
|
| 152 |
def sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
|
| 153 |
model, processor = get_mode(mode_name)
|
|
@@ -185,7 +195,11 @@ if __name__ == "__main__":
|
|
| 185 |
choices=list(MODES.keys()),
|
| 186 |
value="Mixed",
|
| 187 |
label="Mode",
|
| 188 |
-
info=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
),
|
| 190 |
gr.Textbox(
|
| 191 |
label="Music Description",
|
|
|
|
| 49 |
"Mixed": "06b68ce17425f90b7217d00eed7a46de9cfdb57c",
|
| 50 |
"Dual-track (vocals + accompaniment)": "1f020d290238d7d7b0ceac8071ca873d99d20e54",
|
| 51 |
}
|
| 52 |
+
# Revisions that songgen's vendored code pins as well: MERT, and the HuBERT inside X-Codec
|
| 53 |
MERT_REPO, MERT_REVISION = "m-a-p/MERT-v1-330M", "5240c2708a5acaee1007f43fb9735c7dcd0b78c9"
|
| 54 |
+
HUBERT_REPO, HUBERT_REVISION = (
|
| 55 |
+
"ZhenYe234/hubert_base_general_audio", "9f2b3deb722f7bdb94b1e13037c5061b36c6ee3d"
|
| 56 |
+
)
|
| 57 |
|
| 58 |
state = {name: {"model": None, "processor": None} for name in MODES}
|
| 59 |
checkpoint_paths = {}
|
|
|
|
| 78 |
|
| 79 |
def fetch_assets():
|
| 80 |
"""Fetch every file the two modes load, so that no GPU call downloads anything: the X-Codec
|
| 81 |
+
assets and the HuBERT they load, each mode's SongGen checkpoint (5.4 GB), MERT (1.3 GB) and
|
| 82 |
+
Demucs."""
|
| 83 |
global assets_error
|
| 84 |
try:
|
| 85 |
fetch_xcodec_assets()
|
|
|
|
| 87 |
checkpoint_paths[mode] = snapshot_download(repo, revision=CHECKPOINT_REVISIONS[mode])
|
| 88 |
snapshot_download(MERT_REPO, revision=MERT_REVISION,
|
| 89 |
allow_patterns=["*.json", "*.py", "pytorch_model.bin"])
|
| 90 |
+
snapshot_download(HUBERT_REPO, revision=HUBERT_REVISION)
|
| 91 |
pretrained.get_model("htdemucs") # caches its checkpoint for the processor
|
| 92 |
except Exception as e:
|
| 93 |
assets_error = str(e)
|
|
|
|
| 151 |
return sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature)
|
| 152 |
|
| 153 |
|
| 154 |
+
def gpu_seconds(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
|
| 155 |
+
# Measured at about 43 s in Mixed mode, and 180 s for a 30 s song in Dual-track mode, which
|
| 156 |
+
# generates the vocals and the accompaniment as separate streams, building the model included
|
| 157 |
+
return 90 if mode_name == "Mixed" else 200
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@spaces.GPU(duration=gpu_seconds)
|
| 161 |
@torch.inference_mode()
|
| 162 |
def sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
|
| 163 |
model, processor = get_mode(mode_name)
|
|
|
|
| 195 |
choices=list(MODES.keys()),
|
| 196 |
value="Mixed",
|
| 197 |
label="Mode",
|
| 198 |
+
info=(
|
| 199 |
+
"Mixed outputs one finished track. Dual-track also gives you an isolated vocal "
|
| 200 |
+
"stem, but takes about three minutes of GPU time, more than half of a free "
|
| 201 |
+
"account's daily allowance."
|
| 202 |
+
),
|
| 203 |
),
|
| 204 |
gr.Textbox(
|
| 205 |
label="Music Description",
|
songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py
CHANGED
|
@@ -58,7 +58,9 @@ class SoundStream(nn.Module):
|
|
| 58 |
self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
|
| 59 |
elif semantic_techer=='hubert_base_general':
|
| 60 |
|
| 61 |
-
self.semantic_model = AutoModel.from_pretrained(
|
|
|
|
|
|
|
| 62 |
self.semantic_model.eval()
|
| 63 |
|
| 64 |
self.fc_prior = nn.Linear(D+768, D+768 )
|
|
|
|
| 58 |
self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
|
| 59 |
elif semantic_techer=='hubert_base_general':
|
| 60 |
|
| 61 |
+
self.semantic_model = AutoModel.from_pretrained(
|
| 62 |
+
"ZhenYe234/hubert_base_general_audio",
|
| 63 |
+
revision="9f2b3deb722f7bdb94b1e13037c5061b36c6ee3d")
|
| 64 |
self.semantic_model.eval()
|
| 65 |
|
| 66 |
self.fc_prior = nn.Linear(D+768, D+768 )
|