harp-dev commited on
Commit
a156faa
·
verified ·
1 Parent(s): 04ee238

Fetch X-Codec's HuBERT at startup too, pinned, and reserve the GPU time each mode takes.

Browse files
app.py CHANGED
@@ -49,8 +49,11 @@ CHECKPOINT_REVISIONS = {
49
  "Mixed": "06b68ce17425f90b7217d00eed7a46de9cfdb57c",
50
  "Dual-track (vocals + accompaniment)": "1f020d290238d7d7b0ceac8071ca873d99d20e54",
51
  }
52
- # MERT's revision, which songgen's vendored code pins as well
53
  MERT_REPO, MERT_REVISION = "m-a-p/MERT-v1-330M", "5240c2708a5acaee1007f43fb9735c7dcd0b78c9"
 
 
 
54
 
55
  state = {name: {"model": None, "processor": None} for name in MODES}
56
  checkpoint_paths = {}
@@ -75,7 +78,8 @@ def fetch_xcodec_assets():
75
 
76
  def fetch_assets():
77
  """Fetch every file the two modes load, so that no GPU call downloads anything: the X-Codec
78
- assets, each mode's SongGen checkpoint (5.4 GB), MERT (1.3 GB) and Demucs."""
 
79
  global assets_error
80
  try:
81
  fetch_xcodec_assets()
@@ -83,6 +87,7 @@ def fetch_assets():
83
  checkpoint_paths[mode] = snapshot_download(repo, revision=CHECKPOINT_REVISIONS[mode])
84
  snapshot_download(MERT_REPO, revision=MERT_REVISION,
85
  allow_patterns=["*.json", "*.py", "pytorch_model.bin"])
 
86
  pretrained.get_model("htdemucs") # caches its checkpoint for the processor
87
  except Exception as e:
88
  assets_error = str(e)
@@ -146,8 +151,13 @@ def process_fn(mode_name, description, lyrics, ref_voice_path, extract_vocals, t
146
  return sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature)
147
 
148
 
149
- # Measured at about 43 s, building the model included
150
- @spaces.GPU(duration=90)
 
 
 
 
 
151
  @torch.inference_mode()
152
  def sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
153
  model, processor = get_mode(mode_name)
@@ -185,7 +195,11 @@ if __name__ == "__main__":
185
  choices=list(MODES.keys()),
186
  value="Mixed",
187
  label="Mode",
188
- info="Mixed outputs one finished track; Dual-track also gives you an isolated vocal stem",
 
 
 
 
189
  ),
190
  gr.Textbox(
191
  label="Music Description",
 
49
  "Mixed": "06b68ce17425f90b7217d00eed7a46de9cfdb57c",
50
  "Dual-track (vocals + accompaniment)": "1f020d290238d7d7b0ceac8071ca873d99d20e54",
51
  }
52
+ # Revisions that songgen's vendored code pins as well: MERT, and the HuBERT inside X-Codec
53
  MERT_REPO, MERT_REVISION = "m-a-p/MERT-v1-330M", "5240c2708a5acaee1007f43fb9735c7dcd0b78c9"
54
+ HUBERT_REPO, HUBERT_REVISION = (
55
+ "ZhenYe234/hubert_base_general_audio", "9f2b3deb722f7bdb94b1e13037c5061b36c6ee3d"
56
+ )
57
 
58
  state = {name: {"model": None, "processor": None} for name in MODES}
59
  checkpoint_paths = {}
 
78
 
79
  def fetch_assets():
80
  """Fetch every file the two modes load, so that no GPU call downloads anything: the X-Codec
81
+ assets and the HuBERT they load, each mode's SongGen checkpoint (5.4 GB), MERT (1.3 GB) and
82
+ Demucs."""
83
  global assets_error
84
  try:
85
  fetch_xcodec_assets()
 
87
  checkpoint_paths[mode] = snapshot_download(repo, revision=CHECKPOINT_REVISIONS[mode])
88
  snapshot_download(MERT_REPO, revision=MERT_REVISION,
89
  allow_patterns=["*.json", "*.py", "pytorch_model.bin"])
90
+ snapshot_download(HUBERT_REPO, revision=HUBERT_REVISION)
91
  pretrained.get_model("htdemucs") # caches its checkpoint for the processor
92
  except Exception as e:
93
  assets_error = str(e)
 
151
  return sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature)
152
 
153
 
154
+ def gpu_seconds(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
155
+ # Measured at about 43 s in Mixed mode, and 180 s for a 30 s song in Dual-track mode, which
156
+ # generates the vocals and the accompaniment as separate streams, building the model included
157
+ return 90 if mode_name == "Mixed" else 200
158
+
159
+
160
+ @spaces.GPU(duration=gpu_seconds)
161
  @torch.inference_mode()
162
  def sing(mode_name, description, lyrics, ref_voice_path, extract_vocals, temperature):
163
  model, processor = get_mode(mode_name)
 
195
  choices=list(MODES.keys()),
196
  value="Mixed",
197
  label="Mode",
198
+ info=(
199
+ "Mixed outputs one finished track. Dual-track also gives you an isolated vocal "
200
+ "stem, but takes about three minutes of GPU time, more than half of a free "
201
+ "account's daily allowance."
202
+ ),
203
  ),
204
  gr.Textbox(
205
  label="Music Description",
songgen/xcodec_wrapper/xcodec_infer/models/soundstream_semantic.py CHANGED
@@ -58,7 +58,9 @@ class SoundStream(nn.Module):
58
  self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
59
  elif semantic_techer=='hubert_base_general':
60
 
61
- self.semantic_model = AutoModel.from_pretrained("ZhenYe234/hubert_base_general_audio")
 
 
62
  self.semantic_model.eval()
63
 
64
  self.fc_prior = nn.Linear(D+768, D+768 )
 
58
  self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-base-plus")
59
  elif semantic_techer=='hubert_base_general':
60
 
61
+ self.semantic_model = AutoModel.from_pretrained(
62
+ "ZhenYe234/hubert_base_general_audio",
63
+ revision="9f2b3deb722f7bdb94b1e13037c5061b36c6ee3d")
64
  self.semantic_model.eval()
65
 
66
  self.fc_prior = nn.Linear(D+768, D+768 )