redkits commited on
Commit
d22ea38
·
verified ·
1 Parent(s): 1a1b7b4
Files changed (1) hide show
  1. app.py +1129 -0
app.py ADDED
@@ -0,0 +1,1129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sangeet Studio — Suno-style song generation + instant artist-voice cloning.
2
+
3
+ Chain:
4
+ 1. ACE-Step 1.5 turbo (2.4B DiT, MIT) with RagaLoRA (Indian-music LoRA) generates
5
+ a full song with vocals (Hindi/Urdu/Punjabi/Tamil/... lyrics supported).
6
+ 2. Demucs (htdemucs) splits the song into vocals + instrumental stems.
7
+ 3. Seed-VC (SVC mode) converts the vocal stem into the user's cloned artist
8
+ voice from a 5-25s clean reference clip.
9
+ 4. Converted vocals are remixed with the instrumental stem.
10
+
11
+ Licenses: ACE-Step MIT; RagaLoRA Apache-2.0; Seed-VC inference code is GPL-3.0
12
+ (adapted from github.com/Plachtaa/seed-vc app_svc.py) — this Space is therefore GPL-3.0.
13
+ """
14
+
15
+ import os
16
+ import shutil
17
+ import shlex
18
+ import subprocess
19
+ import sys
20
+ import tempfile
21
+ import threading
22
+
23
+ print("[startup] importing numpy/torch…", flush=True)
24
+ import numpy as np
25
+ import torch
26
+ print("[startup] torch ready — importing gradio/soundfile…", flush=True)
27
+ import gradio as gr
28
+ import soundfile as sf
29
+ print("[startup] imports done", flush=True)
30
+
31
+ # ---------------------------------------------------------------------------
32
+ # Repo setup: clone ACE-Step-1.5 and seed-vc, put them on sys.path
33
+ # ---------------------------------------------------------------------------
34
+ APP_DIR = os.path.dirname(os.path.abspath(__file__))
35
+ REPOS_DIR = os.path.join(APP_DIR, "repos")
36
+ ACE_DIR = os.path.join(REPOS_DIR, "ACE-Step-1.5")
37
+ SEEDVC_DIR = os.path.join(REPOS_DIR, "seed-vc")
38
+ CHECKPOINTS_DIR = os.path.join(ACE_DIR, "checkpoints")
39
+ os.environ["ACESTEP_CHECKPOINTS_DIR"] = CHECKPOINTS_DIR
40
+ os.makedirs(CHECKPOINTS_DIR, exist_ok=True)
41
+
42
+
43
+ def _clone(url: str, dest: str, expect_dir: str) -> None:
44
+ """Clone a repo, then VERIFY the expected package directory actually landed.
45
+ A proxied/failed git transfer can exit 0 with an incomplete checkout, which
46
+ produced "No module named 'acestep'" at import time. Fall back to the
47
+ GitHub tarball (different download route) if verification fails."""
48
+ if not os.path.isdir(dest):
49
+ print(f"[startup] cloning {url}…", flush=True)
50
+ subprocess.run(
51
+ ["git", "clone", "--depth", "1", url, dest],
52
+ check=True, timeout=300,
53
+ )
54
+ if not os.path.isdir(os.path.join(dest, expect_dir)):
55
+ print(
56
+ f"[startup] clone incomplete ('{expect_dir}' missing) — retrying via tarball…",
57
+ flush=True,
58
+ )
59
+ shutil.rmtree(dest, ignore_errors=True)
60
+ os.makedirs(dest, exist_ok=True)
61
+ owner_repo = url.replace("https://github.com/", "").replace(".git", "")
62
+ cmd = (
63
+ f"curl -fsSL https://codeload.github.com/{owner_repo}/tar.gz/HEAD "
64
+ f"| tar -xz -C {shlex.quote(dest)} --strip-components=1"
65
+ )
66
+ subprocess.run(cmd, shell=True, check=True, timeout=300)
67
+ if not os.path.isdir(os.path.join(dest, expect_dir)):
68
+ raise RuntimeError(
69
+ f"{dest} is missing '{expect_dir}' even after tarball fallback — "
70
+ "network to GitHub is broken from this container."
71
+ )
72
+ print(f"[startup] repo ready: {dest}", flush=True)
73
+
74
+
75
+ # Repo cloning + checkpoint download happen in the background _prepare_env()
76
+ # thread (defined below) so the Gradio port always binds immediately.
77
+
78
+ # ---------------------------------------------------------------------------
79
+ # Model downloads (network + CPU only here; safe at Space startup)
80
+ # ---------------------------------------------------------------------------
81
+ _ACE_READY = threading.Event()
82
+ _ENV_READY = threading.Event()
83
+
84
+
85
+ def _download_acestep_models() -> None:
86
+ try:
87
+ for _p in (ACE_DIR, SEEDVC_DIR):
88
+ if _p not in sys.path:
89
+ sys.path.insert(0, _p)
90
+ import importlib
91
+ importlib.invalidate_caches()
92
+ print(
93
+ f"[startup] acestep package present: "
94
+ f"{os.path.isdir(os.path.join(ACE_DIR, 'acestep'))}", flush=True,
95
+ )
96
+ from acestep.model_downloader import ensure_main_model, ensure_lm_model
97
+ ok, msg = ensure_main_model(CHECKPOINTS_DIR)
98
+ print(f"[startup] main model: {msg}", flush=True)
99
+ ok, msg = ensure_lm_model("acestep-5Hz-lm-1.7B", CHECKPOINTS_DIR)
100
+ print(f"[startup] LM model: {msg}", flush=True)
101
+ _ACE_READY.set()
102
+ except Exception as e:
103
+ print(f"[startup] ACE-Step checkpoint download failed: {e}", flush=True)
104
+ # Unblock waiting pipelines so they fail fast with a visible error
105
+ _ENV_READY.set()
106
+ _ACE_READY.set()
107
+
108
+
109
+ def _prepare_env() -> None:
110
+ """Clone repos, extend sys.path, then download checkpoints — all in the
111
+ background so the Gradio port binds immediately. Pipelines wait on the
112
+ _ENV_READY / _ACE_READY events."""
113
+ try:
114
+ print("[startup] cloning ACE-Step and seed-vc repos…", flush=True)
115
+ _clone("https://github.com/ACE-Step/ACE-Step-1.5.git", ACE_DIR, "acestep")
116
+ _clone("https://github.com/Plachtaa/seed-vc.git", SEEDVC_DIR, "modules")
117
+ for _p in (ACE_DIR, SEEDVC_DIR):
118
+ if _p not in sys.path:
119
+ sys.path.insert(0, _p)
120
+ import importlib
121
+ importlib.invalidate_caches()
122
+ _ENV_READY.set()
123
+ print("[startup] repos ready — downloading checkpoints…", flush=True)
124
+ _download_acestep_models()
125
+ except Exception as e:
126
+ print(f"[startup] environment preparation failed: {e}", flush=True)
127
+ _ENV_READY.set()
128
+ _ACE_READY.set()
129
+
130
+
131
+ # Everything slow (repo clones + ~10 GB checkpoint download) happens here, in the
132
+ # background, so launch() below is reached within seconds no matter what. The
133
+ # first generation waits on the events.
134
+ threading.Thread(target=_prepare_env, daemon=True).start()
135
+
136
+ # ---------------------------------------------------------------------------
137
+ # Lazy model state
138
+ # ---------------------------------------------------------------------------
139
+ _state = {}
140
+
141
+
142
+ def _load_ace(device: str):
143
+ if not _ENV_READY.wait(timeout=600) or not _ACE_READY.wait(timeout=1800):
144
+ raise RuntimeError(
145
+ "Model download did not finish in time — check the Space runtime logs and restart."
146
+ )
147
+ if _state.get("ace_ok"):
148
+ return _state["dit"], _state["llm"]
149
+
150
+ for _p in (ACE_DIR, SEEDVC_DIR):
151
+ if _p not in sys.path:
152
+ sys.path.insert(0, _p)
153
+ from acestep.handler import AceStepHandler
154
+ from acestep.llm_inference import LLMHandler
155
+ from huggingface_hub import snapshot_download
156
+
157
+ dit = AceStepHandler()
158
+ status, ok = dit.initialize_service(
159
+ project_root=ACE_DIR,
160
+ config_path="acestep-v15-turbo",
161
+ device=device,
162
+ offload_to_cpu=False,
163
+ )
164
+ if not ok:
165
+ raise RuntimeError(f"ACE-Step DiT init failed: {status}")
166
+
167
+ llm = LLMHandler()
168
+ status, ok = llm.initialize(
169
+ checkpoint_dir=CHECKPOINTS_DIR,
170
+ lm_model_path="acestep-5Hz-lm-1.7B",
171
+ backend="pt",
172
+ device=device,
173
+ offload_to_cpu=False,
174
+ )
175
+ if not ok:
176
+ raise RuntimeError(f"ACE-Step LM init failed: {status}")
177
+
178
+ lora_dir = snapshot_download("veeceey/RagaLoRA-indian-music-ace-step")
179
+ dit.add_lora(lora_dir)
180
+ dit.set_use_lora(True)
181
+ dit.set_lora_scale(0.8)
182
+
183
+ _state["dit"], _state["llm"], _state["ace_ok"] = dit, llm, True
184
+ return dit, llm
185
+
186
+
187
+ def _load_svc(device: torch.device):
188
+ if not _ENV_READY.wait(timeout=600):
189
+ raise RuntimeError(
190
+ "Environment preparation did not finish in 10 min — check the Space runtime logs."
191
+ )
192
+ if _state.get("svc_ok"):
193
+ return _state["svc"]
194
+
195
+ for _p in (ACE_DIR, SEEDVC_DIR):
196
+ if _p not in sys.path:
197
+ sys.path.insert(0, _p)
198
+ import yaml
199
+ from types import SimpleNamespace
200
+ from hf_utils import load_custom_model_from_hf
201
+ from modules.commons import build_model, load_checkpoint, recursive_munch
202
+
203
+ fp16 = True
204
+ dit_checkpoint_path, dit_config_path = load_custom_model_from_hf(
205
+ "Plachta/Seed-VC",
206
+ "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema_v2.pth",
207
+ "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml",
208
+ )
209
+ config = yaml.safe_load(open(dit_config_path, "r"))
210
+ model_params = recursive_munch(config["model_params"])
211
+ model_params.dit_type = "DiT"
212
+ model = build_model(model_params, stage="DiT")
213
+ hop_length = config["preprocess_params"]["spect_params"]["hop_length"]
214
+ sr = config["preprocess_params"]["sr"]
215
+
216
+ model, _, _, _ = load_checkpoint(
217
+ model, None, dit_checkpoint_path,
218
+ load_only_params=True, ignore_modules=[], is_distributed=False,
219
+ )
220
+ for key in model:
221
+ model[key].eval()
222
+ model[key].to(device)
223
+ model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192)
224
+
225
+ from modules.campplus.DTDNN import CAMPPlus
226
+ campplus_ckpt_path = load_custom_model_from_hf(
227
+ "funasr/campplus", "campplus_cn_common.bin", config_filename=None
228
+ )
229
+ campplus_model = CAMPPlus(feat_dim=80, embedding_size=192)
230
+ campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu"))
231
+ campplus_model.eval().to(device)
232
+
233
+ vocoder_type = model_params.vocoder.type
234
+ if vocoder_type == "bigvgan":
235
+ from modules.bigvgan import bigvgan
236
+ bigvgan_model = bigvgan.BigVGAN.from_pretrained(
237
+ model_params.vocoder.name, use_cuda_kernel=False
238
+ )
239
+ bigvgan_model.remove_weight_norm()
240
+ vocoder_fn = bigvgan_model.eval().to(device)
241
+ else:
242
+ raise ValueError(f"Unsupported vocoder type: {vocoder_type}")
243
+
244
+ speech_tokenizer_type = model_params.speech_tokenizer.type
245
+ if speech_tokenizer_type != "whisper":
246
+ raise ValueError(f"Unsupported speech tokenizer: {speech_tokenizer_type}")
247
+ from transformers import AutoFeatureExtractor, WhisperModel
248
+ whisper_name = model_params.speech_tokenizer.name
249
+ whisper_model = WhisperModel.from_pretrained(
250
+ whisper_name, torch_dtype=torch.float16
251
+ ).to(device)
252
+ del whisper_model.decoder
253
+ whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name)
254
+
255
+ def semantic_fn(waves_16k):
256
+ ori_inputs = whisper_feature_extractor(
257
+ [waves_16k.squeeze(0).cpu().numpy()],
258
+ return_tensors="pt",
259
+ return_attention_mask=True,
260
+ )
261
+ ori_input_features = whisper_model._mask_input_features(
262
+ ori_inputs.input_features, attention_mask=ori_inputs.attention_mask
263
+ ).to(device)
264
+ with torch.no_grad():
265
+ ori_outputs = whisper_model.encoder(
266
+ ori_input_features.to(whisper_model.encoder.dtype),
267
+ head_mask=None,
268
+ output_attentions=False,
269
+ output_hidden_states=False,
270
+ return_dict=True,
271
+ )
272
+ S_ori = ori_outputs.last_hidden_state.to(torch.float32)
273
+ S_ori = S_ori[:, : waves_16k.size(-1) // 320 + 1]
274
+ return S_ori
275
+
276
+ mel_fn_args = {
277
+ "n_fft": config["preprocess_params"]["spect_params"]["n_fft"],
278
+ "win_size": config["preprocess_params"]["spect_params"]["win_length"],
279
+ "hop_size": config["preprocess_params"]["spect_params"]["hop_length"],
280
+ "num_mels": config["preprocess_params"]["spect_params"]["n_mels"],
281
+ "sampling_rate": sr,
282
+ "fmin": config["preprocess_params"]["spect_params"].get("fmin", 0),
283
+ "fmax": None
284
+ if config["preprocess_params"]["spect_params"].get("fmax", "None") == "None"
285
+ else 8000,
286
+ "center": False,
287
+ }
288
+ from modules.audio import mel_spectrogram
289
+ to_mel = lambda x: mel_spectrogram(x, **mel_fn_args)
290
+
291
+ from modules.rmvpe import RMVPE
292
+ rmvpe_path = load_custom_model_from_hf("lj1995/VoiceConversionWebUI", "rmvpe.pt", None)
293
+ rmvpe = RMVPE(rmvpe_path, is_half=False, device=device)
294
+
295
+ svc = SimpleNamespace(
296
+ model=model,
297
+ semantic_fn=semantic_fn,
298
+ vocoder_fn=vocoder_fn,
299
+ campplus_model=campplus_model,
300
+ to_mel=to_mel,
301
+ f0_fn=rmvpe.infer_from_audio,
302
+ sr=sr,
303
+ hop_length=hop_length,
304
+ fp16=fp16,
305
+ overlap_frame_len=16,
306
+ device=device,
307
+ )
308
+ svc.max_context_window = sr // hop_length * 30
309
+ svc.overlap_wave_len = svc.overlap_frame_len * hop_length
310
+
311
+ _state["svc"], _state["svc_ok"] = svc, True
312
+ return svc
313
+
314
+
315
+ def _adjust_f0_semitones(f0_sequence, n_semitones):
316
+ factor = 2 ** (n_semitones / 12)
317
+ return f0_sequence * factor
318
+
319
+
320
+ def _crossfade(chunk1, chunk2, overlap):
321
+ fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
322
+ fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
323
+ chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
324
+ return chunk2
325
+
326
+
327
+ # ---------------------------------------------------------------------------
328
+ # Seed-VC singing-voice conversion (adapted non-streaming from seed-vc app_svc.py)
329
+ # ---------------------------------------------------------------------------
330
+ @torch.no_grad()
331
+ @torch.inference_mode()
332
+ def _voice_convert(svc, source_path, target_path, diffusion_steps=30,
333
+ length_adjust=1.0, inference_cfg_rate=0.7,
334
+ auto_f0_adjust=True, pitch_shift=0):
335
+ import librosa
336
+ import torchaudio
337
+
338
+ device = svc.device
339
+ inference_module = svc.model
340
+ mel_fn = svc.to_mel
341
+ sr = svc.sr
342
+
343
+ source_audio = librosa.load(source_path, sr=sr)[0]
344
+ ref_audio = librosa.load(target_path, sr=sr)[0]
345
+
346
+ source_audio = torch.tensor(source_audio).unsqueeze(0).float().to(device)
347
+ ref_audio = torch.tensor(ref_audio[: sr * 25]).unsqueeze(0).float().to(device)
348
+
349
+ ref_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
350
+ converted_waves_16k = torchaudio.functional.resample(source_audio, sr, 16000)
351
+
352
+ if converted_waves_16k.size(-1) <= 16000 * 30:
353
+ S_alt = svc.semantic_fn(converted_waves_16k)
354
+ else:
355
+ overlapping_time = 5 # seconds
356
+ S_alt_list = []
357
+ buffer = None
358
+ traversed_time = 0
359
+ while traversed_time < converted_waves_16k.size(-1):
360
+ if buffer is None:
361
+ chunk = converted_waves_16k[:, traversed_time: traversed_time + 16000 * 30]
362
+ else:
363
+ chunk = torch.cat(
364
+ [buffer, converted_waves_16k[:, traversed_time: traversed_time + 16000 * (30 - overlapping_time)]],
365
+ dim=-1,
366
+ )
367
+ S_alt = svc.semantic_fn(chunk)
368
+ if traversed_time == 0:
369
+ S_alt_list.append(S_alt)
370
+ else:
371
+ S_alt_list.append(S_alt[:, 50 * overlapping_time:])
372
+ buffer = chunk[:, -16000 * overlapping_time:]
373
+ traversed_time += (
374
+ 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time
375
+ )
376
+ S_alt = torch.cat(S_alt_list, dim=1)
377
+
378
+ ori_waves_16k = torchaudio.functional.resample(ref_audio, sr, 16000)
379
+ S_ori = svc.semantic_fn(ori_waves_16k)
380
+
381
+ mel = mel_fn(source_audio.to(device).float())
382
+ mel2 = mel_fn(ref_audio.to(device).float())
383
+
384
+ target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to(mel.device)
385
+ target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device)
386
+
387
+ feat2 = torchaudio.compliance.kaldi.fbank(
388
+ ref_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000
389
+ )
390
+ feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
391
+ style2 = svc.campplus_model(feat2.unsqueeze(0))
392
+
393
+ F0_ori = svc.f0_fn(ref_waves_16k[0], thred=0.03)
394
+ F0_alt = svc.f0_fn(converted_waves_16k[0], thred=0.03)
395
+
396
+ F0_ori = torch.from_numpy(F0_ori).to(device)[None]
397
+ F0_alt = torch.from_numpy(F0_alt).to(device)[None]
398
+
399
+ voiced_F0_ori = F0_ori[F0_ori > 1]
400
+ voiced_F0_alt = F0_alt[F0_alt > 1]
401
+
402
+ log_f0_alt = torch.log(F0_alt + 1e-5)
403
+ voiced_log_f0_ori = torch.log(voiced_F0_ori + 1e-5)
404
+ voiced_log_f0_alt = torch.log(voiced_F0_alt + 1e-5)
405
+ median_log_f0_ori = torch.median(voiced_log_f0_ori)
406
+ median_log_f0_alt = torch.median(voiced_log_f0_alt)
407
+
408
+ shifted_log_f0_alt = log_f0_alt.clone()
409
+ if auto_f0_adjust:
410
+ shifted_log_f0_alt[F0_alt > 1] = (
411
+ log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori
412
+ )
413
+ shifted_f0_alt = torch.exp(shifted_log_f0_alt)
414
+ if pitch_shift != 0:
415
+ shifted_f0_alt[F0_alt > 1] = _adjust_f0_semitones(
416
+ shifted_f0_alt[F0_alt > 1], pitch_shift
417
+ )
418
+
419
+ cond, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(
420
+ S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt
421
+ )
422
+ prompt_condition, _, codes, commitment_loss, codebook_loss = inference_module.length_regulator(
423
+ S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori
424
+ )
425
+ interpolated_shifted_f0_alt = torch.nn.functional.interpolate(
426
+ shifted_f0_alt.unsqueeze(1), size=cond.size(1), mode="nearest"
427
+ ).squeeze(1)
428
+ max_source_window = svc.max_context_window - mel2.size(2)
429
+
430
+ processed_frames = 0
431
+ generated_wave_chunks = []
432
+ while processed_frames < cond.size(1):
433
+ chunk_cond = cond[:, processed_frames: processed_frames + max_source_window]
434
+ is_last_chunk = processed_frames + max_source_window >= cond.size(1)
435
+ cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
436
+ with torch.autocast(
437
+ device_type=device.type, dtype=torch.float16 if svc.fp16 else torch.float32
438
+ ):
439
+ vc_target = inference_module.cfm.inference(
440
+ cat_condition,
441
+ torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
442
+ mel2, style2, None, diffusion_steps,
443
+ inference_cfg_rate=inference_cfg_rate,
444
+ )
445
+ vc_target = vc_target[:, :, mel2.size(-1):]
446
+ vc_wave = svc.vocoder_fn(vc_target.float()).squeeze().cpu()
447
+ if vc_wave.ndim == 1:
448
+ vc_wave = vc_wave.unsqueeze(0)
449
+ if processed_frames == 0:
450
+ if is_last_chunk:
451
+ generated_wave_chunks.append(vc_wave[0].cpu().numpy())
452
+ break
453
+ generated_wave_chunks.append(vc_wave[0, : -svc.overlap_wave_len].cpu().numpy())
454
+ previous_chunk = vc_wave[0, -svc.overlap_wave_len:]
455
+ processed_frames += vc_target.size(2) - svc.overlap_frame_len
456
+ elif is_last_chunk:
457
+ output_wave = _crossfade(
458
+ previous_chunk.cpu().numpy(), vc_wave[0].cpu().numpy(), svc.overlap_wave_len
459
+ )
460
+ generated_wave_chunks.append(output_wave)
461
+ processed_frames += vc_target.size(2) - svc.overlap_frame_len
462
+ break
463
+ else:
464
+ output_wave = _crossfade(
465
+ previous_chunk.cpu().numpy(),
466
+ vc_wave[0, : -svc.overlap_wave_len].cpu().numpy(),
467
+ svc.overlap_wave_len,
468
+ )
469
+ generated_wave_chunks.append(output_wave)
470
+ previous_chunk = vc_wave[0, -svc.overlap_wave_len:]
471
+ processed_frames += vc_target.size(2) - svc.overlap_frame_len
472
+
473
+ return np.concatenate(generated_wave_chunks), sr
474
+
475
+
476
+ # ---------------------------------------------------------------------------
477
+ # Demucs stem split + remix
478
+ # ---------------------------------------------------------------------------
479
+ def _split_stems(song_path, out_dir):
480
+ from demucs.separate import main as demucs_main
481
+ demucs_main(["--two-stems", "vocals", "-n", "htdemucs", "-o", out_dir, song_path])
482
+ base = os.path.splitext(os.path.basename(song_path))[0]
483
+ d = os.path.join(out_dir, "htdemucs", base)
484
+ return os.path.join(d, "vocals.wav"), os.path.join(d, "no_vocals.wav")
485
+
486
+
487
+ def _remix(instrumental_path, converted_vocals_path, out_path):
488
+ inst, sr_i = sf.read(instrumental_path, always_2d=True)
489
+ voc, sr_v = sf.read(converted_vocals_path, always_2d=True)
490
+ if sr_v != sr_i:
491
+ import torchaudio
492
+ import librosa
493
+ voc_mono = librosa.resample(np.mean(voc, axis=1), orig_sr=sr_v, target_sr=sr_i)
494
+ voc = voc_mono[:, None]
495
+ if voc.shape[1] == 1 and inst.shape[1] == 2:
496
+ voc = np.repeat(voc, 2, axis=1)
497
+ n = max(inst.shape[0], voc.shape[0])
498
+ inst = np.pad(inst, ((0, n - inst.shape[0]), (0, 0)))
499
+ voc = np.pad(voc, ((0, n - voc.shape[0]), (0, 0)))
500
+
501
+ def _norm(x, peak):
502
+ return x / (np.max(np.abs(x)) + 1e-9) * peak
503
+
504
+ mix = _norm(inst, 0.9) + _norm(voc, 0.7)
505
+ sf.write(out_path, np.clip(mix, -1.0, 1.0).astype(np.float32), sr_i)
506
+ return out_path
507
+
508
+
509
+ # ---------------------------------------------------------------------------
510
+ # Languages (ISO 639-1 codes for ACE-Step vocal_language)
511
+ # ---------------------------------------------------------------------------
512
+ LANGS = [
513
+ ("Auto-detect", "unknown"),
514
+ ("English", "en"),
515
+ ("हिन्दी Hindi", "hi"),
516
+ ("اردو Urdu", "ur"),
517
+ ("ਪੰਜਾਬੀ Punjabi", "pa"),
518
+ ("தமிழ் Tamil", "ta"),
519
+ ("తెలుగు Telugu", "te"),
520
+ ("বাংলা Bengali", "bn"),
521
+ ("मराठी Marathi", "mr"),
522
+ ("ગુજરાતી Gujarati", "gu"),
523
+ ("ಕನ್ನಡ Kannada", "kn"),
524
+ ("മലയാളം Malayalam", "ml"),
525
+ ("ଓଡ଼ିଆ Odia", "or"),
526
+ ("অসমীয়া Assamese", "as"),
527
+ ("नेपाली Nepali", "ne"),
528
+ ("සිංහල Sinhala", "si"),
529
+ ]
530
+
531
+
532
+ # ---------------------------------------------------------------------------
533
+ # Main pipeline
534
+ # ---------------------------------------------------------------------------
535
+ def run_pipeline(
536
+ caption, lyrics, language, duration, instrumental, thinking,
537
+ use_lora, lora_scale, seed,
538
+ voice_ref, vc_steps, auto_f0, pitch_shift,
539
+ progress=gr.Progress(),
540
+ ):
541
+ device = "cuda" if torch.cuda.is_available() else "cpu"
542
+ tmp = tempfile.mkdtemp(prefix="sangeet_")
543
+ try:
544
+ lyrics = (lyrics or "").strip()
545
+ caption = (caption or "").strip()
546
+ if not instrumental and not lyrics and not caption:
547
+ yield "❌ Give a style prompt or lyrics first.", None, None, None, None
548
+ return
549
+ lyrics = lyrics[:4000]
550
+ caption = caption[:500]
551
+ if instrumental:
552
+ lyrics = "[Instrumental]"
553
+
554
+ yield "⏳ **Step 1/4** — loading ACE-Step turbo + LM + RagaLoRA…", None, None, None, None
555
+ dit, llm = _load_ace(device)
556
+ dit.set_use_lora(bool(use_lora))
557
+ if use_lora:
558
+ dit.set_lora_scale(float(lora_scale))
559
+
560
+ yield "🎵 **Step 2/4** — generating the song…", None, None, None, None
561
+ from acestep.inference import GenerationParams, GenerationConfig, generate_music
562
+
563
+ params = GenerationParams(
564
+ task_type="text2music",
565
+ thinking=bool(thinking),
566
+ caption=caption,
567
+ lyrics=lyrics,
568
+ instrumental=bool(instrumental),
569
+ vocal_language=language,
570
+ duration=float(duration),
571
+ inference_steps=8,
572
+ guidance_scale=1.0, # turbo auto-forces CFG off; documented value
573
+ shift=3.0, # required timestep shift for turbo
574
+ seed=int(seed),
575
+ )
576
+ config = GenerationConfig(
577
+ batch_size=1,
578
+ audio_format="wav",
579
+ use_random_seed=int(seed) < 0,
580
+ )
581
+ result = generate_music(
582
+ dit, llm, params=params, config=config, save_dir=os.path.join(tmp, "ace")
583
+ )
584
+ if not result.success:
585
+ raise RuntimeError(result.status_message or result.error or "generation failed")
586
+ song_path = result.audios[0]["path"]
587
+
588
+ if voice_ref is None:
589
+ yield (
590
+ "✅ **Done** — song generated (no voice reference given, so ACE-Step's own vocals kept).",
591
+ song_path, song_path, None, None,
592
+ )
593
+ return
594
+
595
+ yield "🎚️ **Step 3/4** — splitting stems (Demucs)…", None, None, None, None
596
+ vocals_path, no_vocals_path = _split_stems(song_path, os.path.join(tmp, "stems"))
597
+
598
+ yield "🎤 **Step 4/4** — converting vocals to the cloned artist voice (Seed-VC)…", None, None, None, None
599
+ svc = _load_svc(device)
600
+ converted_wave, sr = _voice_convert(
601
+ svc, vocals_path, voice_ref,
602
+ diffusion_steps=int(vc_steps),
603
+ auto_f0_adjust=bool(auto_f0),
604
+ pitch_shift=int(pitch_shift),
605
+ )
606
+ converted_path = os.path.join(tmp, "vocals_converted.wav")
607
+ sf.write(converted_path, converted_wave.astype(np.float32), sr)
608
+
609
+ final_path = _remix(no_vocals_path, converted_path, os.path.join(tmp, "final_song.wav"))
610
+ yield (
611
+ "✅ **Done** — song generated, vocals swapped to the cloned artist voice, remixed.",
612
+ final_path, song_path, no_vocals_path, converted_path,
613
+ )
614
+ except Exception as e:
615
+ yield f"❌ **Failed:** `{e}`", None, None, None, None
616
+
617
+
618
+ # ---------------------------------------------------------------------------
619
+ # Live Rap Studio: recording -> auto beat -> polish/master
620
+ # ---------------------------------------------------------------------------
621
+ def _polish_and_mix(vocal_path, beat_path, out_path,
622
+ beat_gain=0.55, vocal_gain=0.9):
623
+ """Polish the vocal (high-pass, denoise-trim, normalize, soft limiter)
624
+ and mix it over the generated beat with a 1 s intro and fades."""
625
+ import librosa
626
+ from scipy.signal import butter, sosfilt
627
+
628
+ voc, vsr = librosa.load(vocal_path, sr=44100, mono=True)
629
+ beat, bsr = sf.read(beat_path, always_2d=True)
630
+ if bsr != 44100:
631
+ beat = np.stack(
632
+ [librosa.resample(np.ascontiguousarray(beat[:, ch]).astype(np.float32),
633
+ orig_sr=bsr, target_sr=44100)
634
+ for ch in range(beat.shape[1])], axis=1)
635
+ bsr = 44100
636
+ if beat.shape[1] == 1:
637
+ beat = np.repeat(beat, 2, axis=1)
638
+ if vsr != 44100:
639
+ voc = librosa.resample(voc, orig_sr=vsr, target_sr=44100)
640
+
641
+ # High-pass to remove rumble / handling noise, trim leading-trailing silence
642
+ sos = butter(2, 75, btype="hp", fs=44100, output="sos")
643
+ voc = sosfilt(sos, voc).astype(np.float32)
644
+ voc, _ = librosa.effects.trim(voc, top_db=30)
645
+
646
+ def _norm(x, peak):
647
+ return (x / (np.max(np.abs(x)) + 1e-9)) * peak
648
+
649
+ beat = _norm(beat, beat_gain)
650
+ voc_st = _norm(np.repeat(voc[:, None], 2, axis=1), vocal_gain)
651
+
652
+ # 1 s intro so the beat breathes before the vocal enters
653
+ intro = int(1.0 * 44100)
654
+ n = max(beat.shape[0], voc_st.shape[0] + intro)
655
+ beat_p = np.pad(beat, ((0, max(0, n - beat.shape[0])), (0, 0)))
656
+ voc_p = np.pad(voc_st, ((intro, max(0, n - voc_st.shape[0] - intro)), (0, 0)))
657
+
658
+ mix = beat_p + voc_p
659
+ # Soft limiter for consistent loudness, then fades
660
+ mix = np.tanh(mix * 1.3) / np.tanh(1.3)
661
+ fade = int(0.5 * 44100)
662
+ env = np.ones(mix.shape[0])
663
+ env[:fade] = np.linspace(0, 1, fade)
664
+ env[-fade:] = np.linspace(1, 0, fade)
665
+ mix = mix * env[:, None]
666
+ peak = np.max(np.abs(mix)) + 1e-9
667
+ if peak > 0.99:
668
+ mix = mix / peak * 0.99
669
+ sf.write(out_path, mix.astype(np.float32), 44100)
670
+ return out_path
671
+
672
+
673
+ def run_rap_pipeline(
674
+ rap_vocal, beat_prompt, bpm, auto_bpm, duration_extra,
675
+ use_clone, voice_ref, vc_steps, auto_f0, pitch_shift,
676
+ progress=gr.Progress(),
677
+ ):
678
+ device = "cuda" if torch.cuda.is_available() else "cpu"
679
+ tmp = tempfile.mkdtemp(prefix="rap_")
680
+ try:
681
+ if rap_vocal is None:
682
+ yield "❌ Record or upload a rap vocal first.", None, None, None
683
+ return
684
+
685
+ import librosa
686
+ yield "⏳ **Step 1/4** — analyzing your recording…", None, None, None
687
+ vocal_np, vocal_sr = librosa.load(rap_vocal, sr=None, mono=True)
688
+ rec_dur = len(vocal_np) / vocal_sr
689
+ if rec_dur < 3:
690
+ raise RuntimeError(f"Recording too short ({rec_dur:.1f}s) — rap at least a few bars.")
691
+
692
+ est_bpm = None
693
+ if auto_bpm:
694
+ try:
695
+ tempo = librosa.beat.beat_track(y=vocal_np, sr=vocal_sr)[0]
696
+ est_bpm = float(np.atleast_1d(tempo)[0]) or None
697
+ except Exception:
698
+ est_bpm = None
699
+ use_bpm = int(round(est_bpm)) if (auto_bpm and est_bpm) else int(bpm)
700
+
701
+ yield (
702
+ f"🎛️ **Step 2/4** — generating the matching instrumental "
703
+ f"({rec_dur:.0f}s vocal, {use_bpm} BPM)…",
704
+ None, None, None,
705
+ )
706
+ dit, llm = _load_ace(device)
707
+ from acestep.inference import GenerationParams, GenerationConfig, generate_music
708
+
709
+ caption = (beat_prompt or "hard-hitting rap beat").strip()[:500]
710
+ caption = f"{caption}, rap hip-hop instrumental backing track, no vocals"
711
+ beat_duration = max(10.0, min(600.0, float(rec_dur) + float(duration_extra)))
712
+ params = GenerationParams(
713
+ task_type="text2music",
714
+ thinking=True,
715
+ caption=caption,
716
+ lyrics="[Instrumental]",
717
+ instrumental=True,
718
+ vocal_language="unknown",
719
+ duration=beat_duration,
720
+ bpm=use_bpm,
721
+ inference_steps=8,
722
+ guidance_scale=1.0,
723
+ shift=3.0,
724
+ seed=-1,
725
+ )
726
+ config = GenerationConfig(batch_size=1, audio_format="wav", use_random_seed=True)
727
+ result = generate_music(
728
+ dit, llm, params=params, config=config, save_dir=os.path.join(tmp, "ace")
729
+ )
730
+ if not result.success:
731
+ raise RuntimeError(result.status_message or result.error or "beat generation failed")
732
+ beat_path = result.audios[0]["path"]
733
+
734
+ voc_proc = rap_vocal
735
+ if use_clone and voice_ref is not None:
736
+ yield "🎤 **Step 3/4** — converting your recording to the cloned voice (Seed-VC)…", None, None, None
737
+ svc = _load_svc(device)
738
+ wave, sr = _voice_convert(
739
+ svc, rap_vocal, voice_ref,
740
+ diffusion_steps=int(vc_steps),
741
+ auto_f0_adjust=bool(auto_f0),
742
+ pitch_shift=int(pitch_shift),
743
+ )
744
+ voc_proc = os.path.join(tmp, "rap_converted.wav")
745
+ sf.write(voc_proc, wave.astype(np.float32), sr)
746
+
747
+ yield "✨ **Step 4/4** — polishing and mastering the mix…", None, None, None
748
+ final_path = _polish_and_mix(voc_proc, beat_path, os.path.join(tmp, "rap_final.wav"))
749
+ yield (
750
+ f"✅ **Done** — {rec_dur:.0f}s rap over a {use_bpm} BPM generated beat, "
751
+ "polished and mastered.",
752
+ final_path, beat_path, voc_proc,
753
+ )
754
+ except Exception as e:
755
+ yield f"❌ **Failed:** `{e}`", None, None, None
756
+
757
+
758
+ # ---------------------------------------------------------------------------
759
+ # AI Agent Studio: client brief -> LLM producer/lyricist -> variants -> Hub
760
+ # ---------------------------------------------------------------------------
761
+ AGENT_LLM_MODEL = os.environ.get("AGENT_LLM_MODEL", "Qwen/Qwen2.5-72B-Instruct")
762
+ OUTPUTS_REPO = os.environ.get("AGENT_OUTPUTS_REPO", "red-kit/sangeet-studio-outputs")
763
+
764
+
765
+ def _agent_llm():
766
+ token = os.environ.get("HF_TOKEN")
767
+ if not token:
768
+ return None
769
+ try:
770
+ from huggingface_hub import InferenceClient
771
+ return InferenceClient(model=AGENT_LLM_MODEL, token=token)
772
+ except Exception as e:
773
+ print(f"[agent] LLM client unavailable: {e}", flush=True)
774
+ return None
775
+
776
+
777
+ def _llm_json(client, system, user):
778
+ import json
779
+ import re
780
+ text = client.chat_completion(
781
+ messages=[{"role": "system", "content": system},
782
+ {"role": "user", "content": user}],
783
+ max_tokens=1800, temperature=0.85,
784
+ ).choices[0].message.content
785
+ m = re.search(r"\{.*\}", text, re.DOTALL)
786
+ if not m:
787
+ raise ValueError(f"LLM did not return JSON — got: {text[:300]}")
788
+ return json.loads(m.group(0))
789
+
790
+
791
+ def _agent_plan(client, artist_style, brief, song_type, language_label, duration):
792
+ """LLM producer: turns the client brief into a concrete production plan."""
793
+ if client is None:
794
+ caption = f"{artist_style or song_type}, {song_type}, polished studio production"
795
+ lyrics = (brief or "").strip()
796
+ if not lyrics:
797
+ lyrics = f"[Verse]\n{song_type} night, city lights and longing\n[Chorus]\nRaise your heart, sing along"
798
+ return {"caption": caption, "lyrics": lyrics, "bpm": None,
799
+ "lora_scale": 0.8, "language": "unknown",
800
+ "note": "Fallback planner — add the HF_TOKEN secret for the full LLM lyricist."}
801
+ system = (
802
+ "You are a world-class music producer and lyricist for South Asian music "
803
+ "(Bollywood, qawwali, ghazal, Punjabi, Sufi, Carnatic, filmi, hip-hop). "
804
+ "Given a client brief, return ONLY a JSON object with keys: "
805
+ '"caption" (rich English style prompt for a music model: instruments, mood, tempo, era, vocal character), '
806
+ '"lyrics" (full song lyrics in the requested language using [Verse]/[Chorus]/[Bridge] tags; original words, no real artist names), '
807
+ '"bpm" (integer 60-180 or null for auto), '
808
+ '"lora_scale" (0.0-1.0 strength of Indian-style LoRA), '
809
+ '"language" (ISO 639-1 code for the vocals), '
810
+ '"note" (one-line production note for the client).'
811
+ )
812
+ user = (
813
+ f"Artist style (described in words — inspiration, not a voice to clone): {artist_style or 'producer\'s choice'}\n"
814
+ f"Song type/genre: {song_type}\n"
815
+ f"Vocal language requested: {language_label}\n"
816
+ f"Target duration: {int(duration)} seconds\n"
817
+ f"Client lyrics (full lyrics if provided) or keywords/themes: {brief or 'none — write from the style'}\n\n"
818
+ "If the client provided full lyrics, polish structure and flow but keep their words; return them as 'lyrics'. "
819
+ "If they provided keywords, write original lyrics from those themes. "
820
+ "Match lyric length to the target duration (roughly 2 words per second for sung vocals)."
821
+ )
822
+ return _llm_json(client, system, user)
823
+
824
+
825
+ def _save_variant_to_hub(wav_path, name):
826
+ """Returns (link, error): link is set on success, error carries the real
827
+ failure (e.g. a token without write access to the outputs dataset) so it
828
+ can be shown in the agent chat instead of silently vanishing."""
829
+ token = os.environ.get("HF_TOKEN")
830
+ if not token:
831
+ return None, None
832
+ try:
833
+ from huggingface_hub import HfApi
834
+ HfApi(token=token).upload_file(
835
+ path_or_fileobj=wav_path,
836
+ path_in_repo=f"variants/{name}.wav",
837
+ repo_id=OUTPUTS_REPO,
838
+ repo_type="dataset",
839
+ )
840
+ return f"https://huggingface.co/datasets/{OUTPUTS_REPO}/blob/main/variants/{name}.wav", None
841
+ except Exception as e:
842
+ print(f"[agent] saving variant failed: {e}", flush=True)
843
+ return None, str(e)
844
+
845
+
846
+ def run_agent_pipeline(
847
+ artist_style, brief, song_type, agent_language, duration, n_variants,
848
+ voice_ref, consent, vc_steps, auto_f0, pitch_shift,
849
+ progress=gr.Progress(),
850
+ ):
851
+ import random
852
+ import time as _time
853
+ device = "cuda" if torch.cuda.is_available() else "cpu"
854
+ tmp = tempfile.mkdtemp(prefix="agent_")
855
+ chat = []
856
+ outs = [None, None, None]
857
+
858
+ def emit(text):
859
+ chat.append({"role": "assistant", "content": text})
860
+ return (list(chat), outs[0], outs[1], outs[2])
861
+
862
+ try:
863
+ if not (artist_style or "").strip() and not (brief or "").strip():
864
+ yield emit("❌ Give me at least a style, a song type, or lyrics/keywords to work with.")
865
+ return
866
+ if voice_ref is not None and not consent:
867
+ yield emit("⚠️ Voice conversion needs the consent checkbox ticked — you must own the voice or have permission to use it.")
868
+ return
869
+ n_variants = int(max(1, min(3, n_variants)))
870
+
871
+ yield emit("🧠 **Producer agent:** planning the production and writing the lyrics…")
872
+ client = _agent_llm()
873
+ lang_label = dict(LANGS).get(agent_language, "auto")
874
+ plan = _agent_plan(client, artist_style, brief, song_type, lang_label, duration)
875
+ caption = str(plan.get("caption", "")).strip()[:500]
876
+ lyrics = str(plan.get("lyrics", "")).strip()[:4000]
877
+ bpm = plan.get("bpm")
878
+ bpm = int(bpm) if bpm else None
879
+ lora_scale = float(plan.get("lora_scale", 0.8))
880
+ plan_language = str(plan.get("language") or agent_language or "unknown")
881
+ note = str(plan.get("note", ""))
882
+
883
+ yield emit(
884
+ f"📋 **Production plan**\n\n"
885
+ f"- Style: `{caption}`\n"
886
+ f"- BPM: {bpm or 'auto'} · LoRA strength: {lora_scale:.2f} · Language: `{plan_language}`\n"
887
+ f"- Producer note: {note}\n"
888
+ f"- Lyricist: `{AGENT_LLM_MODEL}`" + ("" if client else " *(fallback planner — add the HF_TOKEN secret for the full LLM)*"),
889
+ )
890
+
891
+ dit, llm = _load_ace(device)
892
+ dit.set_use_lora(True)
893
+ dit.set_lora_scale(lora_scale)
894
+
895
+ from acestep.inference import GenerationParams, GenerationConfig, generate_music
896
+
897
+ links = []
898
+ for i in range(n_variants):
899
+ seed = random.randint(0, 999999)
900
+ yield emit(f"🎛️ **Variant {i + 1}/{n_variants}** — generating (seed {seed})…")
901
+ params = GenerationParams(
902
+ task_type="text2music",
903
+ thinking=True,
904
+ caption=caption,
905
+ lyrics=lyrics or "[Instrumental]",
906
+ instrumental=not lyrics,
907
+ vocal_language=plan_language,
908
+ duration=float(duration),
909
+ bpm=bpm,
910
+ inference_steps=8,
911
+ guidance_scale=1.0,
912
+ shift=3.0,
913
+ seed=seed,
914
+ )
915
+ config = GenerationConfig(batch_size=1, audio_format="wav", use_random_seed=False)
916
+ result = generate_music(
917
+ dit, llm, params=params, config=config, save_dir=os.path.join(tmp, f"v{i}")
918
+ )
919
+ if not result.success:
920
+ raise RuntimeError(result.status_message or result.error or "generation failed")
921
+ song_path = result.audios[0]["path"]
922
+
923
+ if voice_ref is not None:
924
+ yield emit(f"🎤 **Variant {i + 1}** — swapping in the reference voice…")
925
+ vocals_path, no_vocals_path = _split_stems(song_path, os.path.join(tmp, f"stems{i}"))
926
+ svc = _load_svc(device)
927
+ wave, sr = _voice_convert(
928
+ svc, vocals_path, voice_ref,
929
+ diffusion_steps=int(vc_steps),
930
+ auto_f0_adjust=bool(auto_f0),
931
+ pitch_shift=int(pitch_shift),
932
+ )
933
+ converted = os.path.join(tmp, f"conv{i}.wav")
934
+ sf.write(converted, wave.astype(np.float32), sr)
935
+ song_path = _remix(no_vocals_path, converted, os.path.join(tmp, f"final{i}.wav"))
936
+
937
+ outs[i] = song_path
938
+ stamp = _time.strftime("%Y%m%d-%H%M%S")
939
+ link, save_err = _save_variant_to_hub(song_path, f"{stamp}-variant{i + 1}")
940
+ if link:
941
+ links.append(f"[Variant {i + 1}]({link})")
942
+ yield emit(f"✅ **Variant {i + 1}** ready — saved to the Hub: {link}")
943
+ elif save_err:
944
+ yield emit(
945
+ f"✅ **Variant {i + 1}** ready, but archiving to the Hub failed: `{save_err}` "
946
+ f"— check that the HF_TOKEN secret has write access to `{OUTPUTS_REPO}`."
947
+ )
948
+ else:
949
+ yield emit(f"✅ **Variant {i + 1}** ready (add the HF_TOKEN secret to also archive variants on the Hub).")
950
+
951
+ if links:
952
+ yield emit("📦 **All variants saved:**\n" + "\n".join(links))
953
+ else:
954
+ yield emit("📦 All variants delivered above.")
955
+ except Exception as e:
956
+ chat.append({"role": "assistant", "content": f"❌ **Failed:** `{e}`"})
957
+ yield (list(chat), outs[0], outs[1], outs[2])
958
+
959
+
960
+ # ---------------------------------------------------------------------------
961
+ # Gradio UI
962
+ # ---------------------------------------------------------------------------
963
+ print("[startup] building Gradio UI…", flush=True)
964
+ demo = gr.Blocks(title="Sangeet Studio — AI Song Generation + Voice Cloning")
965
+ with demo:
966
+ gr.Markdown(
967
+ """# 🎵 Sangeet Studio
968
+ **Suno-style full-song generation (ACE-Step 1.5 turbo + RagaLoRA for Indian styles) with
969
+ instant artist-voice cloning (Seed-VC).**
970
+
971
+ 1. Describe the style, paste lyrics (write them in the **native script** — Devanagari,
972
+ Shahmukhi, Gurmukhi, Tamil… — transliteration degrades pronunciation), pick the language.
973
+ 2. Optionally upload a **5–25 s clean reference clip** of the artist voice you want singing it.
974
+ 3. Hit generate. The app produces the song, splits the stems, converts the vocals into the
975
+ cloned voice, and remixes.
976
+ """
977
+ )
978
+ with gr.Row():
979
+ with gr.Column(scale=2):
980
+ caption = gr.Textbox(
981
+ label="Style prompt",
982
+ placeholder="e.g. Coke Studio style Sufi qawwali with tabla, harmonium, deep male vocals",
983
+ )
984
+ lyrics = gr.Textbox(
985
+ label="Lyrics (use section tags like [Verse] / [Chorus])",
986
+ lines=7,
987
+ placeholder="[Verse]\nراند کھڑکی اے...\n[Chorus]\n...",
988
+ )
989
+ language = gr.Dropdown(
990
+ choices=LANGS, value="hi", label="Vocal language",
991
+ info="Punjabi/Urdu and other long-tail languages are supported but rougher than Hindi/English.",
992
+ )
993
+ duration = gr.Slider(10, 240, value=60, step=5, label="Duration (seconds)")
994
+ instrumental = gr.Checkbox(False, label="Instrumental (no vocals)")
995
+ with gr.Accordion("Generation options", open=False):
996
+ thinking = gr.Checkbox(True, label="LM planning (CoT) — better structure, a bit slower")
997
+ use_lora = gr.Checkbox(True, label="Use RagaLoRA (Indian music styles)")
998
+ lora_scale = gr.Slider(0.0, 1.0, value=0.8, step=0.05, label="RagaLoRA strength")
999
+ seed = gr.Number(value=-1, precision=0, label="Seed (-1 = random)")
1000
+ with gr.Column(scale=2):
1001
+ voice_ref = gr.Audio(
1002
+ label="Artist voice reference (5–25 s clean clip, optional)",
1003
+ type="filepath",
1004
+ )
1005
+ vc_steps = gr.Slider(5, 100, value=30, step=5,
1006
+ label="Voice-conversion diffusion steps",
1007
+ info="30–50 = best quality, slower")
1008
+ auto_f0 = gr.Checkbox(True, label="Auto F0 adjust (match the clone's pitch range)")
1009
+ pitch_shift = gr.Slider(-24, 24, value=0, step=1,
1010
+ label="Pitch shift (semitones)", info="e.g. -5 for male↔female correction")
1011
+ with gr.Accordion("🎤 Live Rap Studio — record → auto beat → polished track", open=False):
1012
+ gr.Markdown(
1013
+ """**Record your rap with the browser mic** (or upload a dry vocal). The app estimates the
1014
+ tempo, generates a **matching rap instrumental in the same style**, keeps **your voice as the
1015
+ artist** (or converts it to a cloned artist voice), then polishes and masters the mix into a
1016
+ production-ready track."""
1017
+ )
1018
+ with gr.Row():
1019
+ rap_vocal = gr.Audio(
1020
+ label="Your rap vocal — record with mic or upload",
1021
+ sources=["microphone", "upload"],
1022
+ type="filepath",
1023
+ )
1024
+ with gr.Column():
1025
+ beat_prompt = gr.Textbox(
1026
+ label="Beat style prompt",
1027
+ value="hard-hitting Indian hip-hop trap beat, 808 bass, tabla accents, moody",
1028
+ )
1029
+ with gr.Row():
1030
+ bpm = gr.Slider(60, 180, value=90, step=1, label="BPM fallback",
1031
+ info="Used when auto-estimation is unsure")
1032
+ auto_bpm = gr.Checkbox(True, label="Auto-estimate BPM from my recording")
1033
+ duration_extra = gr.Slider(0, 15, value=4, step=1,
1034
+ label="Beat length beyond vocal (s)")
1035
+ with gr.Row():
1036
+ use_clone = gr.Checkbox(False, label="Convert my recording to a cloned artist voice")
1037
+ rap_voice_ref = gr.Audio(label="Artist voice reference (if converting)",
1038
+ sources=["upload"], type="filepath")
1039
+ rap_vc_steps = gr.Slider(5, 100, value=30, step=5, label="VC diffusion steps")
1040
+ rap_auto_f0 = gr.Checkbox(True, label="Auto F0 adjust")
1041
+ rap_pitch = gr.Slider(-24, 24, value=0, step=1, label="Pitch shift (semitones)")
1042
+ rap_btn = gr.Button("🎧 Turn my recording into a production-ready rap", variant="primary")
1043
+ rap_status = gr.Markdown("Ready.")
1044
+ with gr.Row():
1045
+ rap_final = gr.Audio(label="Final polished rap")
1046
+ rap_beat = gr.Audio(label="Generated instrumental")
1047
+ rap_raw = gr.Audio(label="Processed vocal (as mixed)")
1048
+
1049
+ with gr.Accordion("🤖 AI Agent Studio — brief in, finished songs out", open=False):
1050
+ gr.Markdown(
1051
+ """Describe the **style in words**, give **full lyrics or keywords** + song type. The agent plans the
1052
+ production, writes/polishes lyrics with an LLM, generates **multiple variants**, converts them into your
1053
+ reference voice if provided (consented, user-supplied), and **saves every variant to the Hub**.
1054
+ Add the `HF_TOKEN` secret to this Space to enable the LLM lyricist and Hub archiving."""
1055
+ )
1056
+ with gr.Row():
1057
+ with gr.Column():
1058
+ agent_style = gr.Textbox(
1059
+ label="Artist style (describe in words)",
1060
+ placeholder="e.g. soulful devotional qawwali intensity, soaring male vocals, harmonium + tabla",
1061
+ )
1062
+ agent_brief = gr.Textbox(
1063
+ label="Full lyrics OR keywords/themes",
1064
+ lines=5,
1065
+ placeholder="Paste full lyrics (kept + polished) — or just themes: ‘monsoon, first love, train station’",
1066
+ )
1067
+ with gr.Row():
1068
+ agent_song_type = gr.Dropdown(
1069
+ ["Bollywood", "Qawwali", "Ghazal", "Punjabi Bhangra", "Punjabi Pop",
1070
+ "Sufi Rock", "Filmi", "Carnatic Fusion", "Hindustani Classical",
1071
+ "Bhajan / Devotional", "Indian Hip-Hop / Rap", "Indie"],
1072
+ value="Sufi Rock", label="Song type",
1073
+ )
1074
+ agent_language = gr.Dropdown(choices=LANGS, value="hi", label="Vocal language")
1075
+ with gr.Column():
1076
+ agent_duration = gr.Slider(10, 240, value=60, step=5, label="Duration per variant (s)")
1077
+ agent_n = gr.Slider(1, 3, value=2, step=1, label="Number of variants")
1078
+ agent_voice = gr.Audio(
1079
+ label="Reference voice (optional — user-supplied)",
1080
+ sources=["upload"], type="filepath",
1081
+ )
1082
+ agent_consent = gr.Checkbox(
1083
+ False,
1084
+ label="I confirm I have the right to use this voice (own voice or licensed/consented reference)",
1085
+ )
1086
+ with gr.Row():
1087
+ agent_vc_steps = gr.Slider(5, 100, value=30, step=5, label="VC diffusion steps")
1088
+ agent_auto_f0 = gr.Checkbox(True, label="Auto F0 adjust")
1089
+ agent_pitch = gr.Slider(-24, 24, value=0, step=1, label="Pitch shift (semitones)")
1090
+ agent_btn = gr.Button("🚀 Produce the songs", variant="primary")
1091
+ agent_chat = gr.Chatbot(label="Agent chat", height=380)
1092
+ with gr.Row():
1093
+ agent_v1 = gr.Audio(label="Variant 1")
1094
+ agent_v2 = gr.Audio(label="Variant 2")
1095
+ agent_v3 = gr.Audio(label="Variant 3")
1096
+
1097
+ btn = gr.Button("🎵 Generate song (+ clone voice if reference given)", variant="primary")
1098
+ status = gr.Markdown("Ready. First generation loads the models (~2–5 min on GPU).")
1099
+ with gr.Row():
1100
+ final_audio = gr.Audio(label="Final song")
1101
+ original_audio = gr.Audio(label="Original generated song")
1102
+ with gr.Row():
1103
+ inst_audio = gr.Audio(label="Instrumental stem")
1104
+ voc_audio = gr.Audio(label="Vocals stem (voice-converted)")
1105
+
1106
+ btn.click(
1107
+ run_pipeline,
1108
+ inputs=[caption, lyrics, language, duration, instrumental, thinking,
1109
+ use_lora, lora_scale, seed,
1110
+ voice_ref, vc_steps, auto_f0, pitch_shift],
1111
+ outputs=[status, final_audio, original_audio, inst_audio, voc_audio],
1112
+ )
1113
+
1114
+ rap_btn.click(
1115
+ run_rap_pipeline,
1116
+ inputs=[rap_vocal, beat_prompt, bpm, auto_bpm, duration_extra,
1117
+ use_clone, rap_voice_ref, rap_vc_steps, rap_auto_f0, rap_pitch],
1118
+ outputs=[rap_status, rap_final, rap_beat, rap_raw],
1119
+ )
1120
+
1121
+ agent_btn.click(
1122
+ run_agent_pipeline,
1123
+ inputs=[agent_style, agent_brief, agent_song_type, agent_language, agent_duration,
1124
+ agent_n, agent_voice, agent_consent, agent_vc_steps, agent_auto_f0, agent_pitch],
1125
+ outputs=[agent_chat, agent_v1, agent_v2, agent_v3],
1126
+ )
1127
+
1128
+ demo.queue(max_size=8).launch()
1129
+ print("[startup] Gradio is up on port 7860", flush=True)