"""VoTSpeech — instruction-following voice-design TTS (48 kHz) on ZeroGPU. Given synthesis text and a natural-language voice description, VoTSpeech first designs a continuous voice latent (flow-based Voice DiT conditioned on the instruction), then generates speech in that voice with the dots.tts-soar continuous autoregressive backbone + 48 kHz vocoder. Inference mirrors the authors' reference script (`inference/scripts/run_batch_infer_voice_design.sh` → `dots_tts.batch_cli`): template `voice_design`, bf16, speaker_scale 1.5, language "zh", max_generate_length 500, checkpoint-default voice sampler settings, seed 42. """ import spaces # ZeroGPU: MUST be imported before torch / dots_tts. import os os.environ.setdefault("DOTS_TTS_DELAYED_DIT_BACKEND", "sdpa") os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import random import tempfile import warnings from pathlib import Path warnings.filterwarnings("ignore", category=FutureWarning) import gradio as gr import soundfile as sf from dots_tts.runtime import DotsTtsRuntime from dots_tts.utils.logging import configure_logging from dots_tts.utils.util import seed_everything configure_logging(level="INFO") # silence per-patch DEBUG progress spam MODEL_ID = "Ywbn16/VoTSpeech" DEFAULT_SEED = 42 MAX_TEXT_CHARS = 300 MAX_INSTRUCTION_CHARS = 500 LANGUAGES = ["zh", "auto_detect", "none"] runtime = DotsTtsRuntime.from_pretrained( MODEL_ID, precision="bfloat16", optimize=False, # eager sdpa — torch.compile does not persist across ZeroGPU calls max_generate_length=500, max_sequence_length=2048, ) # Examples taken from the authors' own demo page (ywbn.github.io/VoTSpeech): # the README example, InstructTTSEval-ZH (APS / DSD / RP) and MINT cases. EXAMPLES = [ [ "今天阳光很好，我们一起出去走走吧。", "一位年轻女性，声音温暖明亮，语速自然，带着轻松愉快的情绪。", ], [ "那座古老的石桥静静地横跨在河上，见证了无数个春去秋来和人世间的悲欢离合。", "声音要听起来有厚度、有质感，像一个专业的男播音员。", ], [ "刚刚长大了，一定会开飞船的。", "小公主满怀憧憬地描述她心中的理想国度。", ], [ "知道了知道了，我去做还不行吗。真是的，一天到晚就知道使唤我，就不能让我休息一下吗？我的手都酸了，腰也疼，根本干不动活儿。", "模仿一个被逼着做事的熊孩子，说话时唉声叹气，满腹牢骚。", ], [ "自从你爸爸去世以后，我把所有的心血都放在你们兄弟俩身上，艰难的经营着这家医院。我不允许任何人来破坏它。", "声音应展现出中老年女性的特质,吐字清晰,前半段语速平缓,随着情感变化逐渐加快;音量正常开始,语气由隐忍转为愤慨,最终达到激昂高亢的状态。", ], [ "有些道理，是需要用一辈子的时间去慢慢理解的。", "年龄：老年, 音色：沙哑", ], ] def _gpu_duration(text, *args, **kwargs) -> int: # Measured on ZeroGPU: ~0.19 s of audio per CJK char at RTF ~1.0 # (66 chars -> 12.3 s audio in 12.2 s). Output is capped at 500 patches (~80 s). return int(min(90, max(15, (5 + 0.2 * len(text)) * 1.15))) @spaces.GPU(duration=_gpu_duration) def _generate(text: str, instruction: str, seed: int, language: str, speaker_scale: float): seed_everything(seed) result = runtime.generate( text=text, instruction=instruction, template_name="voice_design", language=language, speaker_scale=speaker_scale, ) audio = result["audio"].detach().float().cpu().squeeze().numpy() out_path = Path(tempfile.mkdtemp()) / f"votspeech_{seed}.wav" sf.write(str(out_path), audio, int(result["sample_rate"]), subtype="PCM_16") duration = audio.shape[-1] / result["sample_rate"] status = ( f"Generated {duration:.1f} s of audio in {result['time_used']:.1f} s " f"(RTF {result['rtf']:.2f}) · seed {seed} · {result['sample_rate'] // 1000} kHz" ) return str(out_path), status def synthesize( text: str, instruction: str, seed: float = DEFAULT_SEED, randomize_seed: bool = False, language: str = "zh", speaker_scale: float = 1.5, ): """Speak `text` in a voice designed from a natural-language description (48 kHz). Args: text: The words to speak (VoTSpeech is trained for Mandarin Chinese). instruction: Natural-language description of the desired voice (gender, age, timbre, emotion, speaking style, persona / scenario...). seed: Generation seed — different seeds design different voices. randomize_seed: Ignore `seed` and pick a random one. language: Language tag added to the text: "zh", "auto_detect" or "none". speaker_scale: Strength of the designed-voice conditioning (default 1.5). Returns: (path to the generated 48 kHz wav, status line with timing and seed) """ text = (text or "").strip() instruction = (instruction or "").strip() if not text: raise gr.Error("Please enter the text to speak.") if not instruction: raise gr.Error("Please enter a voice description (instruction).") if len(text) > MAX_TEXT_CHARS: raise gr.Error(f"Text is too long; use at most {MAX_TEXT_CHARS} characters.") if len(instruction) > MAX_INSTRUCTION_CHARS: raise gr.Error( f"Instruction is too long; use at most {MAX_INSTRUCTION_CHARS} characters." ) if randomize_seed: seed = random.randint(0, 2**31 - 1) seed = int(seed) try: return _generate(text, instruction, seed, language or "zh", float(speaker_scale)) except gr.Error: raise except Exception as exc: raise gr.Error(f"Generation failed: {exc}") from exc CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="VoTSpeech") as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# 🗣️ VoTSpeech\n" "Instruction-following TTS that **designs a voice from a text description**, " "then speaks your text in it at 48 kHz. Trained for **Mandarin Chinese** — " "describe the voice (gender, age, timbre, emotion, role…) in Chinese.\n\n" "[Model](https://huggingface.co/Ywbn16/VoTSpeech) · " "[Code](https://github.com/ywbn/VoTSpeech) · " "[Project page](https://ywbn.github.io/VoTSpeech/) · " "Weights: CC BY-NC 4.0 (non-commercial). Outputs are synthetic speech." ) with gr.Row(): with gr.Column(): text = gr.Textbox( label="Text to speak", lines=4, placeholder="今天阳光很好，我们一起出去走走吧。", ) instruction = gr.Textbox( label="Voice description (instruction)", lines=3, placeholder="一位年轻女性，声音温暖明亮，语速自然，带着轻松愉快的情绪。", ) generate_btn = gr.Button("Generate", variant="primary") with gr.Accordion("Advanced settings", open=False): seed = gr.Number(value=DEFAULT_SEED, precision=0, label="Seed") randomize_seed = gr.Checkbox(value=False, label="Randomize seed") language = gr.Dropdown( choices=LANGUAGES, value="zh", label="Language tag" ) speaker_scale = gr.Slider( 0.5, 3.0, value=1.5, step=0.1, label="Speaker (voice) scale" ) with gr.Column(): audio_out = gr.Audio(label="Generated speech (48 kHz)", type="filepath") status = gr.Markdown() gr.Examples( examples=EXAMPLES, inputs=[text, instruction], outputs=[audio_out, status], fn=synthesize, cache_examples=True, cache_mode="lazy", ) generate_btn.click( fn=synthesize, inputs=[text, instruction, seed, randomize_seed, language, speaker_scale], outputs=[audio_out, status], ) demo.queue(max_size=16) demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)