app.py

"""VoTSpeech — instruction-following voice-design TTS (48 kHz) on ZeroGPU.

Given synthesis text and a natural-language voice description, VoTSpeech first
designs a continuous voice latent (flow-based Voice DiT conditioned on the
instruction), then generates speech in that voice with the dots.tts-soar
continuous autoregressive backbone + 48 kHz vocoder.

Inference mirrors the authors' reference script
(`inference/scripts/run_batch_infer_voice_design.sh` → `dots_tts.batch_cli`):
template `voice_design`, bf16, speaker_scale 1.5, language "zh",
max_generate_length 500, checkpoint-default voice sampler settings, seed 42.
"""

import spaces  # ZeroGPU: MUST be imported before torch / dots_tts.

import os

os.environ.setdefault("DOTS_TTS_DELAYED_DIT_BACKEND", "sdpa")
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import random
import tempfile
import warnings
from pathlib import Path

warnings.filterwarnings("ignore", category=FutureWarning)

import gradio as gr
import soundfile as sf

from dots_tts.runtime import DotsTtsRuntime
from dots_tts.utils.logging import configure_logging
from dots_tts.utils.util import seed_everything

configure_logging(level="INFO")  # silence per-patch DEBUG progress spam

MODEL_ID = "Ywbn16/VoTSpeech"
DEFAULT_SEED = 42
MAX_TEXT_CHARS = 300
MAX_INSTRUCTION_CHARS = 500
LANGUAGES = ["zh", "auto_detect", "none"]

runtime = DotsTtsRuntime.from_pretrained(
    MODEL_ID,
    precision="bfloat16",
    optimize=False,  # eager sdpa — torch.compile does not persist across ZeroGPU calls
    max_generate_length=500,
    max_sequence_length=2048,
)

# Examples taken from the authors' own demo page (ywbn.github.io/VoTSpeech):
# the README example, InstructTTSEval-ZH (APS / DSD / RP) and MINT cases.
EXAMPLES = [
    [
        "今天阳光很好，我们一起出去走走吧。",
        "一位年轻女性，声音温暖明亮，语速自然，带着轻松愉快的情绪。",
    ],
    [
        "那座古老的石桥静静地横跨在河上，见证了无数个春去秋来和人世间的悲欢离合。",
        "声音要听起来有厚度、有质感，像一个专业的男播音员。",
    ],
    [
        "刚刚长大了，一定会开飞船的。",
        "小公主满怀憧憬地描述她心中的理想国度。",
    ],
    [
        "知道了知道了，我去做还不行吗。真是的，一天到晚就知道使唤我，就不能让我休息一下吗？我的手都酸了，腰也疼，根本干不动活儿。",
        "模仿一个被逼着做事的熊孩子，说话时唉声叹气，满腹牢骚。",
    ],
    [
        "自从你爸爸去世以后，我把所有的心血都放在你们兄弟俩身上，艰难的经营着这家医院。我不允许任何人来破坏它。",
        "声音应展现出中老年女性的特质,吐字清晰,前半段语速平缓,随着情感变化逐渐加快;音量正常开始,语气由隐忍转为愤慨,最终达到激昂高亢的状态。",
    ],
    [
        "有些道理，是需要用一辈子的时间去慢慢理解的。",
        "年龄：老年, 音色：沙哑",
    ],
]


def _gpu_duration(text, *args, **kwargs) -> int:
    # Measured on ZeroGPU: ~0.19 s of audio per CJK char at RTF ~1.0
    # (66 chars -> 12.3 s audio in 12.2 s). Output is capped at 500 patches (~80 s).
    return int(min(90, max(15, (5 + 0.2 * len(text)) * 1.15)))


@spaces.GPU(duration=_gpu_duration)
def _generate(text: str, instruction: str, seed: int, language: str, speaker_scale: float):
    seed_everything(seed)
    result = runtime.generate(
        text=text,
        instruction=instruction,
        template_name="voice_design",
        language=language,
        speaker_scale=speaker_scale,
    )
    audio = result["audio"].detach().float().cpu().squeeze().numpy()
    out_path = Path(tempfile.mkdtemp()) / f"votspeech_{seed}.wav"
    sf.write(str(out_path), audio, int(result["sample_rate"]), subtype="PCM_16")
    duration = audio.shape[-1] / result["sample_rate"]
    status = (
        f"Generated {duration:.1f} s of audio in {result['time_used']:.1f} s "
        f"(RTF {result['rtf']:.2f}) · seed {seed} · {result['sample_rate'] // 1000} kHz"
    )
    return str(out_path), status


def synthesize(
    text: str,
    instruction: str,
    seed: float = DEFAULT_SEED,
    randomize_seed: bool = False,
    language: str = "zh",
    speaker_scale: float = 1.5,
):
    """Speak `text` in a voice designed from a natural-language description (48 kHz).

    Args:
        text: The words to speak (VoTSpeech is trained for Mandarin Chinese).
        instruction: Natural-language description of the desired voice
            (gender, age, timbre, emotion, speaking style, persona / scenario...).
        seed: Generation seed — different seeds design different voices.
        randomize_seed: Ignore `seed` and pick a random one.
        language: Language tag added to the text: "zh", "auto_detect" or "none".
        speaker_scale: Strength of the designed-voice conditioning (default 1.5).

    Returns:
        (path to the generated 48 kHz wav, status line with timing and seed)
    """
    text = (text or "").strip()
    instruction = (instruction or "").strip()
    if not text:
        raise gr.Error("Please enter the text to speak.")
    if not instruction:
        raise gr.Error("Please enter a voice description (instruction).")
    if len(text) > MAX_TEXT_CHARS:
        raise gr.Error(f"Text is too long; use at most {MAX_TEXT_CHARS} characters.")
    if len(instruction) > MAX_INSTRUCTION_CHARS:
        raise gr.Error(
            f"Instruction is too long; use at most {MAX_INSTRUCTION_CHARS} characters."
        )
    if randomize_seed:
        seed = random.randint(0, 2**31 - 1)
    seed = int(seed)
    try:
        return _generate(text, instruction, seed, language or "zh", float(speaker_scale))
    except gr.Error:
        raise
    except Exception as exc:
        raise gr.Error(f"Generation failed: {exc}") from exc


CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""

with gr.Blocks(title="VoTSpeech") as demo:
    with gr.Column(elem_id="col-container"):
        gr.Markdown(
            "# 🗣️ VoTSpeech\n"
            "Instruction-following TTS that **designs a voice from a text description**, "
            "then speaks your text in it at 48 kHz. Trained for **Mandarin Chinese** — "
            "describe the voice (gender, age, timbre, emotion, role…) in Chinese.\n\n"
            "[Model](https://huggingface.co/Ywbn16/VoTSpeech) · "
            "[Code](https://github.com/ywbn/VoTSpeech) · "
            "[Project page](https://ywbn.github.io/VoTSpeech/) · "
            "Weights: CC BY-NC 4.0 (non-commercial). Outputs are synthetic speech."
        )
        with gr.Row():
            with gr.Column():
                text = gr.Textbox(
                    label="Text to speak",
                    lines=4,
                    placeholder="今天阳光很好，我们一起出去走走吧。",
                )
                instruction = gr.Textbox(
                    label="Voice description (instruction)",
                    lines=3,
                    placeholder="一位年轻女性，声音温暖明亮，语速自然，带着轻松愉快的情绪。",
                )
                generate_btn = gr.Button("Generate", variant="primary")
                with gr.Accordion("Advanced settings", open=False):
                    seed = gr.Number(value=DEFAULT_SEED, precision=0, label="Seed")
                    randomize_seed = gr.Checkbox(value=False, label="Randomize seed")
                    language = gr.Dropdown(
                        choices=LANGUAGES, value="zh", label="Language tag"
                    )
                    speaker_scale = gr.Slider(
                        0.5, 3.0, value=1.5, step=0.1, label="Speaker (voice) scale"
                    )
            with gr.Column():
                audio_out = gr.Audio(label="Generated speech (48 kHz)", type="filepath")
                status = gr.Markdown()

        gr.Examples(
            examples=EXAMPLES,
            inputs=[text, instruction],
            outputs=[audio_out, status],
            fn=synthesize,
            cache_examples=True,
            cache_mode="lazy",
        )

    generate_btn.click(
        fn=synthesize,
        inputs=[text, instruction, seed, randomize_seed, language, speaker_scale],
        outputs=[audio_out, status],
    )

demo.queue(max_size=16)
demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)