import spaces # must come before torch import time import gradio as gr import librosa import numpy as np import sentencepiece as spm import torch from huggingface_hub import hf_hub_download, snapshot_download from transformers import MimiModel from duplex import streaming from duplex.checkpoint import load_published REPO = "Trelis/duplex-0" SR = 24000 FPS = 12.5 SAMPLES_PER_FRAME = 1920 MAX_INPUT_S = 30.0 MAX_GEN_S = 30.0 CHECKPOINTS = { "Audio only": "conversational-none", "One text track": "conversational-one", "Two text tracks": "conversational-two", } root = snapshot_download( REPO, allow_patterns=[f"resplit-20261117/{c}/{f}" for c in CHECKPOINTS.values() for f in ("config.json", "model.safetensors")], ) MODELS = { name: load_published(f"{root}/resplit-20261117/{folder}").to("cuda").eval() for name, folder in CHECKPOINTS.items() } mimi = MimiModel.from_pretrained("kyutai/mimi").to("cuda").eval() sp = spm.SentencePieceProcessor( model_file=hf_hub_download("kyutai/moshiko-pytorch-bf16", "tokenizer_spm_32k_3.model") ) def _load_audio(path): """Audio file -> list of 1 or 2 float32 channels at 24 kHz, at most MAX_INPUT_S.""" y, _ = librosa.load(path, sr=SR, mono=False, duration=MAX_INPUT_S) if y.ndim == 1: return [y] if y.shape[0] >= 2 and not np.allclose(y[0], y[1], atol=1e-4): return [y[0], y[1]] return [y.mean(0)] @torch.no_grad() def _encode(wave): x = torch.from_numpy(np.ascontiguousarray(wave, dtype=np.float32)).to("cuda") codes = mimi.encode(x[None, None], num_quantizers=8).audio_codes # [1, 8, T] return codes[0].T.long() # [T, 8] @torch.no_grad() def _decode(codes): """codes [T, 8] (negative = absent) -> waveform numpy.""" codes = codes.clamp(min=0) audio = mimi.decode(codes.T[None]).audio_values # [1, 1, samples] return audio[0, 0].float().cpu().numpy() def _text(written, tracks): lines = [] names = {0: "Track 1 (caller / other speaker)", 1: "Track 2 (model's own voice)"} for track in tracks: ids = [i for frame in written.get(track, []) for i in frame] txt = sp.decode(ids).strip() if ids else "" lines.append(f"**{names[track]}:** {txt or '_(nothing written)_'}") return "\n\n".join(lines) def _duration(audio, mode="Reply", checkpoint="Audio only", seconds=10.0, *args, **kwargs): # Measured on ZeroGPU: ~0.027 s per 80 ms frame (both tracks) + a few seconds of overhead. try: clip = min(MAX_INPUT_S, librosa.get_duration(path=audio)) except Exception: clip = MAX_INPUT_S frames = (clip + min(float(seconds), MAX_GEN_S)) * FPS return int(1.15 * (6 + 0.03 * frames)) + 1 @spaces.GPU(duration=_duration) def generate( audio: str, mode: str = "Reply", checkpoint: str = "Audio only", seconds: float = 10.0, temperature: float = 1.0, top_k: int = 250, seed: int = 1, ): """Run duplex-0 on a speech clip. Args: audio: Path to the input speech (wav/mp3). Mono, or stereo with one speaker per channel. mode: "Reply" - the model listens to the clip as the caller (track 1) and speaks as the other side (track 2) the whole time, then for `seconds` more after the clip ends. "Continue" - the clip is a prompt and the model continues it for `seconds`. checkpoint: "Audio only", "One text track" or "Two text tracks" (conversational stage). seconds: Seconds to generate after the clip ends. temperature: Audio sampling temperature. top_k: Audio top-k. seed: Random seed. Returns: Mixed audio of both tracks, the model's own track, and the text it wrote. """ if audio is None: raise gr.Error("Please provide an audio clip.") t0 = time.perf_counter() model = MODELS[checkpoint] seconds = float(min(max(seconds, 1.0), MAX_GEN_S)) channels = _load_audio(audio) if len(channels[0]) < SR: raise gr.Error("The clip must be at least 1 second long.") new = int(round(seconds * FPS)) SAMPLE, ABSENT = streaming.SAMPLE, streaming.ABSENT if mode == "Reply": caller = _encode(channels[0] if len(channels) == 1 else (channels[0] + channels[1]) / 2) silence = _encode(np.zeros(new * SAMPLES_PER_FRAME, dtype=np.float32))[:new] caller = torch.cat((caller, silence)) frames = torch.full((len(caller), 16), SAMPLE, dtype=torch.long, device="cuda") frames[:, :8] = caller model_tracks = [1] else: prompt = [_encode(c) for c in channels] t = min(len(p) for p in prompt) frames = torch.full((t + new, 16), SAMPLE, dtype=torch.long, device="cuda") if len(prompt) == 2: frames[:t, :8] = prompt[0][:t] frames[:t, 8:] = prompt[1][:t] model_tracks = [0, 1] else: # Single speaker: the clip is track 2, the other track is empty (as in # single-speaker pre-training data). frames[:, :8] = ABSENT frames[:t, 8:] = prompt[0][:t] model_tracks = [1] generator = torch.Generator("cuda").manual_seed(int(seed)) with torch.inference_mode(): result, written = streaming.generate( model, frames, temperature=float(temperature), generator=generator, top_k=int(top_k), ) tracks = [] for j in range(2): codes = result[:, 8 * j: 8 * j + 8] if bool((codes < 0).all()): tracks.append(None) else: tracks.append(_decode(codes)) n = max(len(w) for w in tracks if w is not None) waves = [np.zeros(n, np.float32) if w is None else np.pad(w, (0, n - len(w))) for w in tracks] mix = waves[0] + waves[1] mix = mix / max(1.0, float(np.abs(mix).max()) / 0.95) own = waves[1] / max(1.0, float(np.abs(waves[1]).max()) / 0.95) text = _text(written, model.config.text_tracks) if model.config.text_tracks else \ "_The audio-only checkpoint writes no text — pick a text-track checkpoint to see its words._" print(f"[duplex-0] mode={mode} ckpt={checkpoint} frames={len(frames)} " f"time={time.perf_counter() - t0:.1f}s", flush=True) return (SR, (mix * 32767).astype(np.int16)), (SR, (own * 32767).astype(np.int16)), text CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks() as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# duplex-0\n" "A small (201M) full-duplex speech-to-speech research model from Trelis, trained from scratch on " "~750 h of English speech over two [Mimi](https://huggingface.co/kyutai/mimi) token tracks, one per speaker. " "**Reply**: the model hears your clip as the caller and talks as the other side, in real time. " "**Continue**: the clip is a prompt and the model keeps the conversation going. " "It's a research checkpoint, so the speech is only about half as intelligible as real speech. " "[Model card](https://huggingface.co/Trelis/duplex-0)" ) with gr.Row(): with gr.Column(): audio_in = gr.Audio(label="Input speech (≤30 s; stereo = one speaker per channel)", type="filepath", sources=["upload", "microphone"]) mode = gr.Radio(["Reply", "Continue"], value="Reply", label="Mode") checkpoint = gr.Radio(list(CHECKPOINTS), value="Audio only", label="Checkpoint") seconds = gr.Slider(2, MAX_GEN_S, value=10, step=1, label="Seconds to generate after the clip") run = gr.Button("Generate", variant="primary") with gr.Column(): mix_out = gr.Audio(label="Both tracks mixed", type="numpy") own_out = gr.Audio(label="Model's track (track 2)", type="numpy") text_out = gr.Markdown() with gr.Accordion("Advanced settings", open=False): temperature = gr.Slider(0.1, 1.5, value=1.0, step=0.05, label="Temperature") top_k = gr.Slider(1, 2048, value=250, step=1, label="Top-k") seed = gr.Number(value=1, precision=0, label="Seed") gr.Examples( examples=[ ["examples/prompt1.wav", "Continue"], ["examples/prompt2.wav", "Reply"], ["examples/prompt3.wav", "Continue"], ["examples/prompt4.wav", "Reply"], ], inputs=[audio_in, mode], outputs=[mix_out, own_out, text_out], fn=generate, cache_examples=True, cache_mode="lazy", label="Held-out podcast prompts from the model card (10 s)", ) run.click(generate, [audio_in, mode, checkpoint, seconds, temperature, top_k, seed], [mix_out, own_out, text_out]) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)