Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 9.15 kB
-
https://huggingface.co/spaces/hugging-apps/duplex-0/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/duplex-0/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/duplex-0/resolve/main/app.py
9.15 kB
| import spaces # must come before torch | |
| import time | |
| import gradio as gr | |
| import librosa | |
| import numpy as np | |
| import sentencepiece as spm | |
| import torch | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| from transformers import MimiModel | |
| from duplex import streaming | |
| from duplex.checkpoint import load_published | |
| REPO = "Trelis/duplex-0" | |
| SR = 24000 | |
| FPS = 12.5 | |
| SAMPLES_PER_FRAME = 1920 | |
| MAX_INPUT_S = 30.0 | |
| MAX_GEN_S = 30.0 | |
| CHECKPOINTS = { | |
| "Audio only": "conversational-none", | |
| "One text track": "conversational-one", | |
| "Two text tracks": "conversational-two", | |
| } | |
| root = snapshot_download( | |
| REPO, | |
| allow_patterns=[f"resplit-20261117/{c}/{f}" for c in CHECKPOINTS.values() | |
| for f in ("config.json", "model.safetensors")], | |
| ) | |
| MODELS = { | |
| name: load_published(f"{root}/resplit-20261117/{folder}").to("cuda").eval() | |
| for name, folder in CHECKPOINTS.items() | |
| } | |
| mimi = MimiModel.from_pretrained("kyutai/mimi").to("cuda").eval() | |
| sp = spm.SentencePieceProcessor( | |
| model_file=hf_hub_download("kyutai/moshiko-pytorch-bf16", "tokenizer_spm_32k_3.model") | |
| ) | |
| def _load_audio(path): | |
| """Audio file -> list of 1 or 2 float32 channels at 24 kHz, at most MAX_INPUT_S.""" | |
| y, _ = librosa.load(path, sr=SR, mono=False, duration=MAX_INPUT_S) | |
| if y.ndim == 1: | |
| return [y] | |
| if y.shape[0] >= 2 and not np.allclose(y[0], y[1], atol=1e-4): | |
| return [y[0], y[1]] | |
| return [y.mean(0)] | |
| def _encode(wave): | |
| x = torch.from_numpy(np.ascontiguousarray(wave, dtype=np.float32)).to("cuda") | |
| codes = mimi.encode(x[None, None], num_quantizers=8).audio_codes # [1, 8, T] | |
| return codes[0].T.long() # [T, 8] | |
| def _decode(codes): | |
| """codes [T, 8] (negative = absent) -> waveform numpy.""" | |
| codes = codes.clamp(min=0) | |
| audio = mimi.decode(codes.T[None]).audio_values # [1, 1, samples] | |
| return audio[0, 0].float().cpu().numpy() | |
| def _text(written, tracks): | |
| lines = [] | |
| names = {0: "Track 1 (caller / other speaker)", 1: "Track 2 (model's own voice)"} | |
| for track in tracks: | |
| ids = [i for frame in written.get(track, []) for i in frame] | |
| txt = sp.decode(ids).strip() if ids else "" | |
| lines.append(f"**{names[track]}:** {txt or '_(nothing written)_'}") | |
| return "\n\n".join(lines) | |
| def _duration(audio, mode="Reply", checkpoint="Audio only", seconds=10.0, *args, **kwargs): | |
| # Measured on ZeroGPU: ~0.027 s per 80 ms frame (both tracks) + a few seconds of overhead. | |
| try: | |
| clip = min(MAX_INPUT_S, librosa.get_duration(path=audio)) | |
| except Exception: | |
| clip = MAX_INPUT_S | |
| frames = (clip + min(float(seconds), MAX_GEN_S)) * FPS | |
| return int(1.15 * (6 + 0.03 * frames)) + 1 | |
| def generate( | |
| audio: str, | |
| mode: str = "Reply", | |
| checkpoint: str = "Audio only", | |
| seconds: float = 10.0, | |
| temperature: float = 1.0, | |
| top_k: int = 250, | |
| seed: int = 1, | |
| ): | |
| """Run duplex-0 on a speech clip. | |
| Args: | |
| audio: Path to the input speech (wav/mp3). Mono, or stereo with one speaker per channel. | |
| mode: "Reply" - the model listens to the clip as the caller (track 1) and speaks as | |
| the other side (track 2) the whole time, then for `seconds` more after the clip ends. | |
| "Continue" - the clip is a prompt and the model continues it for `seconds`. | |
| checkpoint: "Audio only", "One text track" or "Two text tracks" (conversational stage). | |
| seconds: Seconds to generate after the clip ends. | |
| temperature: Audio sampling temperature. | |
| top_k: Audio top-k. | |
| seed: Random seed. | |
| Returns: | |
| Mixed audio of both tracks, the model's own track, and the text it wrote. | |
| """ | |
| if audio is None: | |
| raise gr.Error("Please provide an audio clip.") | |
| t0 = time.perf_counter() | |
| model = MODELS[checkpoint] | |
| seconds = float(min(max(seconds, 1.0), MAX_GEN_S)) | |
| channels = _load_audio(audio) | |
| if len(channels[0]) < SR: | |
| raise gr.Error("The clip must be at least 1 second long.") | |
| new = int(round(seconds * FPS)) | |
| SAMPLE, ABSENT = streaming.SAMPLE, streaming.ABSENT | |
| if mode == "Reply": | |
| caller = _encode(channels[0] if len(channels) == 1 else (channels[0] + channels[1]) / 2) | |
| silence = _encode(np.zeros(new * SAMPLES_PER_FRAME, dtype=np.float32))[:new] | |
| caller = torch.cat((caller, silence)) | |
| frames = torch.full((len(caller), 16), SAMPLE, dtype=torch.long, device="cuda") | |
| frames[:, :8] = caller | |
| model_tracks = [1] | |
| else: | |
| prompt = [_encode(c) for c in channels] | |
| t = min(len(p) for p in prompt) | |
| frames = torch.full((t + new, 16), SAMPLE, dtype=torch.long, device="cuda") | |
| if len(prompt) == 2: | |
| frames[:t, :8] = prompt[0][:t] | |
| frames[:t, 8:] = prompt[1][:t] | |
| model_tracks = [0, 1] | |
| else: | |
| # Single speaker: the clip is track 2, the other track is empty (as in | |
| # single-speaker pre-training data). | |
| frames[:, :8] = ABSENT | |
| frames[:t, 8:] = prompt[0][:t] | |
| model_tracks = [1] | |
| generator = torch.Generator("cuda").manual_seed(int(seed)) | |
| with torch.inference_mode(): | |
| result, written = streaming.generate( | |
| model, frames, temperature=float(temperature), generator=generator, | |
| top_k=int(top_k), | |
| ) | |
| tracks = [] | |
| for j in range(2): | |
| codes = result[:, 8 * j: 8 * j + 8] | |
| if bool((codes < 0).all()): | |
| tracks.append(None) | |
| else: | |
| tracks.append(_decode(codes)) | |
| n = max(len(w) for w in tracks if w is not None) | |
| waves = [np.zeros(n, np.float32) if w is None else np.pad(w, (0, n - len(w))) for w in tracks] | |
| mix = waves[0] + waves[1] | |
| mix = mix / max(1.0, float(np.abs(mix).max()) / 0.95) | |
| own = waves[1] / max(1.0, float(np.abs(waves[1]).max()) / 0.95) | |
| text = _text(written, model.config.text_tracks) if model.config.text_tracks else \ | |
| "_The audio-only checkpoint writes no text — pick a text-track checkpoint to see its words._" | |
| print(f"[duplex-0] mode={mode} ckpt={checkpoint} frames={len(frames)} " | |
| f"time={time.perf_counter() - t0:.1f}s", flush=True) | |
| return (SR, (mix * 32767).astype(np.int16)), (SR, (own * 32767).astype(np.int16)), text | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks() as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown( | |
| "# duplex-0\n" | |
| "A small (201M) full-duplex speech-to-speech research model from Trelis, trained from scratch on " | |
| "~750 h of English speech over two [Mimi](https://huggingface.co/kyutai/mimi) token tracks, one per speaker. " | |
| "**Reply**: the model hears your clip as the caller and talks as the other side, in real time. " | |
| "**Continue**: the clip is a prompt and the model keeps the conversation going. " | |
| "It's a research checkpoint, so the speech is only about half as intelligible as real speech. " | |
| "[Model card](https://huggingface.co/Trelis/duplex-0)" | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| audio_in = gr.Audio(label="Input speech (≤30 s; stereo = one speaker per channel)", | |
| type="filepath", sources=["upload", "microphone"]) | |
| mode = gr.Radio(["Reply", "Continue"], value="Reply", label="Mode") | |
| checkpoint = gr.Radio(list(CHECKPOINTS), value="Audio only", label="Checkpoint") | |
| seconds = gr.Slider(2, MAX_GEN_S, value=10, step=1, | |
| label="Seconds to generate after the clip") | |
| run = gr.Button("Generate", variant="primary") | |
| with gr.Column(): | |
| mix_out = gr.Audio(label="Both tracks mixed", type="numpy") | |
| own_out = gr.Audio(label="Model's track (track 2)", type="numpy") | |
| text_out = gr.Markdown() | |
| with gr.Accordion("Advanced settings", open=False): | |
| temperature = gr.Slider(0.1, 1.5, value=1.0, step=0.05, label="Temperature") | |
| top_k = gr.Slider(1, 2048, value=250, step=1, label="Top-k") | |
| seed = gr.Number(value=1, precision=0, label="Seed") | |
| gr.Examples( | |
| examples=[ | |
| ["examples/prompt1.wav", "Continue"], | |
| ["examples/prompt2.wav", "Reply"], | |
| ["examples/prompt3.wav", "Continue"], | |
| ["examples/prompt4.wav", "Reply"], | |
| ], | |
| inputs=[audio_in, mode], | |
| outputs=[mix_out, own_out, text_out], | |
| fn=generate, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Held-out podcast prompts from the model card (10 s)", | |
| ) | |
| run.click(generate, [audio_in, mode, checkpoint, seconds, temperature, top_k, seed], | |
| [mix_out, own_out, text_out]) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |