duplex-0 / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
dbaa52d verified
Raw History Blame Contribute Delete
9.15 kB
import spaces # must come before torch
import time
import gradio as gr
import librosa
import numpy as np
import sentencepiece as spm
import torch
from huggingface_hub import hf_hub_download, snapshot_download
from transformers import MimiModel
from duplex import streaming
from duplex.checkpoint import load_published
REPO = "Trelis/duplex-0"
SR = 24000
FPS = 12.5
SAMPLES_PER_FRAME = 1920
MAX_INPUT_S = 30.0
MAX_GEN_S = 30.0
CHECKPOINTS = {
"Audio only": "conversational-none",
"One text track": "conversational-one",
"Two text tracks": "conversational-two",
}
root = snapshot_download(
REPO,
allow_patterns=[f"resplit-20261117/{c}/{f}" for c in CHECKPOINTS.values()
for f in ("config.json", "model.safetensors")],
)
MODELS = {
name: load_published(f"{root}/resplit-20261117/{folder}").to("cuda").eval()
for name, folder in CHECKPOINTS.items()
}
mimi = MimiModel.from_pretrained("kyutai/mimi").to("cuda").eval()
sp = spm.SentencePieceProcessor(
model_file=hf_hub_download("kyutai/moshiko-pytorch-bf16", "tokenizer_spm_32k_3.model")
)
def _load_audio(path):
"""Audio file -> list of 1 or 2 float32 channels at 24 kHz, at most MAX_INPUT_S."""
y, _ = librosa.load(path, sr=SR, mono=False, duration=MAX_INPUT_S)
if y.ndim == 1:
return [y]
if y.shape[0] >= 2 and not np.allclose(y[0], y[1], atol=1e-4):
return [y[0], y[1]]
return [y.mean(0)]
@torch.no_grad()
def _encode(wave):
x = torch.from_numpy(np.ascontiguousarray(wave, dtype=np.float32)).to("cuda")
codes = mimi.encode(x[None, None], num_quantizers=8).audio_codes # [1, 8, T]
return codes[0].T.long() # [T, 8]
@torch.no_grad()
def _decode(codes):
"""codes [T, 8] (negative = absent) -> waveform numpy."""
codes = codes.clamp(min=0)
audio = mimi.decode(codes.T[None]).audio_values # [1, 1, samples]
return audio[0, 0].float().cpu().numpy()
def _text(written, tracks):
lines = []
names = {0: "Track 1 (caller / other speaker)", 1: "Track 2 (model's own voice)"}
for track in tracks:
ids = [i for frame in written.get(track, []) for i in frame]
txt = sp.decode(ids).strip() if ids else ""
lines.append(f"**{names[track]}:** {txt or '_(nothing written)_'}")
return "\n\n".join(lines)
def _duration(audio, mode="Reply", checkpoint="Audio only", seconds=10.0, *args, **kwargs):
# Measured on ZeroGPU: ~0.027 s per 80 ms frame (both tracks) + a few seconds of overhead.
try:
clip = min(MAX_INPUT_S, librosa.get_duration(path=audio))
except Exception:
clip = MAX_INPUT_S
frames = (clip + min(float(seconds), MAX_GEN_S)) * FPS
return int(1.15 * (6 + 0.03 * frames)) + 1
@spaces.GPU(duration=_duration)
def generate(
audio: str,
mode: str = "Reply",
checkpoint: str = "Audio only",
seconds: float = 10.0,
temperature: float = 1.0,
top_k: int = 250,
seed: int = 1,
):
"""Run duplex-0 on a speech clip.
Args:
audio: Path to the input speech (wav/mp3). Mono, or stereo with one speaker per channel.
mode: "Reply" - the model listens to the clip as the caller (track 1) and speaks as
the other side (track 2) the whole time, then for `seconds` more after the clip ends.
"Continue" - the clip is a prompt and the model continues it for `seconds`.
checkpoint: "Audio only", "One text track" or "Two text tracks" (conversational stage).
seconds: Seconds to generate after the clip ends.
temperature: Audio sampling temperature.
top_k: Audio top-k.
seed: Random seed.
Returns:
Mixed audio of both tracks, the model's own track, and the text it wrote.
"""
if audio is None:
raise gr.Error("Please provide an audio clip.")
t0 = time.perf_counter()
model = MODELS[checkpoint]
seconds = float(min(max(seconds, 1.0), MAX_GEN_S))
channels = _load_audio(audio)
if len(channels[0]) < SR:
raise gr.Error("The clip must be at least 1 second long.")
new = int(round(seconds * FPS))
SAMPLE, ABSENT = streaming.SAMPLE, streaming.ABSENT
if mode == "Reply":
caller = _encode(channels[0] if len(channels) == 1 else (channels[0] + channels[1]) / 2)
silence = _encode(np.zeros(new * SAMPLES_PER_FRAME, dtype=np.float32))[:new]
caller = torch.cat((caller, silence))
frames = torch.full((len(caller), 16), SAMPLE, dtype=torch.long, device="cuda")
frames[:, :8] = caller
model_tracks = [1]
else:
prompt = [_encode(c) for c in channels]
t = min(len(p) for p in prompt)
frames = torch.full((t + new, 16), SAMPLE, dtype=torch.long, device="cuda")
if len(prompt) == 2:
frames[:t, :8] = prompt[0][:t]
frames[:t, 8:] = prompt[1][:t]
model_tracks = [0, 1]
else:
# Single speaker: the clip is track 2, the other track is empty (as in
# single-speaker pre-training data).
frames[:, :8] = ABSENT
frames[:t, 8:] = prompt[0][:t]
model_tracks = [1]
generator = torch.Generator("cuda").manual_seed(int(seed))
with torch.inference_mode():
result, written = streaming.generate(
model, frames, temperature=float(temperature), generator=generator,
top_k=int(top_k),
)
tracks = []
for j in range(2):
codes = result[:, 8 * j: 8 * j + 8]
if bool((codes < 0).all()):
tracks.append(None)
else:
tracks.append(_decode(codes))
n = max(len(w) for w in tracks if w is not None)
waves = [np.zeros(n, np.float32) if w is None else np.pad(w, (0, n - len(w))) for w in tracks]
mix = waves[0] + waves[1]
mix = mix / max(1.0, float(np.abs(mix).max()) / 0.95)
own = waves[1] / max(1.0, float(np.abs(waves[1]).max()) / 0.95)
text = _text(written, model.config.text_tracks) if model.config.text_tracks else \
"_The audio-only checkpoint writes no text — pick a text-track checkpoint to see its words._"
print(f"[duplex-0] mode={mode} ckpt={checkpoint} frames={len(frames)} "
f"time={time.perf_counter() - t0:.1f}s", flush=True)
return (SR, (mix * 32767).astype(np.int16)), (SR, (own * 32767).astype(np.int16)), text
CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks() as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"# duplex-0\n"
"A small (201M) full-duplex speech-to-speech research model from Trelis, trained from scratch on "
"~750 h of English speech over two [Mimi](https://huggingface.co/kyutai/mimi) token tracks, one per speaker. "
"**Reply**: the model hears your clip as the caller and talks as the other side, in real time. "
"**Continue**: the clip is a prompt and the model keeps the conversation going. "
"It's a research checkpoint, so the speech is only about half as intelligible as real speech. "
"[Model card](https://huggingface.co/Trelis/duplex-0)"
)
with gr.Row():
with gr.Column():
audio_in = gr.Audio(label="Input speech (≤30 s; stereo = one speaker per channel)",
type="filepath", sources=["upload", "microphone"])
mode = gr.Radio(["Reply", "Continue"], value="Reply", label="Mode")
checkpoint = gr.Radio(list(CHECKPOINTS), value="Audio only", label="Checkpoint")
seconds = gr.Slider(2, MAX_GEN_S, value=10, step=1,
label="Seconds to generate after the clip")
run = gr.Button("Generate", variant="primary")
with gr.Column():
mix_out = gr.Audio(label="Both tracks mixed", type="numpy")
own_out = gr.Audio(label="Model's track (track 2)", type="numpy")
text_out = gr.Markdown()
with gr.Accordion("Advanced settings", open=False):
temperature = gr.Slider(0.1, 1.5, value=1.0, step=0.05, label="Temperature")
top_k = gr.Slider(1, 2048, value=250, step=1, label="Top-k")
seed = gr.Number(value=1, precision=0, label="Seed")
gr.Examples(
examples=[
["examples/prompt1.wav", "Continue"],
["examples/prompt2.wav", "Reply"],
["examples/prompt3.wav", "Continue"],
["examples/prompt4.wav", "Reply"],
],
inputs=[audio_in, mode],
outputs=[mix_out, own_out, text_out],
fn=generate,
cache_examples=True,
cache_mode="lazy",
label="Held-out podcast prompts from the model card (10 s)",
)
run.click(generate, [audio_in, mode, checkpoint, seconds, temperature, top_k, seed],
[mix_out, own_out, text_out])
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)