Spaces:
Running on Zero
Running on Zero
Upload folder using huggingface_hub
Browse files- .gitattributes +4 -0
- README.md +15 -6
- app.py +222 -0
- duplex/NOTICE +32 -0
- duplex/__init__.py +0 -0
- duplex/checkpoint.py +78 -0
- duplex/kernels.py +177 -0
- duplex/network.py +638 -0
- duplex/streaming.py +542 -0
- examples/prompt1.wav +3 -0
- examples/prompt2.wav +3 -0
- examples/prompt3.wav +3 -0
- examples/prompt4.wav +3 -0
- requirements.txt +7 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
examples/prompt1.wav filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
examples/prompt2.wav filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
examples/prompt3.wav filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
examples/prompt4.wav filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,13 +1,22 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.32.0
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: duplex-0
|
| 3 |
+
emoji: 🎙️
|
| 4 |
+
colorFrom: blue
|
| 5 |
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.32.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
short_description: Small full-duplex speech-to-speech model (Mimi tokens)
|
| 10 |
+
python_version: "3.12"
|
| 11 |
+
startup_duration_timeout: 30m
|
| 12 |
+
models:
|
| 13 |
+
- Trelis/duplex-0
|
| 14 |
+
- kyutai/mimi
|
| 15 |
+
- kyutai/moshiko-pytorch-bf16
|
| 16 |
---
|
| 17 |
|
| 18 |
+
Demo of [Trelis/duplex-0](https://huggingface.co/Trelis/duplex-0), a 201M-parameter two-track speech-to-speech research model.
|
| 19 |
+
|
| 20 |
+
The model code in `duplex/` is copied from the model repo's `scripts/duplex/model` (Apache-2.0, see `duplex/NOTICE`).
|
| 21 |
+
Example prompts are the first 10 seconds of the "real" sample clips in the model repo's `report/samples/` (Apache-2.0 repository).
|
| 22 |
+
Audio is encoded/decoded with Kyutai's [Mimi](https://huggingface.co/kyutai/mimi) codec, and text uses Moshi's tokenizer.
|
app.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import spaces # must come before torch
|
| 2 |
+
|
| 3 |
+
import time
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
import librosa
|
| 7 |
+
import numpy as np
|
| 8 |
+
import sentencepiece as spm
|
| 9 |
+
import torch
|
| 10 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 11 |
+
from transformers import MimiModel
|
| 12 |
+
|
| 13 |
+
from duplex import streaming
|
| 14 |
+
from duplex.checkpoint import load_published
|
| 15 |
+
|
| 16 |
+
REPO = "Trelis/duplex-0"
|
| 17 |
+
SR = 24000
|
| 18 |
+
FPS = 12.5
|
| 19 |
+
SAMPLES_PER_FRAME = 1920
|
| 20 |
+
MAX_INPUT_S = 30.0
|
| 21 |
+
MAX_GEN_S = 30.0
|
| 22 |
+
|
| 23 |
+
CHECKPOINTS = {
|
| 24 |
+
"Audio only": "conversational-none",
|
| 25 |
+
"One text track": "conversational-one",
|
| 26 |
+
"Two text tracks": "conversational-two",
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
root = snapshot_download(
|
| 30 |
+
REPO,
|
| 31 |
+
allow_patterns=[f"resplit-20261117/{c}/{f}" for c in CHECKPOINTS.values()
|
| 32 |
+
for f in ("config.json", "model.safetensors")],
|
| 33 |
+
)
|
| 34 |
+
MODELS = {
|
| 35 |
+
name: load_published(f"{root}/resplit-20261117/{folder}").to("cuda").eval()
|
| 36 |
+
for name, folder in CHECKPOINTS.items()
|
| 37 |
+
}
|
| 38 |
+
mimi = MimiModel.from_pretrained("kyutai/mimi").to("cuda").eval()
|
| 39 |
+
sp = spm.SentencePieceProcessor(
|
| 40 |
+
model_file=hf_hub_download("kyutai/moshiko-pytorch-bf16", "tokenizer_spm_32k_3.model")
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _load_audio(path):
|
| 45 |
+
"""Audio file -> list of 1 or 2 float32 channels at 24 kHz, at most MAX_INPUT_S."""
|
| 46 |
+
y, _ = librosa.load(path, sr=SR, mono=False, duration=MAX_INPUT_S)
|
| 47 |
+
if y.ndim == 1:
|
| 48 |
+
return [y]
|
| 49 |
+
if y.shape[0] >= 2 and not np.allclose(y[0], y[1], atol=1e-4):
|
| 50 |
+
return [y[0], y[1]]
|
| 51 |
+
return [y.mean(0)]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@torch.no_grad()
|
| 55 |
+
def _encode(wave):
|
| 56 |
+
x = torch.from_numpy(np.ascontiguousarray(wave, dtype=np.float32)).to("cuda")
|
| 57 |
+
codes = mimi.encode(x[None, None], num_quantizers=8).audio_codes # [1, 8, T]
|
| 58 |
+
return codes[0].T.long() # [T, 8]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@torch.no_grad()
|
| 62 |
+
def _decode(codes):
|
| 63 |
+
"""codes [T, 8] (negative = absent) -> waveform numpy."""
|
| 64 |
+
codes = codes.clamp(min=0)
|
| 65 |
+
audio = mimi.decode(codes.T[None]).audio_values # [1, 1, samples]
|
| 66 |
+
return audio[0, 0].float().cpu().numpy()
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _text(written, tracks):
|
| 70 |
+
lines = []
|
| 71 |
+
names = {0: "Track 1 (caller / other speaker)", 1: "Track 2 (model's own voice)"}
|
| 72 |
+
for track in tracks:
|
| 73 |
+
ids = [i for frame in written.get(track, []) for i in frame]
|
| 74 |
+
txt = sp.decode(ids).strip() if ids else ""
|
| 75 |
+
lines.append(f"**{names[track]}:** {txt or '_(nothing written)_'}")
|
| 76 |
+
return "\n\n".join(lines)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _duration(audio, mode="Reply", checkpoint="Audio only", seconds=10.0, *args, **kwargs):
|
| 80 |
+
total = MAX_INPUT_S + float(seconds)
|
| 81 |
+
per_s = 1.2 if checkpoint == "Audio only" else 2.0
|
| 82 |
+
return int(min(180, 20 + total * per_s))
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@spaces.GPU(duration=_duration)
|
| 86 |
+
def generate(
|
| 87 |
+
audio: str,
|
| 88 |
+
mode: str = "Reply",
|
| 89 |
+
checkpoint: str = "Audio only",
|
| 90 |
+
seconds: float = 10.0,
|
| 91 |
+
temperature: float = 1.0,
|
| 92 |
+
top_k: int = 250,
|
| 93 |
+
seed: int = 1,
|
| 94 |
+
):
|
| 95 |
+
"""Run duplex-0 on a speech clip.
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
audio: Path to the input speech (wav/mp3). Mono, or stereo with one speaker per channel.
|
| 99 |
+
mode: "Reply" - the model listens to the clip as the caller (track 1) and speaks as
|
| 100 |
+
the other side (track 2) the whole time, then for `seconds` more after the clip ends.
|
| 101 |
+
"Continue" - the clip is a prompt and the model continues it for `seconds`.
|
| 102 |
+
checkpoint: "Audio only", "One text track" or "Two text tracks" (conversational stage).
|
| 103 |
+
seconds: Seconds to generate after the clip ends.
|
| 104 |
+
temperature: Audio sampling temperature.
|
| 105 |
+
top_k: Audio top-k.
|
| 106 |
+
seed: Random seed.
|
| 107 |
+
|
| 108 |
+
Returns:
|
| 109 |
+
Mixed audio of both tracks, the model's own track, and the text it wrote.
|
| 110 |
+
"""
|
| 111 |
+
if audio is None:
|
| 112 |
+
raise gr.Error("Please provide an audio clip.")
|
| 113 |
+
t0 = time.perf_counter()
|
| 114 |
+
model = MODELS[checkpoint]
|
| 115 |
+
seconds = float(min(max(seconds, 1.0), MAX_GEN_S))
|
| 116 |
+
channels = _load_audio(audio)
|
| 117 |
+
if len(channels[0]) < SR:
|
| 118 |
+
raise gr.Error("The clip must be at least 1 second long.")
|
| 119 |
+
new = int(round(seconds * FPS))
|
| 120 |
+
SAMPLE, ABSENT = streaming.SAMPLE, streaming.ABSENT
|
| 121 |
+
|
| 122 |
+
if mode == "Reply":
|
| 123 |
+
caller = _encode(channels[0] if len(channels) == 1 else (channels[0] + channels[1]) / 2)
|
| 124 |
+
silence = _encode(np.zeros(new * SAMPLES_PER_FRAME, dtype=np.float32))[:new]
|
| 125 |
+
caller = torch.cat((caller, silence))
|
| 126 |
+
frames = torch.full((len(caller), 16), SAMPLE, dtype=torch.long, device="cuda")
|
| 127 |
+
frames[:, :8] = caller
|
| 128 |
+
model_tracks = [1]
|
| 129 |
+
else:
|
| 130 |
+
prompt = [_encode(c) for c in channels]
|
| 131 |
+
t = min(len(p) for p in prompt)
|
| 132 |
+
frames = torch.full((t + new, 16), SAMPLE, dtype=torch.long, device="cuda")
|
| 133 |
+
if len(prompt) == 2:
|
| 134 |
+
frames[:t, :8] = prompt[0][:t]
|
| 135 |
+
frames[:t, 8:] = prompt[1][:t]
|
| 136 |
+
model_tracks = [0, 1]
|
| 137 |
+
else:
|
| 138 |
+
# Single speaker: the clip is track 2, the other track is empty (as in
|
| 139 |
+
# single-speaker pre-training data).
|
| 140 |
+
frames[:, :8] = ABSENT
|
| 141 |
+
frames[:t, 8:] = prompt[0][:t]
|
| 142 |
+
model_tracks = [1]
|
| 143 |
+
|
| 144 |
+
generator = torch.Generator("cuda").manual_seed(int(seed))
|
| 145 |
+
with torch.inference_mode():
|
| 146 |
+
result, written = streaming.generate(
|
| 147 |
+
model, frames, temperature=float(temperature), generator=generator,
|
| 148 |
+
top_k=int(top_k),
|
| 149 |
+
)
|
| 150 |
+
tracks = []
|
| 151 |
+
for j in range(2):
|
| 152 |
+
codes = result[:, 8 * j: 8 * j + 8]
|
| 153 |
+
if bool((codes < 0).all()):
|
| 154 |
+
tracks.append(None)
|
| 155 |
+
else:
|
| 156 |
+
tracks.append(_decode(codes))
|
| 157 |
+
n = max(len(w) for w in tracks if w is not None)
|
| 158 |
+
waves = [np.zeros(n, np.float32) if w is None else np.pad(w, (0, n - len(w))) for w in tracks]
|
| 159 |
+
mix = waves[0] + waves[1]
|
| 160 |
+
mix = mix / max(1.0, float(np.abs(mix).max()) / 0.95)
|
| 161 |
+
own = waves[1] / max(1.0, float(np.abs(waves[1]).max()) / 0.95)
|
| 162 |
+
text = _text(written, model.config.text_tracks) if model.config.text_tracks else \
|
| 163 |
+
"_The audio-only checkpoint writes no text — pick a text-track checkpoint to see its words._"
|
| 164 |
+
print(f"[duplex-0] mode={mode} ckpt={checkpoint} frames={len(frames)} "
|
| 165 |
+
f"time={time.perf_counter() - t0:.1f}s", flush=True)
|
| 166 |
+
return (SR, (mix * 32767).astype(np.int16)), (SR, (own * 32767).astype(np.int16)), text
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
CSS = """
|
| 170 |
+
#col-container { max-width: 1100px; margin: 0 auto; }
|
| 171 |
+
.dark .gradio-container { color: var(--body-text-color); }
|
| 172 |
+
"""
|
| 173 |
+
|
| 174 |
+
with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
|
| 175 |
+
with gr.Column(elem_id="col-container"):
|
| 176 |
+
gr.Markdown(
|
| 177 |
+
"# duplex-0\n"
|
| 178 |
+
"A small (201M) full-duplex speech-to-speech research model from Trelis, trained from scratch on "
|
| 179 |
+
"~750 h of English speech over two [Mimi](https://huggingface.co/kyutai/mimi) token tracks, one per speaker. "
|
| 180 |
+
"**Reply**: the model hears your clip as the caller and talks as the other side, in real time. "
|
| 181 |
+
"**Continue**: the clip is a prompt and the model keeps the conversation going. "
|
| 182 |
+
"It's a research checkpoint, so the speech is only about half as intelligible as real speech. "
|
| 183 |
+
"[Model card](https://huggingface.co/Trelis/duplex-0)"
|
| 184 |
+
)
|
| 185 |
+
with gr.Row():
|
| 186 |
+
with gr.Column():
|
| 187 |
+
audio_in = gr.Audio(label="Input speech (≤30 s; stereo = one speaker per channel)",
|
| 188 |
+
type="filepath", sources=["upload", "microphone"])
|
| 189 |
+
mode = gr.Radio(["Reply", "Continue"], value="Reply", label="Mode")
|
| 190 |
+
checkpoint = gr.Radio(list(CHECKPOINTS), value="Audio only", label="Checkpoint")
|
| 191 |
+
seconds = gr.Slider(2, MAX_GEN_S, value=10, step=1,
|
| 192 |
+
label="Seconds to generate after the clip")
|
| 193 |
+
run = gr.Button("Generate", variant="primary")
|
| 194 |
+
with gr.Column():
|
| 195 |
+
mix_out = gr.Audio(label="Both tracks mixed", type="numpy")
|
| 196 |
+
own_out = gr.Audio(label="Model's track (track 2)", type="numpy")
|
| 197 |
+
text_out = gr.Markdown()
|
| 198 |
+
with gr.Accordion("Advanced settings", open=False):
|
| 199 |
+
temperature = gr.Slider(0.1, 1.5, value=1.0, step=0.05, label="Temperature")
|
| 200 |
+
top_k = gr.Slider(1, 2048, value=250, step=1, label="Top-k")
|
| 201 |
+
seed = gr.Number(value=1, precision=0, label="Seed")
|
| 202 |
+
|
| 203 |
+
gr.Examples(
|
| 204 |
+
examples=[
|
| 205 |
+
["examples/prompt1.wav", "Continue"],
|
| 206 |
+
["examples/prompt2.wav", "Reply"],
|
| 207 |
+
["examples/prompt3.wav", "Continue"],
|
| 208 |
+
["examples/prompt4.wav", "Reply"],
|
| 209 |
+
],
|
| 210 |
+
inputs=[audio_in, mode],
|
| 211 |
+
outputs=[mix_out, own_out, text_out],
|
| 212 |
+
fn=generate,
|
| 213 |
+
cache_examples=True,
|
| 214 |
+
cache_mode="lazy",
|
| 215 |
+
label="Held-out podcast prompts from the model card (10 s)",
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
run.click(generate, [audio_in, mode, checkpoint, seconds, temperature, top_k, seed],
|
| 219 |
+
[mix_out, own_out, text_out])
|
| 220 |
+
|
| 221 |
+
if __name__ == "__main__":
|
| 222 |
+
demo.launch(mcp_server=True)
|
duplex/NOTICE
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The attention/MLP implementation follows architecture and tensor conventions
|
| 2 |
+
from karpathy/nanochat, inspected at 92d63d4e8bb4df75c3b71618f31ddde2378b2bcd.
|
| 3 |
+
Upstream license follows:
|
| 4 |
+
|
| 5 |
+
MIT License
|
| 6 |
+
|
| 7 |
+
Copyright (c) 2025 Andrej Karpathy
|
| 8 |
+
|
| 9 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 10 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 11 |
+
in the Software without restriction, including without limitation the rights
|
| 12 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 13 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 14 |
+
furnished to do so, subject to the following conditions:
|
| 15 |
+
|
| 16 |
+
The above copyright notice and this permission notice shall be included in all
|
| 17 |
+
copies or substantial portions of the Software.
|
| 18 |
+
|
| 19 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 20 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 21 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 22 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 23 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 24 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 25 |
+
SOFTWARE.
|
| 26 |
+
|
| 27 |
+
kernels.py adapts the depth attention kernel design (two frames per 16-row
|
| 28 |
+
tile, block-diagonal causal mask, recomputed backward) from naklecha/caramel
|
| 29 |
+
kernels.py, inspected at 84a9271d265841543287d4044d7ef80991a2b003, licensed
|
| 30 |
+
under the Apache License, Version 2.0
|
| 31 |
+
(https://www.apache.org/licenses/LICENSE-2.0). It was rewritten for separate
|
| 32 |
+
query, key and value tensors in [rows, positions, heads, head_width] layout.
|
duplex/__init__.py
ADDED
|
File without changes
|
duplex/checkpoint.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Versioned research checkpoints, exact single-device RNG/optimizer resume."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from dataclasses import asdict
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from .network import Config, DuplexModel, factorization, from_legacy
|
| 10 |
+
|
| 11 |
+
# The one- and two-stream format of 14 November, before text tracks.
|
| 12 |
+
LEGACY_FACTORIZATION = "delay-0-1-1-1-1-1-1-1"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def save(path, model, optimizer, step, provenance, sampler):
|
| 16 |
+
path = Path(path)
|
| 17 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 18 |
+
state = {
|
| 19 |
+
"schema_version": 3,
|
| 20 |
+
"factorization": factorization(model.config),
|
| 21 |
+
"config": asdict(model.config),
|
| 22 |
+
"model": model.state_dict(),
|
| 23 |
+
"optimizer": optimizer.state_dict(),
|
| 24 |
+
"step": step,
|
| 25 |
+
"provenance": provenance,
|
| 26 |
+
"torch_rng": torch.get_rng_state(),
|
| 27 |
+
"sampler_rng": sampler.get_state(),
|
| 28 |
+
"cuda_rng": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [],
|
| 29 |
+
}
|
| 30 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 31 |
+
torch.save(state, temporary)
|
| 32 |
+
temporary.replace(path)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def model_from(config, weights, stored):
|
| 36 |
+
"""A model from a stored config and weights; 14 November checkpoints convert."""
|
| 37 |
+
if stored == LEGACY_FACTORIZATION:
|
| 38 |
+
return from_legacy(config, weights)
|
| 39 |
+
# Checkpoints before 17 November 2026 have no shift or head fields and load
|
| 40 |
+
# with the defaults they were trained under.
|
| 41 |
+
config = Config(**config)
|
| 42 |
+
if stored != factorization(config):
|
| 43 |
+
raise ValueError("Unsupported codebook factorization")
|
| 44 |
+
model = DuplexModel(config)
|
| 45 |
+
model.load_state_dict(weights, strict=True)
|
| 46 |
+
return model
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def load(path, device="cpu"):
|
| 50 |
+
# weights_only rejects arbitrary pickled classes; only tensors/basic containers are used.
|
| 51 |
+
state = torch.load(path, map_location="cpu", weights_only=True)
|
| 52 |
+
if state["schema_version"] not in (2, 3):
|
| 53 |
+
raise ValueError("Unsupported checkpoint schema")
|
| 54 |
+
model = model_from(state["config"], state["model"], state["factorization"])
|
| 55 |
+
return model.to(device), state
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def load_published(folder, device="cpu"):
|
| 59 |
+
"""A checkpoint folder written by `publish.py` (config.json and model.safetensors)."""
|
| 60 |
+
from safetensors.torch import load_file
|
| 61 |
+
|
| 62 |
+
folder = Path(folder)
|
| 63 |
+
config = json.loads((folder / "config.json").read_text())
|
| 64 |
+
stored = config.pop("factorization")
|
| 65 |
+
weights = load_file(folder / "model.safetensors")
|
| 66 |
+
return model_from(config, weights, stored).to(device).eval()
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def resume(state, optimizer, sampler):
|
| 70 |
+
if state["schema_version"] != 3:
|
| 71 |
+
raise ValueError("Resume needs a checkpoint of this trainer")
|
| 72 |
+
optimizer.load_state_dict(state["optimizer"])
|
| 73 |
+
torch.set_rng_state(state["torch_rng"])
|
| 74 |
+
sampler.set_state(state["sampler_rng"])
|
| 75 |
+
if state["cuda_rng"]:
|
| 76 |
+
if not torch.cuda.is_available():
|
| 77 |
+
raise ValueError("Exact CUDA resume requires CUDA")
|
| 78 |
+
torch.cuda.set_rng_state_all(state["cuda_rng"])
|
duplex/kernels.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Depth attention over the codebooks of one frame, specialised for short rows.
|
| 2 |
+
|
| 3 |
+
General attention kernels spend most of their tiles on padding when the sequence
|
| 4 |
+
is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one
|
| 5 |
+
head into a 16-row tile (two frames of one stream, or one frame of two streams)
|
| 6 |
+
with a block-diagonal causal mask, the approach of Caramel
|
| 7 |
+
(github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch
|
| 8 |
+
reference, which defines the math the kernel must match.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import math
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch.nn import functional as F
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
import triton
|
| 18 |
+
import triton.language as tl
|
| 19 |
+
except ImportError:
|
| 20 |
+
triton = None
|
| 21 |
+
|
| 22 |
+
ROWS_PER_TILE = 16
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def reference(q, k, v):
|
| 26 |
+
"""Causal attention for [rows, positions, heads, head_width] inputs."""
|
| 27 |
+
q, k, v = (y.transpose(1, 2) for y in (q, k, v))
|
| 28 |
+
return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
if triton is not None:
|
| 32 |
+
|
| 33 |
+
@triton.jit
|
| 34 |
+
def _probabilities(
|
| 35 |
+
Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
|
| 36 |
+
):
|
| 37 |
+
rows: tl.constexpr = FPT * POS
|
| 38 |
+
pid = tl.program_id(0).to(tl.int64)
|
| 39 |
+
r, d = tl.arange(0, rows), tl.arange(0, HD)
|
| 40 |
+
n, position = (pid // H) * FPT + r // POS, r % POS
|
| 41 |
+
valid = (n < N)[:, None]
|
| 42 |
+
offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :]
|
| 43 |
+
q = tl.load(Q + offsets, mask=valid, other=0.0)
|
| 44 |
+
k = tl.load(K + offsets, mask=valid, other=0.0)
|
| 45 |
+
s = tl.dot(q, tl.trans(k)) * scale
|
| 46 |
+
same_frame = (r[:, None] // POS) == (r[None, :] // POS)
|
| 47 |
+
causal = position[:, None] >= position[None, :]
|
| 48 |
+
s = tl.where(same_frame & causal, s, float("-inf"))
|
| 49 |
+
p = tl.exp(s - tl.max(s, axis=1)[:, None])
|
| 50 |
+
return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid
|
| 51 |
+
|
| 52 |
+
@triton.jit
|
| 53 |
+
def _forward(
|
| 54 |
+
Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
|
| 55 |
+
):
|
| 56 |
+
_, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
|
| 57 |
+
v = tl.load(V + offsets, mask=valid, other=0.0)
|
| 58 |
+
o = tl.dot(p, v.to(tl.float32), input_precision="ieee")
|
| 59 |
+
tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid)
|
| 60 |
+
|
| 61 |
+
@triton.jit
|
| 62 |
+
def _backward(
|
| 63 |
+
Q,
|
| 64 |
+
K,
|
| 65 |
+
V,
|
| 66 |
+
DO,
|
| 67 |
+
DQ,
|
| 68 |
+
DK,
|
| 69 |
+
DV,
|
| 70 |
+
N,
|
| 71 |
+
H,
|
| 72 |
+
scale,
|
| 73 |
+
HD: tl.constexpr,
|
| 74 |
+
POS: tl.constexpr,
|
| 75 |
+
FPT: tl.constexpr,
|
| 76 |
+
):
|
| 77 |
+
# Recomputes the small probability tile, so forward saves only its inputs.
|
| 78 |
+
q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
|
| 79 |
+
v = tl.load(V + offsets, mask=valid, other=0.0)
|
| 80 |
+
do = tl.load(DO + offsets, mask=valid, other=0.0)
|
| 81 |
+
dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee")
|
| 82 |
+
dp = tl.dot(do, tl.trans(v))
|
| 83 |
+
ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale
|
| 84 |
+
dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee")
|
| 85 |
+
dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee")
|
| 86 |
+
ty = DQ.dtype.element_ty
|
| 87 |
+
tl.store(DQ + offsets, dq.to(ty), mask=valid)
|
| 88 |
+
tl.store(DK + offsets, dk.to(ty), mask=valid)
|
| 89 |
+
tl.store(DV + offsets, dv.to(ty), mask=valid)
|
| 90 |
+
|
| 91 |
+
def _launch(kernel, q, *tensors):
|
| 92 |
+
n, positions, heads, head_width = q.shape
|
| 93 |
+
frames = ROWS_PER_TILE // positions
|
| 94 |
+
kernel[(triton.cdiv(n, frames) * heads,)](
|
| 95 |
+
q,
|
| 96 |
+
*tensors,
|
| 97 |
+
n,
|
| 98 |
+
heads,
|
| 99 |
+
1.0 / math.sqrt(head_width),
|
| 100 |
+
HD=head_width,
|
| 101 |
+
POS=positions,
|
| 102 |
+
FPT=frames,
|
| 103 |
+
num_warps=1,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
# Custom ops make each kernel one opaque node for torch.compile.
|
| 107 |
+
@torch.library.custom_op("duplex::depth_attention", mutates_args=())
|
| 108 |
+
def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
| 109 |
+
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
|
| 110 |
+
o = torch.empty_like(q)
|
| 111 |
+
_launch(_forward, q, k, v, o)
|
| 112 |
+
return o
|
| 113 |
+
|
| 114 |
+
@torch.library.custom_op("duplex::depth_attention_backward", mutates_args=())
|
| 115 |
+
def _attention_backward(
|
| 116 |
+
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor
|
| 117 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 118 |
+
q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous()
|
| 119 |
+
dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q)
|
| 120 |
+
_launch(_backward, q, k, v, do, dq, dk, dv)
|
| 121 |
+
return dq, dk, dv
|
| 122 |
+
|
| 123 |
+
_attention.register_fake(lambda q, k, v: torch.empty_like(q))
|
| 124 |
+
_attention_backward.register_fake(
|
| 125 |
+
lambda q, k, v, do: (
|
| 126 |
+
torch.empty_like(q),
|
| 127 |
+
torch.empty_like(q),
|
| 128 |
+
torch.empty_like(q),
|
| 129 |
+
)
|
| 130 |
+
)
|
| 131 |
+
_attention.register_autograd(
|
| 132 |
+
lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad),
|
| 133 |
+
setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs),
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def depth_attention(q, k, v):
|
| 138 |
+
"""q, k and v are [rows, positions, heads, head_width]; so is the result."""
|
| 139 |
+
if (
|
| 140 |
+
triton is None
|
| 141 |
+
or not q.is_cuda
|
| 142 |
+
or q.dtype == torch.float32
|
| 143 |
+
or ROWS_PER_TILE % q.shape[1]
|
| 144 |
+
):
|
| 145 |
+
return reference(q, k, v)
|
| 146 |
+
return _attention(q, k, v)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
if __name__ == "__main__":
|
| 150 |
+
# GPU check: the packed kernel against the reference for one and two streams.
|
| 151 |
+
for positions in (8, 16):
|
| 152 |
+
q, k, v = (
|
| 153 |
+
torch.randn(
|
| 154 |
+
4096,
|
| 155 |
+
positions,
|
| 156 |
+
4,
|
| 157 |
+
64,
|
| 158 |
+
device="cuda",
|
| 159 |
+
dtype=torch.bfloat16,
|
| 160 |
+
requires_grad=True,
|
| 161 |
+
)
|
| 162 |
+
for _ in range(3)
|
| 163 |
+
)
|
| 164 |
+
grad = torch.randn_like(q)
|
| 165 |
+
results = []
|
| 166 |
+
for function in (depth_attention, reference):
|
| 167 |
+
out = function(q, k, v)
|
| 168 |
+
out.backward(grad)
|
| 169 |
+
results.append([out.float(), *(x.grad.float() for x in (q, k, v))])
|
| 170 |
+
for x in (q, k, v):
|
| 171 |
+
x.grad = None
|
| 172 |
+
errors = [
|
| 173 |
+
((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True)
|
| 174 |
+
]
|
| 175 |
+
print({"positions": positions, "relative_errors": errors}, flush=True)
|
| 176 |
+
if max(errors) > 0.01:
|
| 177 |
+
raise SystemExit("kernel differs from reference")
|
duplex/network.py
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Temporal/depth autoregression over two audio tracks and two text tracks.
|
| 2 |
+
|
| 3 |
+
A frame holds `codebooks` audio codes for each of the two tracks, track 1 then
|
| 4 |
+
track 2. In every track the acoustic codebooks run one frame behind codebook 0
|
| 5 |
+
inside the model. Callers pass and receive canonical undelayed frames.
|
| 6 |
+
|
| 7 |
+
Text runs at a variable rate (training readiness owns the design). Sequence step s
|
| 8 |
+
is one temporal position for the audio, followed by the text tokens of frame s of
|
| 9 |
+
each present text track, track 1 first. A track's text ends with the advance token,
|
| 10 |
+
which is only ever a target: each text event is predicted from the last input before
|
| 11 |
+
it, and the step's last position conditions the depth transformer.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from dataclasses import dataclass, replace
|
| 15 |
+
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
from torch import nn
|
| 19 |
+
from torch.nn import functional as F
|
| 20 |
+
|
| 21 |
+
from .kernels import depth_attention
|
| 22 |
+
|
| 23 |
+
TRACKS = 2
|
| 24 |
+
# Moshi's SentencePiece ids are 0..31999; the advance token follows them.
|
| 25 |
+
TEXT_IDS = 32000
|
| 26 |
+
ADVANCE = TEXT_IDS
|
| 27 |
+
# Sampling forces the advance after this many tokens of one track in one frame.
|
| 28 |
+
MAX_TEXT_PER_FRAME = 16
|
| 29 |
+
# Moshi's equation 7: codebook 0 of each track against 1 for each acoustic codebook,
|
| 30 |
+
# and its padding weight for the advance token.
|
| 31 |
+
SEMANTIC_WEIGHT = 100.0
|
| 32 |
+
ADVANCE_WEIGHT = 0.5
|
| 33 |
+
TEXT_OPTIONS = {"none": (), "one": (1,), "two": (0, 1)}
|
| 34 |
+
# Text against audio per track, in 80 ms frames: a word starting at frame f is
|
| 35 |
+
# written on track p at frame f - TEXT_SHIFT[p]. Positive puts the text ahead of the
|
| 36 |
+
# audio, negative behind it, 0 at the word's start. Track 1 (listened to) is 480 ms
|
| 37 |
+
# behind, a transcript; track 2 (spoken as) writes words where they start. The shift
|
| 38 |
+
# follows the track after the speaker swap; windows apply it, generation follows it.
|
| 39 |
+
# The default of `Config.text_shift`, which each checkpoint records.
|
| 40 |
+
TEXT_SHIFT = (-6, 0)
|
| 41 |
+
# Text output heads: one shared by both tracks, or one per track.
|
| 42 |
+
TEXT_HEADS = ("shared", "per-track")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class Config:
|
| 47 |
+
layers: int = 12
|
| 48 |
+
width: int = 768
|
| 49 |
+
heads: int = 6
|
| 50 |
+
depth_layers: int = 2
|
| 51 |
+
depth_width: int = 256
|
| 52 |
+
depth_heads: int = 4
|
| 53 |
+
# Temporal positions per window: frames plus text tokens, plus one flush step.
|
| 54 |
+
context: int = 4096
|
| 55 |
+
codebooks: int = 8
|
| 56 |
+
cardinality: int = 2048
|
| 57 |
+
fps: float = 12.5
|
| 58 |
+
# The training option; the parameters are the same for all three.
|
| 59 |
+
text: str = "none"
|
| 60 |
+
# Frames each track's text sits from its word start (see TEXT_SHIFT).
|
| 61 |
+
text_shift: tuple = TEXT_SHIFT
|
| 62 |
+
text_heads: str = "shared"
|
| 63 |
+
|
| 64 |
+
@property
|
| 65 |
+
def total_codebooks(self):
|
| 66 |
+
return self.codebooks * TRACKS
|
| 67 |
+
|
| 68 |
+
@property
|
| 69 |
+
def text_tracks(self):
|
| 70 |
+
return TEXT_OPTIONS[self.text]
|
| 71 |
+
|
| 72 |
+
def __post_init__(self):
|
| 73 |
+
for width, heads in (
|
| 74 |
+
(self.width, self.heads),
|
| 75 |
+
(self.depth_width, self.depth_heads),
|
| 76 |
+
):
|
| 77 |
+
if heads < 1 or width % heads or (width // heads) % 2:
|
| 78 |
+
raise ValueError("Head dimensions must be even")
|
| 79 |
+
if (
|
| 80 |
+
min(
|
| 81 |
+
self.layers,
|
| 82 |
+
self.depth_layers,
|
| 83 |
+
self.context,
|
| 84 |
+
self.codebooks,
|
| 85 |
+
self.cardinality,
|
| 86 |
+
)
|
| 87 |
+
< 1
|
| 88 |
+
):
|
| 89 |
+
raise ValueError("Dimensions must be positive")
|
| 90 |
+
if self.fps <= 0:
|
| 91 |
+
raise ValueError("fps must be positive")
|
| 92 |
+
if self.text not in TEXT_OPTIONS:
|
| 93 |
+
raise ValueError(f"text must be one of {sorted(TEXT_OPTIONS)}")
|
| 94 |
+
# A config read from JSON holds a list.
|
| 95 |
+
object.__setattr__(self, "text_shift", tuple(int(x) for x in self.text_shift))
|
| 96 |
+
if len(self.text_shift) != TRACKS:
|
| 97 |
+
raise ValueError("text_shift has one value per track")
|
| 98 |
+
if self.text_heads not in TEXT_HEADS:
|
| 99 |
+
raise ValueError(f"text_heads must be one of {TEXT_HEADS}")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# Sequence order of the codebooks and text. Checkpoints carry it; a model trained
|
| 103 |
+
# under one factorization is meaningless under another.
|
| 104 |
+
def factorization(config):
|
| 105 |
+
return "two-tracks-delay-0-1-1-1-1-1-1-1-text-advance-shift" + "".join(
|
| 106 |
+
f"{shift:+d}" for shift in config.text_shift
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
FACTORIZATION = factorization(Config())
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def tiny_config(text="none"):
|
| 114 |
+
# Real codec vocabulary, small transformers. No fake-codec checkpoint confusion.
|
| 115 |
+
return Config(
|
| 116 |
+
layers=2,
|
| 117 |
+
width=32,
|
| 118 |
+
heads=2,
|
| 119 |
+
depth_layers=1,
|
| 120 |
+
depth_width=32,
|
| 121 |
+
depth_heads=2,
|
| 122 |
+
context=64,
|
| 123 |
+
text=text,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def norm(x):
|
| 128 |
+
return F.rms_norm(x, (x.shape[-1],))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def rotary(positions, head_width):
|
| 132 |
+
"""Cos and sin tables [positions, head_width / 2], built once per model."""
|
| 133 |
+
freq = torch.arange(0, head_width, 2, dtype=torch.float32) / head_width
|
| 134 |
+
angles = torch.outer(torch.arange(positions), 100000.0**-freq)
|
| 135 |
+
return torch.stack((angles.cos(), angles.sin()))
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def rotate(y, cos, sin):
|
| 139 |
+
a, b = y.chunk(2, dim=-1)
|
| 140 |
+
return torch.cat((a * cos + b * sin, -a * sin + b * cos), dim=-1)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
class Attention(nn.Module):
|
| 144 |
+
def __init__(self, width, heads):
|
| 145 |
+
super().__init__()
|
| 146 |
+
self.heads = heads
|
| 147 |
+
self.c_q = nn.Linear(width, width, bias=False)
|
| 148 |
+
self.c_k = nn.Linear(width, width, bias=False)
|
| 149 |
+
self.c_v = nn.Linear(width, width, bias=False)
|
| 150 |
+
self.c_proj = nn.Linear(width, width, bias=False)
|
| 151 |
+
|
| 152 |
+
def forward(self, x, rope):
|
| 153 |
+
b, t, w = x.shape
|
| 154 |
+
d = w // self.heads
|
| 155 |
+
q, k, v = [
|
| 156 |
+
layer(x).reshape(b, t, self.heads, d).transpose(1, 2)
|
| 157 |
+
for layer in (self.c_q, self.c_k, self.c_v)
|
| 158 |
+
]
|
| 159 |
+
cos, sin = rope[0, :t].to(x.dtype), rope[1, :t].to(x.dtype)
|
| 160 |
+
q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
|
| 161 |
+
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
|
| 162 |
+
return self.c_proj(y.transpose(1, 2).reshape(x.shape))
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class MLP(nn.Module):
|
| 166 |
+
def __init__(self, width):
|
| 167 |
+
super().__init__()
|
| 168 |
+
self.c_fc = nn.Linear(width, 4 * width, bias=False)
|
| 169 |
+
self.c_proj = nn.Linear(4 * width, width, bias=False)
|
| 170 |
+
|
| 171 |
+
def forward(self, x):
|
| 172 |
+
return self.c_proj(F.relu(self.c_fc(x)).square())
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class Block(nn.Module):
|
| 176 |
+
def __init__(self, width, heads):
|
| 177 |
+
super().__init__()
|
| 178 |
+
self.attn = Attention(width, heads)
|
| 179 |
+
self.mlp = MLP(width)
|
| 180 |
+
|
| 181 |
+
def forward(self, x, rope):
|
| 182 |
+
x = x + self.attn(norm(x), rope)
|
| 183 |
+
return x + self.mlp(norm(x))
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def per_position(x, weight):
|
| 187 |
+
"""x [rows, positions, in] through weight [positions, out, in]: one matrix per position."""
|
| 188 |
+
y = torch.bmm(x.transpose(0, 1), weight.transpose(1, 2).to(x.dtype))
|
| 189 |
+
return y.transpose(0, 1)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class DepthBlock(nn.Module):
|
| 193 |
+
"""A depth block with its own weights at every codebook position.
|
| 194 |
+
|
| 195 |
+
Same widths and arithmetic as `Block`; each [positions, out, in] matrix holds
|
| 196 |
+
one nn.Linear weight per position, initialised as nn.Linear initialises.
|
| 197 |
+
"""
|
| 198 |
+
|
| 199 |
+
def __init__(self, positions, width, heads):
|
| 200 |
+
super().__init__()
|
| 201 |
+
self.heads = heads
|
| 202 |
+
|
| 203 |
+
def weight(out, inner):
|
| 204 |
+
bound = inner**-0.5
|
| 205 |
+
return nn.Parameter(
|
| 206 |
+
torch.empty(positions, out, inner).uniform_(-bound, bound)
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
self.c_q, self.c_k, self.c_v, self.c_proj = (
|
| 210 |
+
weight(width, width) for _ in range(4)
|
| 211 |
+
)
|
| 212 |
+
self.c_fc = weight(4 * width, width)
|
| 213 |
+
self.c_out = weight(width, 4 * width)
|
| 214 |
+
|
| 215 |
+
def forward(self, x, rope):
|
| 216 |
+
n, p, w = x.shape
|
| 217 |
+
d = w // self.heads
|
| 218 |
+
h = norm(x)
|
| 219 |
+
q, k, v = [
|
| 220 |
+
per_position(h, m).reshape(n, p, self.heads, d)
|
| 221 |
+
for m in (self.c_q, self.c_k, self.c_v)
|
| 222 |
+
]
|
| 223 |
+
cos = rope[0, :p].to(x.dtype).unsqueeze(1)
|
| 224 |
+
sin = rope[1, :p].to(x.dtype).unsqueeze(1)
|
| 225 |
+
q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
|
| 226 |
+
# Rotation and QK norm run at the residual precision; the kernel takes
|
| 227 |
+
# the projection precision.
|
| 228 |
+
y = depth_attention(q.to(v.dtype), k.to(v.dtype), v)
|
| 229 |
+
x = x + per_position(y.reshape(n, p, w), self.c_proj)
|
| 230 |
+
h = F.relu(per_position(norm(x), self.c_fc)).square()
|
| 231 |
+
return x + per_position(h, self.c_out)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
@dataclass
|
| 235 |
+
class TextLayout:
|
| 236 |
+
"""Where text sits in a padded batch, built by `text_layout` and `collate_text`.
|
| 237 |
+
|
| 238 |
+
source [batch, positions] indexes the concatenation of the audio steps
|
| 239 |
+
[frames + 1], the text tokens [tokens] and one zero row for padding.
|
| 240 |
+
token and track [batch, tokens] are the text tokens' ids and tracks.
|
| 241 |
+
last [batch, frames + 1] is the position whose output conditions each step's depth.
|
| 242 |
+
event_* [batch, events] are the text predictions: the position predicting it, its
|
| 243 |
+
track, its target id and its loss weight (0 for padding and the flush step).
|
| 244 |
+
pattern [batch] is the bitmask of present text tracks.
|
| 245 |
+
"""
|
| 246 |
+
|
| 247 |
+
source: torch.Tensor
|
| 248 |
+
token: torch.Tensor
|
| 249 |
+
track: torch.Tensor
|
| 250 |
+
last: torch.Tensor
|
| 251 |
+
event_position: torch.Tensor
|
| 252 |
+
event_track: torch.Tensor
|
| 253 |
+
event_target: torch.Tensor
|
| 254 |
+
event_weight: torch.Tensor
|
| 255 |
+
pattern: torch.Tensor
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def text_layout(frames, text):
|
| 259 |
+
"""One window's text layout as numpy arrays; text is per track None (absent) or an
|
| 260 |
+
int array [n, 2] of (frame, id) rows in word order, frames in [0, frames)."""
|
| 261 |
+
present = [j for j in range(TRACKS) if text[j] is not None]
|
| 262 |
+
counts = np.zeros((frames + 1, TRACKS), dtype=np.int64)
|
| 263 |
+
parts = []
|
| 264 |
+
for j in present:
|
| 265 |
+
rows = np.asarray(text[j], dtype=np.int64).reshape(-1, 2)
|
| 266 |
+
if len(rows) and (rows[:, 0].min() < 0 or rows[:, 0].max() >= frames):
|
| 267 |
+
raise ValueError("Text frame outside the window")
|
| 268 |
+
if len(rows) and (rows[:, 1].min() < 0 or rows[:, 1].max() >= TEXT_IDS):
|
| 269 |
+
raise ValueError("Text id outside the vocabulary")
|
| 270 |
+
np.add.at(counts[:, j], rows[:, 0], 1)
|
| 271 |
+
parts.append(
|
| 272 |
+
np.column_stack(
|
| 273 |
+
(rows[:, 0], np.full(len(rows), j), np.arange(len(rows)), rows[:, 1])
|
| 274 |
+
)
|
| 275 |
+
)
|
| 276 |
+
tokens = np.concatenate(parts) if parts else np.zeros((0, 4), dtype=np.int64)
|
| 277 |
+
# Frame, then track, then word order.
|
| 278 |
+
tokens = tokens[np.lexsort((tokens[:, 2], tokens[:, 1], tokens[:, 0]))]
|
| 279 |
+
per_step = counts.sum(1)
|
| 280 |
+
start = np.concatenate(([0], np.cumsum(1 + per_step)[:-1]))
|
| 281 |
+
before = np.concatenate(([0], np.cumsum(per_step)))
|
| 282 |
+
positions = int(frames + 1 + per_step.sum())
|
| 283 |
+
rank = np.arange(len(tokens)) - before[tokens[:, 0]]
|
| 284 |
+
token_position = start[tokens[:, 0]] + 1 + rank
|
| 285 |
+
source = np.arange(frames + 1 + len(tokens))
|
| 286 |
+
order = np.empty(positions, dtype=np.int64)
|
| 287 |
+
order[start] = np.arange(frames + 1)
|
| 288 |
+
order[token_position] = source[frames + 1 :]
|
| 289 |
+
# Events: each token from the input before it; each advance from the last input
|
| 290 |
+
# of its track in the step, or where the track's first token would come from.
|
| 291 |
+
position = [token_position - 1]
|
| 292 |
+
track = [tokens[:, 1]]
|
| 293 |
+
target = [tokens[:, 3]]
|
| 294 |
+
weight = [np.ones(len(tokens))]
|
| 295 |
+
written = np.zeros(frames + 1, dtype=np.int64)
|
| 296 |
+
for j in present:
|
| 297 |
+
written = written + counts[:, j]
|
| 298 |
+
position.append(start + written)
|
| 299 |
+
track.append(np.full(frames + 1, j))
|
| 300 |
+
target.append(np.full(frames + 1, ADVANCE))
|
| 301 |
+
weight.append(np.r_[np.full(frames, ADVANCE_WEIGHT), 0.0])
|
| 302 |
+
position, track, target, weight = (
|
| 303 |
+
np.concatenate(x) for x in (position, track, target, weight)
|
| 304 |
+
)
|
| 305 |
+
# Sampling order: by position, then track.
|
| 306 |
+
events = np.lexsort((track, position))
|
| 307 |
+
return {
|
| 308 |
+
"source": order,
|
| 309 |
+
"token": tokens[:, 3],
|
| 310 |
+
"track": tokens[:, 1],
|
| 311 |
+
"last": start + per_step,
|
| 312 |
+
"event_position": position[events],
|
| 313 |
+
"event_track": track[events],
|
| 314 |
+
"event_target": target[events],
|
| 315 |
+
"event_weight": weight[events].astype(np.float32),
|
| 316 |
+
"pattern": sum(1 << j for j in present),
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def collate_text(layouts, frames, positions, device):
|
| 321 |
+
"""Pads per-window layouts to [batch, positions] and [batch, frames + 1]."""
|
| 322 |
+
b = len(layouts)
|
| 323 |
+
tokens = max(1, max(len(x["token"]) for x in layouts))
|
| 324 |
+
events = max(1, max(len(x["event_target"]) for x in layouts))
|
| 325 |
+
pad = frames + 1 + tokens
|
| 326 |
+
source = torch.full((b, positions), pad, dtype=torch.long)
|
| 327 |
+
token = torch.zeros((b, tokens), dtype=torch.long)
|
| 328 |
+
track = torch.zeros((b, tokens), dtype=torch.long)
|
| 329 |
+
last = torch.zeros((b, frames + 1), dtype=torch.long)
|
| 330 |
+
event = [torch.zeros((b, events), dtype=torch.long) for _ in range(3)]
|
| 331 |
+
event_weight = torch.zeros((b, events))
|
| 332 |
+
for i, x in enumerate(layouts):
|
| 333 |
+
steps, n, m = len(x["last"]), len(x["token"]), len(x["event_target"])
|
| 334 |
+
if len(x["source"]) > positions or steps > frames + 1:
|
| 335 |
+
raise ValueError("Window exceeds the padded batch")
|
| 336 |
+
s = torch.as_tensor(x["source"])
|
| 337 |
+
# Token sources are rebased past this batch's padded audio steps.
|
| 338 |
+
source[i, : len(s)] = torch.where(s < steps, s, s - steps + frames + 1)
|
| 339 |
+
token[i, :n] = torch.as_tensor(x["token"])
|
| 340 |
+
track[i, :n] = torch.as_tensor(x["track"])
|
| 341 |
+
last[i, :steps] = torch.as_tensor(x["last"])
|
| 342 |
+
for tensor, key in zip(
|
| 343 |
+
event, ("event_position", "event_track", "event_target"), strict=True
|
| 344 |
+
):
|
| 345 |
+
tensor[i, :m] = torch.as_tensor(x[key])
|
| 346 |
+
event_weight[i, :m] = torch.as_tensor(x["event_weight"])
|
| 347 |
+
pattern = torch.tensor([x["pattern"] for x in layouts])
|
| 348 |
+
return TextLayout(
|
| 349 |
+
source.to(device),
|
| 350 |
+
token.to(device),
|
| 351 |
+
track.to(device),
|
| 352 |
+
last.to(device),
|
| 353 |
+
*(t.to(device) for t in event),
|
| 354 |
+
event_weight.to(device),
|
| 355 |
+
pattern.to(device),
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
class DuplexModel(nn.Module):
|
| 360 |
+
def __init__(self, config: Config):
|
| 361 |
+
super().__init__()
|
| 362 |
+
self.config = config
|
| 363 |
+
c = config
|
| 364 |
+
q = c.total_codebooks
|
| 365 |
+
# One table per role with codebook offsets: a single gather instead of
|
| 366 |
+
# one per codebook. Each track has its own rows.
|
| 367 |
+
self.frame_embeddings = nn.EmbeddingBag(q * c.cardinality, c.width, mode="sum")
|
| 368 |
+
# A zero BOS stays zero through bias-free blocks, amplifying RMSNorm's
|
| 369 |
+
# derivative at zero until deep models overflow on the first backward.
|
| 370 |
+
self.bos = nn.Parameter(torch.randn(c.width))
|
| 371 |
+
# Text: one token table for both tracks, a vector per track, an embedding of
|
| 372 |
+
# which text tracks are present added to every audio position (patterns 1 to
|
| 373 |
+
# 3; none adds nothing), and a query per track before the shared head.
|
| 374 |
+
self.text_embeddings = nn.Embedding(TEXT_IDS, c.width)
|
| 375 |
+
self.text_tracks = nn.Parameter(torch.zeros(TRACKS, c.width))
|
| 376 |
+
self.text_presence = nn.Parameter(torch.zeros(2**TRACKS - 1, c.width))
|
| 377 |
+
self.text_queries = nn.Parameter(torch.zeros(TRACKS, c.width))
|
| 378 |
+
if c.text_heads == "shared":
|
| 379 |
+
self.text_head = nn.Linear(c.width, TEXT_IDS + 1, bias=False)
|
| 380 |
+
else:
|
| 381 |
+
# [track, events, width], initialised as nn.Linear initialises one head.
|
| 382 |
+
bound = c.width**-0.5
|
| 383 |
+
self.text_head = nn.Parameter(
|
| 384 |
+
torch.empty(TRACKS, TEXT_IDS + 1, c.width).uniform_(-bound, bound)
|
| 385 |
+
)
|
| 386 |
+
self.temporal = nn.ModuleList(
|
| 387 |
+
[Block(c.width, c.heads) for _ in range(c.layers)]
|
| 388 |
+
)
|
| 389 |
+
self.to_depth = nn.Linear(c.width, c.depth_width, bias=False)
|
| 390 |
+
self.depth_embeddings = nn.Embedding((q - 1) * c.cardinality, c.depth_width)
|
| 391 |
+
self.depth_positions = nn.Parameter(torch.zeros(q, c.depth_width))
|
| 392 |
+
self.depth = nn.ModuleList(
|
| 393 |
+
[DepthBlock(q, c.depth_width, c.depth_heads) for _ in range(c.depth_layers)]
|
| 394 |
+
)
|
| 395 |
+
# Per-codebook output heads [codebook, cardinality, depth_width].
|
| 396 |
+
bound = c.depth_width**-0.5
|
| 397 |
+
self.outputs = nn.Parameter(
|
| 398 |
+
torch.empty(q, c.cardinality, c.depth_width).uniform_(-bound, bound)
|
| 399 |
+
)
|
| 400 |
+
self.register_buffer(
|
| 401 |
+
"offsets", torch.arange(q) * c.cardinality, persistent=False
|
| 402 |
+
)
|
| 403 |
+
# Codebook 0 of each track has delay zero; its acoustic codebooks one.
|
| 404 |
+
self.delays = [int(k % c.codebooks != 0) for k in range(q)]
|
| 405 |
+
self.register_buffer(
|
| 406 |
+
"semantic",
|
| 407 |
+
torch.tensor([d == 0 for d in self.delays]),
|
| 408 |
+
persistent=False,
|
| 409 |
+
)
|
| 410 |
+
self.register_buffer(
|
| 411 |
+
"track_of", torch.arange(q) // c.codebooks, persistent=False
|
| 412 |
+
)
|
| 413 |
+
self.register_buffer(
|
| 414 |
+
"temporal_rope",
|
| 415 |
+
# One position more than the context flushes the last acoustic frame.
|
| 416 |
+
rotary(c.context + 1, c.width // c.heads),
|
| 417 |
+
persistent=False,
|
| 418 |
+
)
|
| 419 |
+
self.register_buffer(
|
| 420 |
+
"depth_rope",
|
| 421 |
+
rotary(q, c.depth_width // c.depth_heads),
|
| 422 |
+
persistent=False,
|
| 423 |
+
)
|
| 424 |
+
|
| 425 |
+
def delayed(self, tokens, valid=None):
|
| 426 |
+
"""Token ids and presence for sequence steps 0..frames, [batch, frames + 1, codebooks].
|
| 427 |
+
|
| 428 |
+
Step s holds codebook 0 of frame s and the acoustic codebooks of frame
|
| 429 |
+
s - 1 in every track. valid is [batch, frames, tracks]: a token is absent
|
| 430 |
+
before the first frame, after the last, on padded frames and on absent tracks.
|
| 431 |
+
"""
|
| 432 |
+
b, t, _ = tokens.shape
|
| 433 |
+
if valid is None:
|
| 434 |
+
valid = tokens.new_ones((b, t, TRACKS), dtype=torch.bool)
|
| 435 |
+
valid = valid[..., self.track_of]
|
| 436 |
+
ids = torch.where(
|
| 437 |
+
self.semantic, F.pad(tokens, (0, 0, 0, 1)), F.pad(tokens, (0, 0, 1, 0))
|
| 438 |
+
)
|
| 439 |
+
present = torch.where(
|
| 440 |
+
self.semantic,
|
| 441 |
+
F.pad(valid, (0, 0, 0, 1)),
|
| 442 |
+
F.pad(valid, (0, 0, 1, 0)),
|
| 443 |
+
)
|
| 444 |
+
return ids, present
|
| 445 |
+
|
| 446 |
+
def audio_steps(self, tokens, valid=None):
|
| 447 |
+
"""Temporal inputs of the audio positions, [batch, frames + 1, width]: BOS, then
|
| 448 |
+
the sum of every present token of the step before."""
|
| 449 |
+
b, t, q = tokens.shape
|
| 450 |
+
if q != self.config.total_codebooks:
|
| 451 |
+
raise ValueError("Expected [batch, frames, codebooks x tracks]")
|
| 452 |
+
x = self.bos.expand(b, 1, -1)
|
| 453 |
+
if t:
|
| 454 |
+
ids, present = self.delayed(tokens, valid)
|
| 455 |
+
weights = self.frame_embeddings.weight
|
| 456 |
+
embedded = self.frame_embeddings(
|
| 457 |
+
(ids[:, :t] + self.offsets).reshape(-1, q),
|
| 458 |
+
per_sample_weights=present[:, :t].reshape(-1, q).to(weights.dtype),
|
| 459 |
+
)
|
| 460 |
+
x = torch.cat((x, embedded.reshape(b, t, -1)), dim=1)
|
| 461 |
+
return x
|
| 462 |
+
|
| 463 |
+
def temporal_outputs(self, tokens, valid=None, text=None):
|
| 464 |
+
"""Normalised temporal outputs [batch, positions, width]."""
|
| 465 |
+
x = self.audio_steps(tokens, valid)
|
| 466 |
+
if text is not None:
|
| 467 |
+
b, _, w = x.shape
|
| 468 |
+
presence = torch.cat((x.new_zeros(1, w), self.text_presence))
|
| 469 |
+
x = x + presence[text.pattern][:, None]
|
| 470 |
+
words = self.text_embeddings(text.token) + self.text_tracks[text.track]
|
| 471 |
+
table = torch.cat((x, words.to(x.dtype), x.new_zeros(b, 1, w)), dim=1)
|
| 472 |
+
x = table.gather(1, text.source[..., None].expand(-1, -1, w))
|
| 473 |
+
if x.shape[1] > self.config.context + 1:
|
| 474 |
+
raise ValueError("Window exceeds the temporal context")
|
| 475 |
+
x = norm(x)
|
| 476 |
+
for block in self.temporal:
|
| 477 |
+
x = block(x, self.temporal_rope)
|
| 478 |
+
return norm(x)
|
| 479 |
+
|
| 480 |
+
def text_logits(self, hidden, track):
|
| 481 |
+
"""Logits [..., TEXT_IDS + 1] of text events from normalised temporal outputs."""
|
| 482 |
+
x = norm(hidden + self.text_queries[track].to(hidden.dtype))
|
| 483 |
+
if self.config.text_heads == "shared":
|
| 484 |
+
return self.text_head(x)
|
| 485 |
+
weights = self.text_head.to(x.dtype)
|
| 486 |
+
track = torch.as_tensor(track, device=x.device)
|
| 487 |
+
if track.dim() == 0:
|
| 488 |
+
return x @ weights[track].T
|
| 489 |
+
# Both heads for every event, each event keeping its own track's: fixed
|
| 490 |
+
# shapes, so the compiled step stays fast (mask selection has
|
| 491 |
+
# data-dependent shapes and trained at about half the step rate).
|
| 492 |
+
return torch.where(track[..., None] == 0, x @ weights[0].T, x @ weights[1].T)
|
| 493 |
+
|
| 494 |
+
def depth_logits(self, context, inputs):
|
| 495 |
+
"""Logits [batch, steps, codebooks, cardinality] in delayed step order.
|
| 496 |
+
|
| 497 |
+
inputs is [batch, steps, codebooks - 1, depth_width]: the embedding of
|
| 498 |
+
the token before each depth position, zero where there is none.
|
| 499 |
+
"""
|
| 500 |
+
b, s, _ = context.shape
|
| 501 |
+
q = self.config.total_codebooks
|
| 502 |
+
x = context.unsqueeze(2) + self.depth_positions
|
| 503 |
+
x = torch.cat((x[:, :, :1], x[:, :, 1:] + inputs), dim=2)
|
| 504 |
+
x = x.reshape(b * s, q, -1)
|
| 505 |
+
for block in self.depth:
|
| 506 |
+
x = block(x, self.depth_rope)
|
| 507 |
+
# [codebook, steps, width] x [codebook, width, cardinality]
|
| 508 |
+
x = norm(x).transpose(0, 1)
|
| 509 |
+
logits = torch.bmm(x, self.outputs.transpose(1, 2).to(x.dtype))
|
| 510 |
+
return logits.transpose(0, 1).reshape(b, s, q, -1)
|
| 511 |
+
|
| 512 |
+
def forward(self, tokens, valid=None, text=None):
|
| 513 |
+
"""Canonical frames in; audio logits aligned to the same [batch, frames,
|
| 514 |
+
codebooks], and text logits [batch, events, TEXT_IDS + 1] or None.
|
| 515 |
+
|
| 516 |
+
valid is the [batch, frames, tracks] mask of real frames of present tracks.
|
| 517 |
+
text is a `TextLayout` or None for audio only.
|
| 518 |
+
|
| 519 |
+
Fillers never appear as targets: codebook 0 of frame t is read from step
|
| 520 |
+
t and its acoustic codebooks from step t + 1, so the final step flushes
|
| 521 |
+
the last frame and every source token is scored once.
|
| 522 |
+
"""
|
| 523 |
+
hidden = self.temporal_outputs(tokens, valid, text)
|
| 524 |
+
steps = hidden
|
| 525 |
+
if text is not None:
|
| 526 |
+
w = hidden.shape[-1]
|
| 527 |
+
steps = hidden.gather(1, text.last[..., None].expand(-1, -1, w))
|
| 528 |
+
context = self.to_depth(steps)
|
| 529 |
+
ids, present = self.delayed(tokens, valid)
|
| 530 |
+
# Each depth position after the first takes the token before it in
|
| 531 |
+
# step order. The step that flushes a short row's last frame sees no
|
| 532 |
+
# codebook 0, as in generation, instead of batch padding.
|
| 533 |
+
inputs = self.depth_embeddings(ids[:, :, :-1] + self.offsets[:-1])
|
| 534 |
+
inputs = inputs * present[:, :, :-1, None].to(inputs.dtype)
|
| 535 |
+
logits = self.depth_logits(context, inputs)
|
| 536 |
+
audio = torch.where(self.semantic[:, None], logits[:, :-1], logits[:, 1:])
|
| 537 |
+
if text is None:
|
| 538 |
+
return audio, None
|
| 539 |
+
w = hidden.shape[-1]
|
| 540 |
+
events = hidden.gather(1, text.event_position[..., None].expand(-1, -1, w))
|
| 541 |
+
return audio, self.text_logits(events, text.event_track)
|
| 542 |
+
|
| 543 |
+
@torch.no_grad()
|
| 544 |
+
def generate(self, prefix, frames, temperature=1.0, top_k=250):
|
| 545 |
+
"""Reference audio sampler without KV cache or text; returns canonical tokens."""
|
| 546 |
+
if frames < 0 or temperature < 0 or top_k < 0:
|
| 547 |
+
raise ValueError("frames, temperature and top_k must be nonnegative")
|
| 548 |
+
c = self.config
|
| 549 |
+
b, forced, q = prefix.shape
|
| 550 |
+
result = torch.cat((prefix, prefix.new_zeros((b, frames, q))), dim=1)
|
| 551 |
+
if not frames:
|
| 552 |
+
return result
|
| 553 |
+
# One step past the last frame flushes its acoustic codebooks.
|
| 554 |
+
for step in range(forced, forced + frames + 1):
|
| 555 |
+
window = result[:, max(0, step - c.context) : step]
|
| 556 |
+
context = self.to_depth(self.temporal_outputs(window)[:, -1:])
|
| 557 |
+
inputs = context.new_zeros((b, 1, q - 1, c.depth_width))
|
| 558 |
+
for position, delay in enumerate(self.delays):
|
| 559 |
+
frame = step - delay
|
| 560 |
+
if not 0 <= frame < forced + frames:
|
| 561 |
+
continue
|
| 562 |
+
# The prefix keeps its own tokens, including the acoustic
|
| 563 |
+
# codebooks of its last frame; later frames are sampled.
|
| 564 |
+
if frame >= forced:
|
| 565 |
+
logits = self.depth_logits(context, inputs)[:, 0, position].float()
|
| 566 |
+
result[:, frame, position] = sample(
|
| 567 |
+
logits, temperature, top_k
|
| 568 |
+
).squeeze(-1)
|
| 569 |
+
if position < q - 1:
|
| 570 |
+
inputs[:, 0, position] = self.depth_embeddings(
|
| 571 |
+
result[:, frame, position] + self.offsets[position]
|
| 572 |
+
)
|
| 573 |
+
return result
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
def sample(logits, temperature, top_k=0, generator=None):
|
| 577 |
+
"""One draw per row of [..., classes] logits; top_k 0 keeps every class."""
|
| 578 |
+
if temperature == 0:
|
| 579 |
+
return logits.argmax(-1, keepdim=True)
|
| 580 |
+
if top_k and top_k < logits.shape[-1]:
|
| 581 |
+
floor = logits.topk(top_k).values[..., -1:]
|
| 582 |
+
logits = logits.masked_fill(logits < floor, float("-inf"))
|
| 583 |
+
probabilities = (logits / temperature).softmax(-1)
|
| 584 |
+
return torch.multinomial(
|
| 585 |
+
probabilities.reshape(-1, logits.shape[-1]), 1, generator=generator
|
| 586 |
+
).reshape(*logits.shape[:-1], 1)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def from_legacy(config, state):
|
| 590 |
+
"""A model in this format from a 14 November checkpoint (one or two streams,
|
| 591 |
+
shared depth weights, no text).
|
| 592 |
+
|
| 593 |
+
A two-stream checkpoint converts exactly: the shared depth block weights are
|
| 594 |
+
copied to every position and the new text weights stay unused while no text
|
| 595 |
+
is present. A one-stream checkpoint is widened as two-stream training began:
|
| 596 |
+
each track gets a copy of the stream's embeddings, positions and heads, and
|
| 597 |
+
track 1's codebook 7 feeding track 2's codebook 0 starts at zero.
|
| 598 |
+
"""
|
| 599 |
+
config = dict(config)
|
| 600 |
+
config.pop("factorization", None)
|
| 601 |
+
streams = config.pop("streams", 1)
|
| 602 |
+
if streams not in (1, 2):
|
| 603 |
+
raise ValueError("Expected a one- or two-stream checkpoint")
|
| 604 |
+
model = DuplexModel(Config(**config))
|
| 605 |
+
c = model.config
|
| 606 |
+
new = model.state_dict()
|
| 607 |
+
state = dict(state)
|
| 608 |
+
if streams == 1:
|
| 609 |
+
for name in ("frame_embeddings.weight", "depth_positions", "outputs"):
|
| 610 |
+
value = state[name]
|
| 611 |
+
state[name] = value.repeat(TRACKS, *[1] * (value.dim() - 1))
|
| 612 |
+
value = state["depth_embeddings.weight"]
|
| 613 |
+
joint = value.new_zeros((c.cardinality, value.shape[1]))
|
| 614 |
+
state["depth_embeddings.weight"] = torch.cat([value, joint, value])
|
| 615 |
+
names = {
|
| 616 |
+
"attn.c_q": "c_q",
|
| 617 |
+
"attn.c_k": "c_k",
|
| 618 |
+
"attn.c_v": "c_v",
|
| 619 |
+
"attn.c_proj": "c_proj",
|
| 620 |
+
"mlp.c_fc": "c_fc",
|
| 621 |
+
"mlp.c_proj": "c_out",
|
| 622 |
+
}
|
| 623 |
+
for layer in range(c.depth_layers):
|
| 624 |
+
for old, name in names.items():
|
| 625 |
+
weight = state.pop(f"depth.{layer}.{old}.weight")
|
| 626 |
+
state[f"depth.{layer}.{name}"] = weight.repeat(c.total_codebooks, 1, 1)
|
| 627 |
+
for name, value in new.items():
|
| 628 |
+
if name.startswith("text_"):
|
| 629 |
+
state[name] = value
|
| 630 |
+
model.load_state_dict(state, strict=True)
|
| 631 |
+
return model
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
def with_text(model, text):
|
| 635 |
+
"""The same weights under another training option, such as a later stage's."""
|
| 636 |
+
widened = DuplexModel(replace(model.config, text=text)).to(model.bos.device)
|
| 637 |
+
widened.load_state_dict(model.state_dict(), strict=True)
|
| 638 |
+
return widened
|
duplex/streaming.py
ADDED
|
@@ -0,0 +1,542 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Incremental sampling for one DuplexModel sequence, for serving.
|
| 2 |
+
|
| 3 |
+
`DuplexModel.generate` recomputes the whole temporal context for every sequence step.
|
| 4 |
+
`Sampler` keeps a key/value cache instead, so a step costs its temporal positions
|
| 5 |
+
and the depth positions it needs. It computes the same distribution as training:
|
| 6 |
+
step s runs the audio position, then each present text track's tokens of frame s
|
| 7 |
+
until its advance token, then predicts codebook 0 of frame s and the acoustic
|
| 8 |
+
codebooks of frame s - 1 in every track from the step's last position. The text
|
| 9 |
+
written on track p at step s is the words starting at frame s + the checkpoint's
|
| 10 |
+
`Config.text_shift[p]`, as in training windows: track 2, spoken as, at their start,
|
| 11 |
+
and track 1, listened to, a few frames after it (6 by default).
|
| 12 |
+
|
| 13 |
+
Each audio position of a step is forced to a given token, sampled, left absent, or
|
| 14 |
+
supplied later. A later position must follow every sampled position of its step,
|
| 15 |
+
because no prediction in the step may depend on it; it must be supplied with
|
| 16 |
+
`complete` before the next step. Live serving forces the caller's track this way
|
| 17 |
+
while it arrives, frame by frame. Text never depends on the step's audio.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
from torch.nn import functional as F
|
| 23 |
+
|
| 24 |
+
from .network import ADVANCE, MAX_TEXT_PER_FRAME, TRACKS, norm, rotate, sample
|
| 25 |
+
|
| 26 |
+
SAMPLE, LATER, ABSENT = -1, -2, -3
|
| 27 |
+
# The sampling standard: audio at temperature 1 from the 250 most likely codes;
|
| 28 |
+
# text at temperature 1 from every token.
|
| 29 |
+
TEMPERATURE, TOP_K = 1.0, 250
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _project(attention, x, rope, position):
|
| 33 |
+
"""Rotated, normalised queries and keys and the values for one position per row.
|
| 34 |
+
|
| 35 |
+
x is [rows, width] and position an int or a [rows] tensor; each result is
|
| 36 |
+
[rows, heads, 1, head_width].
|
| 37 |
+
"""
|
| 38 |
+
head_width = x.shape[-1] // attention.heads
|
| 39 |
+
q, k, v = [
|
| 40 |
+
layer(x).reshape(x.shape[0], attention.heads, 1, head_width)
|
| 41 |
+
for layer in (attention.c_q, attention.c_k, attention.c_v)
|
| 42 |
+
]
|
| 43 |
+
cos = rope[0, position].to(x.dtype)
|
| 44 |
+
sin = rope[1, position].to(x.dtype)
|
| 45 |
+
if cos.dim() == 2:
|
| 46 |
+
cos, sin = cos[:, None, None], sin[:, None, None]
|
| 47 |
+
return norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2, v
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class _Cache:
|
| 51 |
+
"""Keys and values of one transformer stack, [layers, rows, heads, positions, head_width]."""
|
| 52 |
+
|
| 53 |
+
def __init__(self, layers, heads, width, positions, reference, rows=1):
|
| 54 |
+
shape = (layers, rows, heads, positions, width // heads)
|
| 55 |
+
self.keys = reference.new_zeros(shape)
|
| 56 |
+
self.values = reference.new_zeros(shape)
|
| 57 |
+
|
| 58 |
+
def attend(self, i, q, k, v, position):
|
| 59 |
+
self.keys[i, :, :, position] = k[:, :, 0].to(self.keys.dtype)
|
| 60 |
+
self.values[i, :, :, position] = v[:, :, 0].to(self.values.dtype)
|
| 61 |
+
keys = self.keys[i, :, :, : position + 1].to(q.dtype)
|
| 62 |
+
values = self.values[i, :, :, : position + 1].to(q.dtype)
|
| 63 |
+
return F.scaled_dot_product_attention(q, keys, values)
|
| 64 |
+
|
| 65 |
+
def run(self, blocks, x, rope, position):
|
| 66 |
+
"""One temporal position through every block, attending to it and those before."""
|
| 67 |
+
for i, block in enumerate(blocks):
|
| 68 |
+
q, k, v = _project(block.attn, norm(x), rope, position)
|
| 69 |
+
y = self.attend(i, q, k, v, position)
|
| 70 |
+
x = x + block.attn.c_proj(y.reshape(x.shape))
|
| 71 |
+
x = x + block.mlp(norm(x))
|
| 72 |
+
return x
|
| 73 |
+
|
| 74 |
+
def run_rows(self, blocks, x, rope, positions, length):
|
| 75 |
+
"""One temporal position per row, each at its own position [rows], attending to
|
| 76 |
+
its own earlier positions; length bounds every row's position + 1."""
|
| 77 |
+
rows = torch.arange(x.shape[0], device=x.device)
|
| 78 |
+
mask = torch.arange(length, device=x.device) <= positions[:, None]
|
| 79 |
+
mask = mask[:, None, None]
|
| 80 |
+
for i, block in enumerate(blocks):
|
| 81 |
+
q, k, v = _project(block.attn, norm(x), rope, positions)
|
| 82 |
+
self.keys[i, rows, :, positions] = k[:, :, 0].to(self.keys.dtype)
|
| 83 |
+
self.values[i, rows, :, positions] = v[:, :, 0].to(self.values.dtype)
|
| 84 |
+
keys = self.keys[i, :, :, :length].to(q.dtype)
|
| 85 |
+
values = self.values[i, :, :, :length].to(q.dtype)
|
| 86 |
+
y = F.scaled_dot_product_attention(q, keys, values, attn_mask=mask)
|
| 87 |
+
x = x + block.attn.c_proj(y.reshape(x.shape))
|
| 88 |
+
x = x + block.mlp(norm(x))
|
| 89 |
+
return x
|
| 90 |
+
|
| 91 |
+
def run_depth(self, blocks, x, rope, position):
|
| 92 |
+
"""One depth position through every block, with that position's own weights."""
|
| 93 |
+
for i, block in enumerate(blocks):
|
| 94 |
+
heads = block.heads
|
| 95 |
+
d = x.shape[-1] // heads
|
| 96 |
+
h = norm(x)
|
| 97 |
+
q, k, v = [
|
| 98 |
+
(h @ m[position].T.to(h.dtype)).reshape(x.shape[0], heads, 1, d)
|
| 99 |
+
for m in (block.c_q, block.c_k, block.c_v)
|
| 100 |
+
]
|
| 101 |
+
cos = rope[0, position].to(x.dtype)
|
| 102 |
+
sin = rope[1, position].to(x.dtype)
|
| 103 |
+
q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
|
| 104 |
+
y = self.attend(i, q, k, v, position).reshape(x.shape)
|
| 105 |
+
x = x + y @ block.c_proj[position].T.to(y.dtype)
|
| 106 |
+
h = F.relu(norm(x) @ block.c_fc[position].T.to(x.dtype)).square()
|
| 107 |
+
x = x + h @ block.c_out[position].T.to(h.dtype)
|
| 108 |
+
return x
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
class Sampler:
|
| 112 |
+
"""One sequence, one step at a time. Up to `context` + 1 positions; no rollover.
|
| 113 |
+
|
| 114 |
+
text names the present text tracks (0 for track 1, 1 for track 2); the default
|
| 115 |
+
is the checkpoint's training option. With `record`, `log` collects the log
|
| 116 |
+
probability of every text event and every audio token the steps produce or
|
| 117 |
+
are given, in sampling order: ("text", track, id, logp) and ("audio", position,
|
| 118 |
+
id, logp).
|
| 119 |
+
"""
|
| 120 |
+
|
| 121 |
+
def __init__(
|
| 122 |
+
self,
|
| 123 |
+
model,
|
| 124 |
+
temperature=TEMPERATURE,
|
| 125 |
+
generator=None,
|
| 126 |
+
top_k=TOP_K,
|
| 127 |
+
text=None,
|
| 128 |
+
record=False,
|
| 129 |
+
text_sampling=None,
|
| 130 |
+
):
|
| 131 |
+
if temperature < 0 or top_k < 0:
|
| 132 |
+
raise ValueError("temperature and top_k must be nonnegative")
|
| 133 |
+
self.model = model
|
| 134 |
+
self.temperature = temperature
|
| 135 |
+
# 0 samples from every code; k samples from the k most likely at each position.
|
| 136 |
+
self.top_k = top_k
|
| 137 |
+
# Text's (temperature, top_k); default the audio temperature from every token.
|
| 138 |
+
self.text_sampling = text_sampling or (temperature, 0)
|
| 139 |
+
self.generator = generator
|
| 140 |
+
c = model.config
|
| 141 |
+
self.text = tuple(model.config.text_tracks if text is None else text)
|
| 142 |
+
if any(j not in range(TRACKS) for j in self.text):
|
| 143 |
+
raise ValueError("Text tracks are 0 and 1")
|
| 144 |
+
self.pattern = sum(1 << j for j in self.text)
|
| 145 |
+
self.q = c.total_codebooks
|
| 146 |
+
weights = model.bos.detach()
|
| 147 |
+
self.temporal = _Cache(c.layers, c.heads, c.width, c.context + 1, weights)
|
| 148 |
+
self.steps = 0
|
| 149 |
+
self.positions = 0
|
| 150 |
+
# Tokens of the previous step in delayed order; ABSENT where none.
|
| 151 |
+
self.previous = None
|
| 152 |
+
self.log = [] if record else None
|
| 153 |
+
|
| 154 |
+
def room(self):
|
| 155 |
+
"""Whether one more step fits in the context at the per-frame text limit."""
|
| 156 |
+
need = 1 + len(self.text) * MAX_TEXT_PER_FRAME
|
| 157 |
+
return self.positions + need <= self.model.config.context + 1
|
| 158 |
+
|
| 159 |
+
def _position(self, x):
|
| 160 |
+
if self.positions > self.model.config.context:
|
| 161 |
+
raise ValueError("Sequence exceeds the model context")
|
| 162 |
+
m = self.model
|
| 163 |
+
h = self.temporal.run(m.temporal, norm(x), m.temporal_rope, self.positions)
|
| 164 |
+
self.positions += 1
|
| 165 |
+
return norm(h)
|
| 166 |
+
|
| 167 |
+
@torch.no_grad()
|
| 168 |
+
def step(self, forced, text=None):
|
| 169 |
+
"""Run the next step. forced has one entry per codebook position: a token,
|
| 170 |
+
SAMPLE, LATER or ABSENT. Acoustic positions of step 0 are absent whatever is
|
| 171 |
+
passed. text has one entry per present text track: SAMPLE, or the list of
|
| 172 |
+
that frame's given token ids (the advance is implied); default all SAMPLE.
|
| 173 |
+
|
| 174 |
+
Returns the step's audio tokens, with LATER where still to be supplied, and
|
| 175 |
+
the text tokens written on each present track."""
|
| 176 |
+
m = self.model
|
| 177 |
+
if self.previous is not None and LATER in self.previous:
|
| 178 |
+
raise ValueError("Complete the previous step first")
|
| 179 |
+
tokens = [int(t) for t in forced]
|
| 180 |
+
if len(tokens) != self.q:
|
| 181 |
+
raise ValueError(f"Expected {self.q} positions")
|
| 182 |
+
if self.steps == 0:
|
| 183 |
+
tokens = [t if d == 0 else ABSENT for t, d in zip(tokens, m.delays)]
|
| 184 |
+
text = [SAMPLE] * len(self.text) if text is None else list(text)
|
| 185 |
+
if len(text) != len(self.text):
|
| 186 |
+
raise ValueError("One text entry per present text track")
|
| 187 |
+
sampled = [k for k, t in enumerate(tokens) if t == SAMPLE]
|
| 188 |
+
later = [k for k, t in enumerate(tokens) if t == LATER]
|
| 189 |
+
if later and sampled and min(later) < max(sampled):
|
| 190 |
+
raise ValueError("A later position precedes a sampled one")
|
| 191 |
+
|
| 192 |
+
if self.previous is None:
|
| 193 |
+
x = m.bos[None]
|
| 194 |
+
else:
|
| 195 |
+
present = [k for k, t in enumerate(self.previous) if t >= 0]
|
| 196 |
+
ids = torch.tensor([self.previous[k] for k in present], device=m.bos.device)
|
| 197 |
+
rows = m.frame_embeddings.weight[ids + m.offsets[present]]
|
| 198 |
+
x = rows.sum(0, keepdim=True).to(m.bos.dtype)
|
| 199 |
+
if self.pattern:
|
| 200 |
+
x = x + m.text_presence[self.pattern - 1].to(x.dtype)
|
| 201 |
+
h = self._position(x)
|
| 202 |
+
|
| 203 |
+
written = []
|
| 204 |
+
for track, given in zip(self.text, text, strict=True):
|
| 205 |
+
out = []
|
| 206 |
+
while True:
|
| 207 |
+
logits = m.text_logits(h, torch.tensor(track, device=h.device))[0]
|
| 208 |
+
logits = logits.float()
|
| 209 |
+
if given == SAMPLE:
|
| 210 |
+
if len(out) == MAX_TEXT_PER_FRAME:
|
| 211 |
+
token = ADVANCE
|
| 212 |
+
else:
|
| 213 |
+
token = int(sample(logits, *self.text_sampling, self.generator))
|
| 214 |
+
else:
|
| 215 |
+
token = given[len(out)] if len(out) < len(given) else ADVANCE
|
| 216 |
+
if self.log is not None:
|
| 217 |
+
self.log.append(
|
| 218 |
+
("text", track, token, float(logits.log_softmax(-1)[token]))
|
| 219 |
+
)
|
| 220 |
+
if token == ADVANCE:
|
| 221 |
+
break
|
| 222 |
+
out.append(token)
|
| 223 |
+
e = m.text_embeddings.weight[token] + m.text_tracks[track]
|
| 224 |
+
h = self._position(e[None].to(h.dtype))
|
| 225 |
+
written.append(out)
|
| 226 |
+
|
| 227 |
+
context = m.to_depth(h)
|
| 228 |
+
scored = [k for k, t in enumerate(tokens) if t >= 0 or t == SAMPLE]
|
| 229 |
+
last = max(scored if self.log is not None else sampled, default=-1)
|
| 230 |
+
if last >= 0:
|
| 231 |
+
c = m.config
|
| 232 |
+
depth = _Cache(
|
| 233 |
+
c.depth_layers, c.depth_heads, c.depth_width, self.q, context
|
| 234 |
+
)
|
| 235 |
+
for position in range(last + 1):
|
| 236 |
+
x = context + m.depth_positions[position]
|
| 237 |
+
if position and tokens[position - 1] >= 0:
|
| 238 |
+
x = x + m.depth_embeddings.weight[
|
| 239 |
+
tokens[position - 1] + m.offsets[position - 1]
|
| 240 |
+
].to(x.dtype)
|
| 241 |
+
x = depth.run_depth(m.depth, x, m.depth_rope, position)
|
| 242 |
+
if tokens[position] != SAMPLE and (
|
| 243 |
+
self.log is None or tokens[position] < 0
|
| 244 |
+
):
|
| 245 |
+
continue
|
| 246 |
+
logits = (norm(x) @ m.outputs[position].T.to(x.dtype))[0].float()
|
| 247 |
+
if tokens[position] == SAMPLE:
|
| 248 |
+
tokens[position] = int(
|
| 249 |
+
sample(logits, self.temperature, self.top_k, self.generator)
|
| 250 |
+
)
|
| 251 |
+
if self.log is not None:
|
| 252 |
+
self.log.append(
|
| 253 |
+
(
|
| 254 |
+
"audio",
|
| 255 |
+
position,
|
| 256 |
+
tokens[position],
|
| 257 |
+
float(logits.log_softmax(-1)[tokens[position]]),
|
| 258 |
+
)
|
| 259 |
+
)
|
| 260 |
+
self.previous = tokens
|
| 261 |
+
self.steps += 1
|
| 262 |
+
return list(tokens), written
|
| 263 |
+
|
| 264 |
+
def complete(self, supplied):
|
| 265 |
+
"""Supply the LATER positions of the last step: {position: token or ABSENT}.
|
| 266 |
+
|
| 267 |
+
Positions the step made absent, the acoustics of step 0, are skipped."""
|
| 268 |
+
for position, token in supplied.items():
|
| 269 |
+
if self.previous[position] == ABSENT:
|
| 270 |
+
continue
|
| 271 |
+
if self.previous[position] != LATER:
|
| 272 |
+
raise ValueError(f"Position {position} was not left for later")
|
| 273 |
+
self.previous[position] = int(token)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class Duplex:
|
| 277 |
+
"""Live two-track sampling: one caller frame in, one model frame out per call.
|
| 278 |
+
|
| 279 |
+
With `model_stream` 2 the caller is track 1 and the model hears each caller
|
| 280 |
+
frame in the step that samples its own. Its frame s is complete one step later,
|
| 281 |
+
so the first call returns None. With `model_stream` 1 the model samples each step
|
| 282 |
+
before the caller's frame arrives and returns the frame of the same index.
|
| 283 |
+
|
| 284 |
+
The model writes the text of its own track when its training option has it.
|
| 285 |
+
Under the `two` option the caller's text track is written by the model too, as
|
| 286 |
+
its reading of the caller. `text` holds the latest text tokens by track.
|
| 287 |
+
"""
|
| 288 |
+
|
| 289 |
+
def __init__(
|
| 290 |
+
self, model, model_stream, temperature=TEMPERATURE, generator=None, top_k=TOP_K
|
| 291 |
+
):
|
| 292 |
+
if model_stream not in (1, 2):
|
| 293 |
+
raise ValueError("model_stream is 1 or 2")
|
| 294 |
+
own = model_stream - 1
|
| 295 |
+
tracks = model.config.text_tracks
|
| 296 |
+
# Track 2 is the one trained to speak with its own text at its start frames.
|
| 297 |
+
if tracks and own != 1:
|
| 298 |
+
raise ValueError("A checkpoint with text speaks only as track 2")
|
| 299 |
+
self.model_stream = model_stream
|
| 300 |
+
self.sampler = Sampler(model, temperature, generator, top_k)
|
| 301 |
+
self.previous = None # the caller's last frame
|
| 302 |
+
self.semantic = None # the model's codebook 0 awaiting its acoustics
|
| 303 |
+
self.text = {}
|
| 304 |
+
if model_stream == 1:
|
| 305 |
+
tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
|
| 306 |
+
self.semantic = tokens[0]
|
| 307 |
+
self.text = dict(zip(tracks, written, strict=True))
|
| 308 |
+
|
| 309 |
+
def full(self):
|
| 310 |
+
return not self.sampler.room()
|
| 311 |
+
|
| 312 |
+
def push(self, caller):
|
| 313 |
+
"""The caller's next frame, 8 codes; returns the model's next frame or None."""
|
| 314 |
+
caller = [int(c) for c in caller]
|
| 315 |
+
given = [caller[0]] + (self.previous[1:] if self.previous else [ABSENT] * 7)
|
| 316 |
+
self.previous = caller
|
| 317 |
+
tracks = self.sampler.text
|
| 318 |
+
if self.model_stream == 1:
|
| 319 |
+
self.sampler.complete(dict(enumerate(given, start=8)))
|
| 320 |
+
tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
|
| 321 |
+
frame, self.semantic = [self.semantic] + tokens[1:8], tokens[0]
|
| 322 |
+
self.text = dict(zip(tracks, written, strict=True))
|
| 323 |
+
return frame
|
| 324 |
+
tokens, written = self.sampler.step(given + [SAMPLE] * 8)
|
| 325 |
+
self.text = dict(zip(tracks, written, strict=True))
|
| 326 |
+
frame = None if self.semantic is None else [self.semantic] + tokens[9:]
|
| 327 |
+
self.semantic = tokens[8]
|
| 328 |
+
return frame
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def generate(
|
| 332 |
+
model,
|
| 333 |
+
frames,
|
| 334 |
+
temperature=TEMPERATURE,
|
| 335 |
+
generator=None,
|
| 336 |
+
top_k=TOP_K,
|
| 337 |
+
text=None,
|
| 338 |
+
given=None,
|
| 339 |
+
text_sampling=None,
|
| 340 |
+
):
|
| 341 |
+
"""Fill the SAMPLE entries of canonical frames [frames, codebooks] in delayed order.
|
| 342 |
+
|
| 343 |
+
A prefix of given frames followed by SAMPLE rows matches `DuplexModel.generate`
|
| 344 |
+
when no text track is present. Any other mix forces some positions while
|
| 345 |
+
sampling the rest, such as one track given and the other generated; ABSENT
|
| 346 |
+
marks an empty track. text names the present text tracks (default: the
|
| 347 |
+
checkpoint's option), each sampled unless `given` maps it to the token ids
|
| 348 |
+
to write in each frame (an empty list writes only the advance; SAMPLE samples
|
| 349 |
+
that frame). text_sampling is text's (temperature, top_k), by default the
|
| 350 |
+
audio temperature from every token. Returns the
|
| 351 |
+
tokens and, per present text track, the token ids written in each frame.
|
| 352 |
+
"""
|
| 353 |
+
given = given or {}
|
| 354 |
+
result = frames.clone()
|
| 355 |
+
total = len(result)
|
| 356 |
+
sampler = Sampler(
|
| 357 |
+
model, temperature, generator, top_k, text, text_sampling=text_sampling
|
| 358 |
+
)
|
| 359 |
+
written = {track: [] for track in sampler.text}
|
| 360 |
+
for step in range(total + 1):
|
| 361 |
+
rows = [step - d for d in model.delays]
|
| 362 |
+
forced = [
|
| 363 |
+
int(result[f, k]) if 0 <= f < total else ABSENT for k, f in enumerate(rows)
|
| 364 |
+
]
|
| 365 |
+
# The flush step has no frame, so its text tracks write nothing.
|
| 366 |
+
tokens, out = sampler.step(
|
| 367 |
+
forced,
|
| 368 |
+
[given[t][step] if t in given else SAMPLE for t in sampler.text]
|
| 369 |
+
if step < total
|
| 370 |
+
else [[] for _ in sampler.text],
|
| 371 |
+
)
|
| 372 |
+
for track, ids in zip(sampler.text, out, strict=True):
|
| 373 |
+
if step < total:
|
| 374 |
+
written[track].append(ids)
|
| 375 |
+
for k, f in enumerate(rows):
|
| 376 |
+
if 0 <= f < total and int(result[f, k]) != ABSENT:
|
| 377 |
+
result[f, k] = tokens[k]
|
| 378 |
+
return result, written
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
# Uniform draws per clip and step: each text track's events, then each codebook.
|
| 382 |
+
_TEXT_SLOTS = MAX_TEXT_PER_FRAME + 1
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def _draw(logits, temperature, top_k, u):
|
| 386 |
+
"""One inverse-CDF draw per row of logits [rows, classes] from uniforms u [rows],
|
| 387 |
+
over the top_k most likely classes (0 keeps every class); argmax at temperature 0."""
|
| 388 |
+
if temperature == 0:
|
| 389 |
+
return logits.argmax(-1)
|
| 390 |
+
k = top_k if top_k and top_k < logits.shape[-1] else logits.shape[-1]
|
| 391 |
+
values, indices = logits.topk(k)
|
| 392 |
+
cdf = (values / temperature).softmax(-1).cumsum(-1)
|
| 393 |
+
target = (u.to(cdf.dtype) * cdf[:, -1]).unsqueeze(-1)
|
| 394 |
+
pick = torch.searchsorted(cdf, target).clamp(max=k - 1)
|
| 395 |
+
return indices.gather(-1, pick).squeeze(-1)
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
@torch.no_grad()
|
| 399 |
+
def generate_batch(
|
| 400 |
+
model,
|
| 401 |
+
frames,
|
| 402 |
+
seeds,
|
| 403 |
+
temperature=TEMPERATURE,
|
| 404 |
+
top_k=TOP_K,
|
| 405 |
+
text_sampling=None,
|
| 406 |
+
given=None,
|
| 407 |
+
):
|
| 408 |
+
"""`generate` for a batch of clips [rows, frames, codebooks].
|
| 409 |
+
|
| 410 |
+
Text is sampled unless `given[row]` maps a track to its token ids per frame, as
|
| 411 |
+
in `generate` (SAMPLE samples that frame); forced rows ignore their draws.
|
| 412 |
+
|
| 413 |
+
Each row keeps its own temporal position, because rows write different numbers
|
| 414 |
+
of text tokens, and draws from its own uniforms, seeded by `seeds[row]`, so a
|
| 415 |
+
row's output does not depend on the rest of the batch, up to float rounding. The distribution is the
|
| 416 |
+
batch-one sampler's; at temperature 0 the tokens are its tokens. Returns the
|
| 417 |
+
tokens and, per present text track, each row's token ids written in each frame.
|
| 418 |
+
"""
|
| 419 |
+
m, c = model, model.config
|
| 420 |
+
device = m.bos.device
|
| 421 |
+
result = frames.to(device).clone()
|
| 422 |
+
rows, total, q = result.shape
|
| 423 |
+
if len(seeds) != rows:
|
| 424 |
+
raise ValueError("One seed per row")
|
| 425 |
+
text_temperature, text_top_k = text_sampling or (temperature, 0)
|
| 426 |
+
tracks = tuple(c.text_tracks)
|
| 427 |
+
uniforms = torch.from_numpy(
|
| 428 |
+
np.stack(
|
| 429 |
+
[
|
| 430 |
+
np.random.default_rng(seed).random(
|
| 431 |
+
(total + 1, len(tracks) * _TEXT_SLOTS + q)
|
| 432 |
+
)
|
| 433 |
+
for seed in seeds
|
| 434 |
+
]
|
| 435 |
+
)
|
| 436 |
+
).to(device)
|
| 437 |
+
# One spare position: rows that already advanced write a placeholder there.
|
| 438 |
+
cache = _Cache(c.layers, c.heads, c.width, c.context + 2, m.bos.detach(), rows)
|
| 439 |
+
positions = torch.zeros(rows, dtype=torch.long, device=device)
|
| 440 |
+
furthest = 0
|
| 441 |
+
written = {track: [[] for _ in range(rows)] for track in tracks}
|
| 442 |
+
delays = torch.tensor(m.delays, device=device)
|
| 443 |
+
codebooks = torch.arange(q, device=device)
|
| 444 |
+
previous = None
|
| 445 |
+
|
| 446 |
+
def advance(x, moving):
|
| 447 |
+
nonlocal furthest
|
| 448 |
+
h = cache.run_rows(
|
| 449 |
+
m.temporal, norm(x), m.temporal_rope, positions, furthest + 1
|
| 450 |
+
)
|
| 451 |
+
positions.add_(moving.long())
|
| 452 |
+
furthest = int(positions.max())
|
| 453 |
+
if furthest > c.context + 1:
|
| 454 |
+
raise ValueError("Sequence exceeds the model context")
|
| 455 |
+
return norm(h)
|
| 456 |
+
|
| 457 |
+
given = given or [{} for _ in range(rows)]
|
| 458 |
+
if len(given) != rows:
|
| 459 |
+
raise ValueError("One given map per row")
|
| 460 |
+
everyone = torch.ones(rows, dtype=torch.bool, device=device)
|
| 461 |
+
for step in range(total + 1):
|
| 462 |
+
frame = step - delays
|
| 463 |
+
inside = (frame >= 0) & (frame < total)
|
| 464 |
+
forced = result[:, frame.clamp(0, total - 1), codebooks]
|
| 465 |
+
forced = torch.where(inside, forced, torch.full_like(forced, ABSENT))
|
| 466 |
+
if previous is None:
|
| 467 |
+
x = m.bos.expand(rows, -1)
|
| 468 |
+
else:
|
| 469 |
+
present = previous >= 0
|
| 470 |
+
embedded = m.frame_embeddings.weight[previous.clamp(min=0) + m.offsets]
|
| 471 |
+
x = (embedded * present[..., None].to(embedded.dtype)).sum(1)
|
| 472 |
+
x = x.to(m.bos.dtype)
|
| 473 |
+
if tracks:
|
| 474 |
+
x = x + m.text_presence[sum(1 << j for j in tracks) - 1].to(x.dtype)
|
| 475 |
+
h = advance(x, everyone)
|
| 476 |
+
# The flush step has no frame, so its text tracks write nothing.
|
| 477 |
+
for slot, track in enumerate(tracks if step < total else ()):
|
| 478 |
+
active = everyone
|
| 479 |
+
forced_text = [
|
| 480 |
+
g[track][step] if track in g and g[track][step] != SAMPLE else None
|
| 481 |
+
for g in given
|
| 482 |
+
]
|
| 483 |
+
longest = max((len(t) for t in forced_text if t is not None), default=0)
|
| 484 |
+
# A sampled track advances after MAX_TEXT_PER_FRAME tokens without a draw;
|
| 485 |
+
# a forced one after its given tokens.
|
| 486 |
+
for count in range(max(MAX_TEXT_PER_FRAME, longest + 1)):
|
| 487 |
+
logits = m.text_logits(h, torch.tensor(track, device=device)).float()
|
| 488 |
+
u = uniforms[:, step, slot * _TEXT_SLOTS + min(count, _TEXT_SLOTS - 1)]
|
| 489 |
+
token = _draw(logits, text_temperature, text_top_k, u)
|
| 490 |
+
if count >= MAX_TEXT_PER_FRAME:
|
| 491 |
+
token = torch.full_like(token, ADVANCE)
|
| 492 |
+
for row, ids in enumerate(forced_text):
|
| 493 |
+
if ids is not None:
|
| 494 |
+
token[row] = ids[count] if count < len(ids) else ADVANCE
|
| 495 |
+
moving = active & (token != ADVANCE)
|
| 496 |
+
if not bool(moving.any()):
|
| 497 |
+
break
|
| 498 |
+
for row in moving.nonzero()[:, 0].tolist():
|
| 499 |
+
if count == 0:
|
| 500 |
+
written[track][row].append([])
|
| 501 |
+
written[track][row][-1].append(int(token[row]))
|
| 502 |
+
e = m.text_embeddings.weight[token.clamp(max=ADVANCE - 1)]
|
| 503 |
+
e = e + m.text_tracks[track]
|
| 504 |
+
h = torch.where(moving[:, None], advance(e.to(h.dtype), moving), h)
|
| 505 |
+
active = moving
|
| 506 |
+
for row in range(rows):
|
| 507 |
+
if len(written[track][row]) == step:
|
| 508 |
+
written[track][row].append([])
|
| 509 |
+
tokens = forced.clone()
|
| 510 |
+
sampled = forced == SAMPLE
|
| 511 |
+
wanted = sampled.any(0).nonzero()
|
| 512 |
+
if len(wanted):
|
| 513 |
+
context = m.to_depth(h)
|
| 514 |
+
depth = _Cache(
|
| 515 |
+
c.depth_layers, c.depth_heads, c.depth_width, q, context, rows
|
| 516 |
+
)
|
| 517 |
+
for position in range(int(wanted.max()) + 1):
|
| 518 |
+
x = context + m.depth_positions[position]
|
| 519 |
+
if position:
|
| 520 |
+
before = tokens[:, position - 1]
|
| 521 |
+
e = m.depth_embeddings.weight[
|
| 522 |
+
before.clamp(min=0) + m.offsets[position - 1]
|
| 523 |
+
]
|
| 524 |
+
x = x + (e * (before >= 0)[:, None].to(e.dtype)).to(x.dtype)
|
| 525 |
+
x = depth.run_depth(m.depth, x, m.depth_rope, position)
|
| 526 |
+
if not bool(sampled[:, position].any()):
|
| 527 |
+
continue
|
| 528 |
+
logits = (norm(x) @ m.outputs[position].T.to(x.dtype)).float()
|
| 529 |
+
u = uniforms[:, step, len(tracks) * _TEXT_SLOTS + position]
|
| 530 |
+
draw = _draw(logits, temperature, top_k, u)
|
| 531 |
+
tokens[:, position] = torch.where(
|
| 532 |
+
sampled[:, position], draw, tokens[:, position]
|
| 533 |
+
)
|
| 534 |
+
previous = tokens
|
| 535 |
+
keep = inside & (result[:, frame.clamp(0, total - 1), codebooks] != ABSENT)
|
| 536 |
+
update = torch.where(
|
| 537 |
+
keep, tokens, result[:, frame.clamp(0, total - 1), codebooks]
|
| 538 |
+
)
|
| 539 |
+
result[:, frame.clamp(0, total - 1)[inside], codebooks[inside]] = update[
|
| 540 |
+
:, inside
|
| 541 |
+
]
|
| 542 |
+
return result, written
|
examples/prompt1.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:45409befa6feaad396a2370213e38caff1adea69804c3a4f6f4f020b34d9fb7d
|
| 3 |
+
size 480044
|
examples/prompt2.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a21273bb937f6119637c4a754eaa447f09f0b0fc054668e4774f36c4e9406f7
|
| 3 |
+
size 480044
|
examples/prompt3.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f75656b3877b665e0f317ba9d5a8ffbbc6be24e1eba09dfbaf2d8a14eb47c259
|
| 3 |
+
size 480044
|
examples/prompt4.wav
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2c43ce501f6a4d6f011fe79fcae57ddbd01e136fba1a661b0e6ee813b037b50b
|
| 3 |
+
size 480044
|
requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch
|
| 2 |
+
transformers
|
| 3 |
+
sentencepiece
|
| 4 |
+
safetensors
|
| 5 |
+
librosa
|
| 6 |
+
soundfile
|
| 7 |
+
numpy
|