multimodalart HF Staff commited on
Commit
d20eb3e
·
verified ·
1 Parent(s): ab1e0f7

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ examples/prompt1.wav filter=lfs diff=lfs merge=lfs -text
37
+ examples/prompt2.wav filter=lfs diff=lfs merge=lfs -text
38
+ examples/prompt3.wav filter=lfs diff=lfs merge=lfs -text
39
+ examples/prompt4.wav filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,13 +1,22 @@
1
  ---
2
- title: Duplex 0
3
- emoji: 🦀
4
- colorFrom: pink
5
  colorTo: gray
6
  sdk: gradio
7
  sdk_version: 6.32.0
8
- python_version: '3.12'
9
  app_file: app.py
10
- pinned: false
 
 
 
 
 
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
1
  ---
2
+ title: duplex-0
3
+ emoji: 🎙️
4
+ colorFrom: blue
5
  colorTo: gray
6
  sdk: gradio
7
  sdk_version: 6.32.0
 
8
  app_file: app.py
9
+ short_description: Small full-duplex speech-to-speech model (Mimi tokens)
10
+ python_version: "3.12"
11
+ startup_duration_timeout: 30m
12
+ models:
13
+ - Trelis/duplex-0
14
+ - kyutai/mimi
15
+ - kyutai/moshiko-pytorch-bf16
16
  ---
17
 
18
+ Demo of [Trelis/duplex-0](https://huggingface.co/Trelis/duplex-0), a 201M-parameter two-track speech-to-speech research model.
19
+
20
+ The model code in `duplex/` is copied from the model repo's `scripts/duplex/model` (Apache-2.0, see `duplex/NOTICE`).
21
+ Example prompts are the first 10 seconds of the "real" sample clips in the model repo's `report/samples/` (Apache-2.0 repository).
22
+ Audio is encoded/decoded with Kyutai's [Mimi](https://huggingface.co/kyutai/mimi) codec, and text uses Moshi's tokenizer.
app.py ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces # must come before torch
2
+
3
+ import time
4
+
5
+ import gradio as gr
6
+ import librosa
7
+ import numpy as np
8
+ import sentencepiece as spm
9
+ import torch
10
+ from huggingface_hub import hf_hub_download, snapshot_download
11
+ from transformers import MimiModel
12
+
13
+ from duplex import streaming
14
+ from duplex.checkpoint import load_published
15
+
16
+ REPO = "Trelis/duplex-0"
17
+ SR = 24000
18
+ FPS = 12.5
19
+ SAMPLES_PER_FRAME = 1920
20
+ MAX_INPUT_S = 30.0
21
+ MAX_GEN_S = 30.0
22
+
23
+ CHECKPOINTS = {
24
+ "Audio only": "conversational-none",
25
+ "One text track": "conversational-one",
26
+ "Two text tracks": "conversational-two",
27
+ }
28
+
29
+ root = snapshot_download(
30
+ REPO,
31
+ allow_patterns=[f"resplit-20261117/{c}/{f}" for c in CHECKPOINTS.values()
32
+ for f in ("config.json", "model.safetensors")],
33
+ )
34
+ MODELS = {
35
+ name: load_published(f"{root}/resplit-20261117/{folder}").to("cuda").eval()
36
+ for name, folder in CHECKPOINTS.items()
37
+ }
38
+ mimi = MimiModel.from_pretrained("kyutai/mimi").to("cuda").eval()
39
+ sp = spm.SentencePieceProcessor(
40
+ model_file=hf_hub_download("kyutai/moshiko-pytorch-bf16", "tokenizer_spm_32k_3.model")
41
+ )
42
+
43
+
44
+ def _load_audio(path):
45
+ """Audio file -> list of 1 or 2 float32 channels at 24 kHz, at most MAX_INPUT_S."""
46
+ y, _ = librosa.load(path, sr=SR, mono=False, duration=MAX_INPUT_S)
47
+ if y.ndim == 1:
48
+ return [y]
49
+ if y.shape[0] >= 2 and not np.allclose(y[0], y[1], atol=1e-4):
50
+ return [y[0], y[1]]
51
+ return [y.mean(0)]
52
+
53
+
54
+ @torch.no_grad()
55
+ def _encode(wave):
56
+ x = torch.from_numpy(np.ascontiguousarray(wave, dtype=np.float32)).to("cuda")
57
+ codes = mimi.encode(x[None, None], num_quantizers=8).audio_codes # [1, 8, T]
58
+ return codes[0].T.long() # [T, 8]
59
+
60
+
61
+ @torch.no_grad()
62
+ def _decode(codes):
63
+ """codes [T, 8] (negative = absent) -> waveform numpy."""
64
+ codes = codes.clamp(min=0)
65
+ audio = mimi.decode(codes.T[None]).audio_values # [1, 1, samples]
66
+ return audio[0, 0].float().cpu().numpy()
67
+
68
+
69
+ def _text(written, tracks):
70
+ lines = []
71
+ names = {0: "Track 1 (caller / other speaker)", 1: "Track 2 (model's own voice)"}
72
+ for track in tracks:
73
+ ids = [i for frame in written.get(track, []) for i in frame]
74
+ txt = sp.decode(ids).strip() if ids else ""
75
+ lines.append(f"**{names[track]}:** {txt or '_(nothing written)_'}")
76
+ return "\n\n".join(lines)
77
+
78
+
79
+ def _duration(audio, mode="Reply", checkpoint="Audio only", seconds=10.0, *args, **kwargs):
80
+ total = MAX_INPUT_S + float(seconds)
81
+ per_s = 1.2 if checkpoint == "Audio only" else 2.0
82
+ return int(min(180, 20 + total * per_s))
83
+
84
+
85
+ @spaces.GPU(duration=_duration)
86
+ def generate(
87
+ audio: str,
88
+ mode: str = "Reply",
89
+ checkpoint: str = "Audio only",
90
+ seconds: float = 10.0,
91
+ temperature: float = 1.0,
92
+ top_k: int = 250,
93
+ seed: int = 1,
94
+ ):
95
+ """Run duplex-0 on a speech clip.
96
+
97
+ Args:
98
+ audio: Path to the input speech (wav/mp3). Mono, or stereo with one speaker per channel.
99
+ mode: "Reply" - the model listens to the clip as the caller (track 1) and speaks as
100
+ the other side (track 2) the whole time, then for `seconds` more after the clip ends.
101
+ "Continue" - the clip is a prompt and the model continues it for `seconds`.
102
+ checkpoint: "Audio only", "One text track" or "Two text tracks" (conversational stage).
103
+ seconds: Seconds to generate after the clip ends.
104
+ temperature: Audio sampling temperature.
105
+ top_k: Audio top-k.
106
+ seed: Random seed.
107
+
108
+ Returns:
109
+ Mixed audio of both tracks, the model's own track, and the text it wrote.
110
+ """
111
+ if audio is None:
112
+ raise gr.Error("Please provide an audio clip.")
113
+ t0 = time.perf_counter()
114
+ model = MODELS[checkpoint]
115
+ seconds = float(min(max(seconds, 1.0), MAX_GEN_S))
116
+ channels = _load_audio(audio)
117
+ if len(channels[0]) < SR:
118
+ raise gr.Error("The clip must be at least 1 second long.")
119
+ new = int(round(seconds * FPS))
120
+ SAMPLE, ABSENT = streaming.SAMPLE, streaming.ABSENT
121
+
122
+ if mode == "Reply":
123
+ caller = _encode(channels[0] if len(channels) == 1 else (channels[0] + channels[1]) / 2)
124
+ silence = _encode(np.zeros(new * SAMPLES_PER_FRAME, dtype=np.float32))[:new]
125
+ caller = torch.cat((caller, silence))
126
+ frames = torch.full((len(caller), 16), SAMPLE, dtype=torch.long, device="cuda")
127
+ frames[:, :8] = caller
128
+ model_tracks = [1]
129
+ else:
130
+ prompt = [_encode(c) for c in channels]
131
+ t = min(len(p) for p in prompt)
132
+ frames = torch.full((t + new, 16), SAMPLE, dtype=torch.long, device="cuda")
133
+ if len(prompt) == 2:
134
+ frames[:t, :8] = prompt[0][:t]
135
+ frames[:t, 8:] = prompt[1][:t]
136
+ model_tracks = [0, 1]
137
+ else:
138
+ # Single speaker: the clip is track 2, the other track is empty (as in
139
+ # single-speaker pre-training data).
140
+ frames[:, :8] = ABSENT
141
+ frames[:t, 8:] = prompt[0][:t]
142
+ model_tracks = [1]
143
+
144
+ generator = torch.Generator("cuda").manual_seed(int(seed))
145
+ with torch.inference_mode():
146
+ result, written = streaming.generate(
147
+ model, frames, temperature=float(temperature), generator=generator,
148
+ top_k=int(top_k),
149
+ )
150
+ tracks = []
151
+ for j in range(2):
152
+ codes = result[:, 8 * j: 8 * j + 8]
153
+ if bool((codes < 0).all()):
154
+ tracks.append(None)
155
+ else:
156
+ tracks.append(_decode(codes))
157
+ n = max(len(w) for w in tracks if w is not None)
158
+ waves = [np.zeros(n, np.float32) if w is None else np.pad(w, (0, n - len(w))) for w in tracks]
159
+ mix = waves[0] + waves[1]
160
+ mix = mix / max(1.0, float(np.abs(mix).max()) / 0.95)
161
+ own = waves[1] / max(1.0, float(np.abs(waves[1]).max()) / 0.95)
162
+ text = _text(written, model.config.text_tracks) if model.config.text_tracks else \
163
+ "_The audio-only checkpoint writes no text — pick a text-track checkpoint to see its words._"
164
+ print(f"[duplex-0] mode={mode} ckpt={checkpoint} frames={len(frames)} "
165
+ f"time={time.perf_counter() - t0:.1f}s", flush=True)
166
+ return (SR, (mix * 32767).astype(np.int16)), (SR, (own * 32767).astype(np.int16)), text
167
+
168
+
169
+ CSS = """
170
+ #col-container { max-width: 1100px; margin: 0 auto; }
171
+ .dark .gradio-container { color: var(--body-text-color); }
172
+ """
173
+
174
+ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
175
+ with gr.Column(elem_id="col-container"):
176
+ gr.Markdown(
177
+ "# duplex-0\n"
178
+ "A small (201M) full-duplex speech-to-speech research model from Trelis, trained from scratch on "
179
+ "~750 h of English speech over two [Mimi](https://huggingface.co/kyutai/mimi) token tracks, one per speaker. "
180
+ "**Reply**: the model hears your clip as the caller and talks as the other side, in real time. "
181
+ "**Continue**: the clip is a prompt and the model keeps the conversation going. "
182
+ "It's a research checkpoint, so the speech is only about half as intelligible as real speech. "
183
+ "[Model card](https://huggingface.co/Trelis/duplex-0)"
184
+ )
185
+ with gr.Row():
186
+ with gr.Column():
187
+ audio_in = gr.Audio(label="Input speech (≤30 s; stereo = one speaker per channel)",
188
+ type="filepath", sources=["upload", "microphone"])
189
+ mode = gr.Radio(["Reply", "Continue"], value="Reply", label="Mode")
190
+ checkpoint = gr.Radio(list(CHECKPOINTS), value="Audio only", label="Checkpoint")
191
+ seconds = gr.Slider(2, MAX_GEN_S, value=10, step=1,
192
+ label="Seconds to generate after the clip")
193
+ run = gr.Button("Generate", variant="primary")
194
+ with gr.Column():
195
+ mix_out = gr.Audio(label="Both tracks mixed", type="numpy")
196
+ own_out = gr.Audio(label="Model's track (track 2)", type="numpy")
197
+ text_out = gr.Markdown()
198
+ with gr.Accordion("Advanced settings", open=False):
199
+ temperature = gr.Slider(0.1, 1.5, value=1.0, step=0.05, label="Temperature")
200
+ top_k = gr.Slider(1, 2048, value=250, step=1, label="Top-k")
201
+ seed = gr.Number(value=1, precision=0, label="Seed")
202
+
203
+ gr.Examples(
204
+ examples=[
205
+ ["examples/prompt1.wav", "Continue"],
206
+ ["examples/prompt2.wav", "Reply"],
207
+ ["examples/prompt3.wav", "Continue"],
208
+ ["examples/prompt4.wav", "Reply"],
209
+ ],
210
+ inputs=[audio_in, mode],
211
+ outputs=[mix_out, own_out, text_out],
212
+ fn=generate,
213
+ cache_examples=True,
214
+ cache_mode="lazy",
215
+ label="Held-out podcast prompts from the model card (10 s)",
216
+ )
217
+
218
+ run.click(generate, [audio_in, mode, checkpoint, seconds, temperature, top_k, seed],
219
+ [mix_out, own_out, text_out])
220
+
221
+ if __name__ == "__main__":
222
+ demo.launch(mcp_server=True)
duplex/NOTICE ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ The attention/MLP implementation follows architecture and tensor conventions
2
+ from karpathy/nanochat, inspected at 92d63d4e8bb4df75c3b71618f31ddde2378b2bcd.
3
+ Upstream license follows:
4
+
5
+ MIT License
6
+
7
+ Copyright (c) 2025 Andrej Karpathy
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+
27
+ kernels.py adapts the depth attention kernel design (two frames per 16-row
28
+ tile, block-diagonal causal mask, recomputed backward) from naklecha/caramel
29
+ kernels.py, inspected at 84a9271d265841543287d4044d7ef80991a2b003, licensed
30
+ under the Apache License, Version 2.0
31
+ (https://www.apache.org/licenses/LICENSE-2.0). It was rewritten for separate
32
+ query, key and value tensors in [rows, positions, heads, head_width] layout.
duplex/__init__.py ADDED
File without changes
duplex/checkpoint.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Versioned research checkpoints, exact single-device RNG/optimizer resume."""
2
+
3
+ import json
4
+ from dataclasses import asdict
5
+ from pathlib import Path
6
+
7
+ import torch
8
+
9
+ from .network import Config, DuplexModel, factorization, from_legacy
10
+
11
+ # The one- and two-stream format of 14 November, before text tracks.
12
+ LEGACY_FACTORIZATION = "delay-0-1-1-1-1-1-1-1"
13
+
14
+
15
+ def save(path, model, optimizer, step, provenance, sampler):
16
+ path = Path(path)
17
+ path.parent.mkdir(parents=True, exist_ok=True)
18
+ state = {
19
+ "schema_version": 3,
20
+ "factorization": factorization(model.config),
21
+ "config": asdict(model.config),
22
+ "model": model.state_dict(),
23
+ "optimizer": optimizer.state_dict(),
24
+ "step": step,
25
+ "provenance": provenance,
26
+ "torch_rng": torch.get_rng_state(),
27
+ "sampler_rng": sampler.get_state(),
28
+ "cuda_rng": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [],
29
+ }
30
+ temporary = path.with_suffix(path.suffix + ".tmp")
31
+ torch.save(state, temporary)
32
+ temporary.replace(path)
33
+
34
+
35
+ def model_from(config, weights, stored):
36
+ """A model from a stored config and weights; 14 November checkpoints convert."""
37
+ if stored == LEGACY_FACTORIZATION:
38
+ return from_legacy(config, weights)
39
+ # Checkpoints before 17 November 2026 have no shift or head fields and load
40
+ # with the defaults they were trained under.
41
+ config = Config(**config)
42
+ if stored != factorization(config):
43
+ raise ValueError("Unsupported codebook factorization")
44
+ model = DuplexModel(config)
45
+ model.load_state_dict(weights, strict=True)
46
+ return model
47
+
48
+
49
+ def load(path, device="cpu"):
50
+ # weights_only rejects arbitrary pickled classes; only tensors/basic containers are used.
51
+ state = torch.load(path, map_location="cpu", weights_only=True)
52
+ if state["schema_version"] not in (2, 3):
53
+ raise ValueError("Unsupported checkpoint schema")
54
+ model = model_from(state["config"], state["model"], state["factorization"])
55
+ return model.to(device), state
56
+
57
+
58
+ def load_published(folder, device="cpu"):
59
+ """A checkpoint folder written by `publish.py` (config.json and model.safetensors)."""
60
+ from safetensors.torch import load_file
61
+
62
+ folder = Path(folder)
63
+ config = json.loads((folder / "config.json").read_text())
64
+ stored = config.pop("factorization")
65
+ weights = load_file(folder / "model.safetensors")
66
+ return model_from(config, weights, stored).to(device).eval()
67
+
68
+
69
+ def resume(state, optimizer, sampler):
70
+ if state["schema_version"] != 3:
71
+ raise ValueError("Resume needs a checkpoint of this trainer")
72
+ optimizer.load_state_dict(state["optimizer"])
73
+ torch.set_rng_state(state["torch_rng"])
74
+ sampler.set_state(state["sampler_rng"])
75
+ if state["cuda_rng"]:
76
+ if not torch.cuda.is_available():
77
+ raise ValueError("Exact CUDA resume requires CUDA")
78
+ torch.cuda.set_rng_state_all(state["cuda_rng"])
duplex/kernels.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Depth attention over the codebooks of one frame, specialised for short rows.
2
+
3
+ General attention kernels spend most of their tiles on padding when the sequence
4
+ is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one
5
+ head into a 16-row tile (two frames of one stream, or one frame of two streams)
6
+ with a block-diagonal causal mask, the approach of Caramel
7
+ (github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch
8
+ reference, which defines the math the kernel must match.
9
+ """
10
+
11
+ import math
12
+
13
+ import torch
14
+ from torch.nn import functional as F
15
+
16
+ try:
17
+ import triton
18
+ import triton.language as tl
19
+ except ImportError:
20
+ triton = None
21
+
22
+ ROWS_PER_TILE = 16
23
+
24
+
25
+ def reference(q, k, v):
26
+ """Causal attention for [rows, positions, heads, head_width] inputs."""
27
+ q, k, v = (y.transpose(1, 2) for y in (q, k, v))
28
+ return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2)
29
+
30
+
31
+ if triton is not None:
32
+
33
+ @triton.jit
34
+ def _probabilities(
35
+ Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
36
+ ):
37
+ rows: tl.constexpr = FPT * POS
38
+ pid = tl.program_id(0).to(tl.int64)
39
+ r, d = tl.arange(0, rows), tl.arange(0, HD)
40
+ n, position = (pid // H) * FPT + r // POS, r % POS
41
+ valid = (n < N)[:, None]
42
+ offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :]
43
+ q = tl.load(Q + offsets, mask=valid, other=0.0)
44
+ k = tl.load(K + offsets, mask=valid, other=0.0)
45
+ s = tl.dot(q, tl.trans(k)) * scale
46
+ same_frame = (r[:, None] // POS) == (r[None, :] // POS)
47
+ causal = position[:, None] >= position[None, :]
48
+ s = tl.where(same_frame & causal, s, float("-inf"))
49
+ p = tl.exp(s - tl.max(s, axis=1)[:, None])
50
+ return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid
51
+
52
+ @triton.jit
53
+ def _forward(
54
+ Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
55
+ ):
56
+ _, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
57
+ v = tl.load(V + offsets, mask=valid, other=0.0)
58
+ o = tl.dot(p, v.to(tl.float32), input_precision="ieee")
59
+ tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid)
60
+
61
+ @triton.jit
62
+ def _backward(
63
+ Q,
64
+ K,
65
+ V,
66
+ DO,
67
+ DQ,
68
+ DK,
69
+ DV,
70
+ N,
71
+ H,
72
+ scale,
73
+ HD: tl.constexpr,
74
+ POS: tl.constexpr,
75
+ FPT: tl.constexpr,
76
+ ):
77
+ # Recomputes the small probability tile, so forward saves only its inputs.
78
+ q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
79
+ v = tl.load(V + offsets, mask=valid, other=0.0)
80
+ do = tl.load(DO + offsets, mask=valid, other=0.0)
81
+ dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee")
82
+ dp = tl.dot(do, tl.trans(v))
83
+ ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale
84
+ dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee")
85
+ dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee")
86
+ ty = DQ.dtype.element_ty
87
+ tl.store(DQ + offsets, dq.to(ty), mask=valid)
88
+ tl.store(DK + offsets, dk.to(ty), mask=valid)
89
+ tl.store(DV + offsets, dv.to(ty), mask=valid)
90
+
91
+ def _launch(kernel, q, *tensors):
92
+ n, positions, heads, head_width = q.shape
93
+ frames = ROWS_PER_TILE // positions
94
+ kernel[(triton.cdiv(n, frames) * heads,)](
95
+ q,
96
+ *tensors,
97
+ n,
98
+ heads,
99
+ 1.0 / math.sqrt(head_width),
100
+ HD=head_width,
101
+ POS=positions,
102
+ FPT=frames,
103
+ num_warps=1,
104
+ )
105
+
106
+ # Custom ops make each kernel one opaque node for torch.compile.
107
+ @torch.library.custom_op("duplex::depth_attention", mutates_args=())
108
+ def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
109
+ q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
110
+ o = torch.empty_like(q)
111
+ _launch(_forward, q, k, v, o)
112
+ return o
113
+
114
+ @torch.library.custom_op("duplex::depth_attention_backward", mutates_args=())
115
+ def _attention_backward(
116
+ q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor
117
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
118
+ q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous()
119
+ dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q)
120
+ _launch(_backward, q, k, v, do, dq, dk, dv)
121
+ return dq, dk, dv
122
+
123
+ _attention.register_fake(lambda q, k, v: torch.empty_like(q))
124
+ _attention_backward.register_fake(
125
+ lambda q, k, v, do: (
126
+ torch.empty_like(q),
127
+ torch.empty_like(q),
128
+ torch.empty_like(q),
129
+ )
130
+ )
131
+ _attention.register_autograd(
132
+ lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad),
133
+ setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs),
134
+ )
135
+
136
+
137
+ def depth_attention(q, k, v):
138
+ """q, k and v are [rows, positions, heads, head_width]; so is the result."""
139
+ if (
140
+ triton is None
141
+ or not q.is_cuda
142
+ or q.dtype == torch.float32
143
+ or ROWS_PER_TILE % q.shape[1]
144
+ ):
145
+ return reference(q, k, v)
146
+ return _attention(q, k, v)
147
+
148
+
149
+ if __name__ == "__main__":
150
+ # GPU check: the packed kernel against the reference for one and two streams.
151
+ for positions in (8, 16):
152
+ q, k, v = (
153
+ torch.randn(
154
+ 4096,
155
+ positions,
156
+ 4,
157
+ 64,
158
+ device="cuda",
159
+ dtype=torch.bfloat16,
160
+ requires_grad=True,
161
+ )
162
+ for _ in range(3)
163
+ )
164
+ grad = torch.randn_like(q)
165
+ results = []
166
+ for function in (depth_attention, reference):
167
+ out = function(q, k, v)
168
+ out.backward(grad)
169
+ results.append([out.float(), *(x.grad.float() for x in (q, k, v))])
170
+ for x in (q, k, v):
171
+ x.grad = None
172
+ errors = [
173
+ ((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True)
174
+ ]
175
+ print({"positions": positions, "relative_errors": errors}, flush=True)
176
+ if max(errors) > 0.01:
177
+ raise SystemExit("kernel differs from reference")
duplex/network.py ADDED
@@ -0,0 +1,638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Temporal/depth autoregression over two audio tracks and two text tracks.
2
+
3
+ A frame holds `codebooks` audio codes for each of the two tracks, track 1 then
4
+ track 2. In every track the acoustic codebooks run one frame behind codebook 0
5
+ inside the model. Callers pass and receive canonical undelayed frames.
6
+
7
+ Text runs at a variable rate (training readiness owns the design). Sequence step s
8
+ is one temporal position for the audio, followed by the text tokens of frame s of
9
+ each present text track, track 1 first. A track's text ends with the advance token,
10
+ which is only ever a target: each text event is predicted from the last input before
11
+ it, and the step's last position conditions the depth transformer.
12
+ """
13
+
14
+ from dataclasses import dataclass, replace
15
+
16
+ import numpy as np
17
+ import torch
18
+ from torch import nn
19
+ from torch.nn import functional as F
20
+
21
+ from .kernels import depth_attention
22
+
23
+ TRACKS = 2
24
+ # Moshi's SentencePiece ids are 0..31999; the advance token follows them.
25
+ TEXT_IDS = 32000
26
+ ADVANCE = TEXT_IDS
27
+ # Sampling forces the advance after this many tokens of one track in one frame.
28
+ MAX_TEXT_PER_FRAME = 16
29
+ # Moshi's equation 7: codebook 0 of each track against 1 for each acoustic codebook,
30
+ # and its padding weight for the advance token.
31
+ SEMANTIC_WEIGHT = 100.0
32
+ ADVANCE_WEIGHT = 0.5
33
+ TEXT_OPTIONS = {"none": (), "one": (1,), "two": (0, 1)}
34
+ # Text against audio per track, in 80 ms frames: a word starting at frame f is
35
+ # written on track p at frame f - TEXT_SHIFT[p]. Positive puts the text ahead of the
36
+ # audio, negative behind it, 0 at the word's start. Track 1 (listened to) is 480 ms
37
+ # behind, a transcript; track 2 (spoken as) writes words where they start. The shift
38
+ # follows the track after the speaker swap; windows apply it, generation follows it.
39
+ # The default of `Config.text_shift`, which each checkpoint records.
40
+ TEXT_SHIFT = (-6, 0)
41
+ # Text output heads: one shared by both tracks, or one per track.
42
+ TEXT_HEADS = ("shared", "per-track")
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class Config:
47
+ layers: int = 12
48
+ width: int = 768
49
+ heads: int = 6
50
+ depth_layers: int = 2
51
+ depth_width: int = 256
52
+ depth_heads: int = 4
53
+ # Temporal positions per window: frames plus text tokens, plus one flush step.
54
+ context: int = 4096
55
+ codebooks: int = 8
56
+ cardinality: int = 2048
57
+ fps: float = 12.5
58
+ # The training option; the parameters are the same for all three.
59
+ text: str = "none"
60
+ # Frames each track's text sits from its word start (see TEXT_SHIFT).
61
+ text_shift: tuple = TEXT_SHIFT
62
+ text_heads: str = "shared"
63
+
64
+ @property
65
+ def total_codebooks(self):
66
+ return self.codebooks * TRACKS
67
+
68
+ @property
69
+ def text_tracks(self):
70
+ return TEXT_OPTIONS[self.text]
71
+
72
+ def __post_init__(self):
73
+ for width, heads in (
74
+ (self.width, self.heads),
75
+ (self.depth_width, self.depth_heads),
76
+ ):
77
+ if heads < 1 or width % heads or (width // heads) % 2:
78
+ raise ValueError("Head dimensions must be even")
79
+ if (
80
+ min(
81
+ self.layers,
82
+ self.depth_layers,
83
+ self.context,
84
+ self.codebooks,
85
+ self.cardinality,
86
+ )
87
+ < 1
88
+ ):
89
+ raise ValueError("Dimensions must be positive")
90
+ if self.fps <= 0:
91
+ raise ValueError("fps must be positive")
92
+ if self.text not in TEXT_OPTIONS:
93
+ raise ValueError(f"text must be one of {sorted(TEXT_OPTIONS)}")
94
+ # A config read from JSON holds a list.
95
+ object.__setattr__(self, "text_shift", tuple(int(x) for x in self.text_shift))
96
+ if len(self.text_shift) != TRACKS:
97
+ raise ValueError("text_shift has one value per track")
98
+ if self.text_heads not in TEXT_HEADS:
99
+ raise ValueError(f"text_heads must be one of {TEXT_HEADS}")
100
+
101
+
102
+ # Sequence order of the codebooks and text. Checkpoints carry it; a model trained
103
+ # under one factorization is meaningless under another.
104
+ def factorization(config):
105
+ return "two-tracks-delay-0-1-1-1-1-1-1-1-text-advance-shift" + "".join(
106
+ f"{shift:+d}" for shift in config.text_shift
107
+ )
108
+
109
+
110
+ FACTORIZATION = factorization(Config())
111
+
112
+
113
+ def tiny_config(text="none"):
114
+ # Real codec vocabulary, small transformers. No fake-codec checkpoint confusion.
115
+ return Config(
116
+ layers=2,
117
+ width=32,
118
+ heads=2,
119
+ depth_layers=1,
120
+ depth_width=32,
121
+ depth_heads=2,
122
+ context=64,
123
+ text=text,
124
+ )
125
+
126
+
127
+ def norm(x):
128
+ return F.rms_norm(x, (x.shape[-1],))
129
+
130
+
131
+ def rotary(positions, head_width):
132
+ """Cos and sin tables [positions, head_width / 2], built once per model."""
133
+ freq = torch.arange(0, head_width, 2, dtype=torch.float32) / head_width
134
+ angles = torch.outer(torch.arange(positions), 100000.0**-freq)
135
+ return torch.stack((angles.cos(), angles.sin()))
136
+
137
+
138
+ def rotate(y, cos, sin):
139
+ a, b = y.chunk(2, dim=-1)
140
+ return torch.cat((a * cos + b * sin, -a * sin + b * cos), dim=-1)
141
+
142
+
143
+ class Attention(nn.Module):
144
+ def __init__(self, width, heads):
145
+ super().__init__()
146
+ self.heads = heads
147
+ self.c_q = nn.Linear(width, width, bias=False)
148
+ self.c_k = nn.Linear(width, width, bias=False)
149
+ self.c_v = nn.Linear(width, width, bias=False)
150
+ self.c_proj = nn.Linear(width, width, bias=False)
151
+
152
+ def forward(self, x, rope):
153
+ b, t, w = x.shape
154
+ d = w // self.heads
155
+ q, k, v = [
156
+ layer(x).reshape(b, t, self.heads, d).transpose(1, 2)
157
+ for layer in (self.c_q, self.c_k, self.c_v)
158
+ ]
159
+ cos, sin = rope[0, :t].to(x.dtype), rope[1, :t].to(x.dtype)
160
+ q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
161
+ y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
162
+ return self.c_proj(y.transpose(1, 2).reshape(x.shape))
163
+
164
+
165
+ class MLP(nn.Module):
166
+ def __init__(self, width):
167
+ super().__init__()
168
+ self.c_fc = nn.Linear(width, 4 * width, bias=False)
169
+ self.c_proj = nn.Linear(4 * width, width, bias=False)
170
+
171
+ def forward(self, x):
172
+ return self.c_proj(F.relu(self.c_fc(x)).square())
173
+
174
+
175
+ class Block(nn.Module):
176
+ def __init__(self, width, heads):
177
+ super().__init__()
178
+ self.attn = Attention(width, heads)
179
+ self.mlp = MLP(width)
180
+
181
+ def forward(self, x, rope):
182
+ x = x + self.attn(norm(x), rope)
183
+ return x + self.mlp(norm(x))
184
+
185
+
186
+ def per_position(x, weight):
187
+ """x [rows, positions, in] through weight [positions, out, in]: one matrix per position."""
188
+ y = torch.bmm(x.transpose(0, 1), weight.transpose(1, 2).to(x.dtype))
189
+ return y.transpose(0, 1)
190
+
191
+
192
+ class DepthBlock(nn.Module):
193
+ """A depth block with its own weights at every codebook position.
194
+
195
+ Same widths and arithmetic as `Block`; each [positions, out, in] matrix holds
196
+ one nn.Linear weight per position, initialised as nn.Linear initialises.
197
+ """
198
+
199
+ def __init__(self, positions, width, heads):
200
+ super().__init__()
201
+ self.heads = heads
202
+
203
+ def weight(out, inner):
204
+ bound = inner**-0.5
205
+ return nn.Parameter(
206
+ torch.empty(positions, out, inner).uniform_(-bound, bound)
207
+ )
208
+
209
+ self.c_q, self.c_k, self.c_v, self.c_proj = (
210
+ weight(width, width) for _ in range(4)
211
+ )
212
+ self.c_fc = weight(4 * width, width)
213
+ self.c_out = weight(width, 4 * width)
214
+
215
+ def forward(self, x, rope):
216
+ n, p, w = x.shape
217
+ d = w // self.heads
218
+ h = norm(x)
219
+ q, k, v = [
220
+ per_position(h, m).reshape(n, p, self.heads, d)
221
+ for m in (self.c_q, self.c_k, self.c_v)
222
+ ]
223
+ cos = rope[0, :p].to(x.dtype).unsqueeze(1)
224
+ sin = rope[1, :p].to(x.dtype).unsqueeze(1)
225
+ q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
226
+ # Rotation and QK norm run at the residual precision; the kernel takes
227
+ # the projection precision.
228
+ y = depth_attention(q.to(v.dtype), k.to(v.dtype), v)
229
+ x = x + per_position(y.reshape(n, p, w), self.c_proj)
230
+ h = F.relu(per_position(norm(x), self.c_fc)).square()
231
+ return x + per_position(h, self.c_out)
232
+
233
+
234
+ @dataclass
235
+ class TextLayout:
236
+ """Where text sits in a padded batch, built by `text_layout` and `collate_text`.
237
+
238
+ source [batch, positions] indexes the concatenation of the audio steps
239
+ [frames + 1], the text tokens [tokens] and one zero row for padding.
240
+ token and track [batch, tokens] are the text tokens' ids and tracks.
241
+ last [batch, frames + 1] is the position whose output conditions each step's depth.
242
+ event_* [batch, events] are the text predictions: the position predicting it, its
243
+ track, its target id and its loss weight (0 for padding and the flush step).
244
+ pattern [batch] is the bitmask of present text tracks.
245
+ """
246
+
247
+ source: torch.Tensor
248
+ token: torch.Tensor
249
+ track: torch.Tensor
250
+ last: torch.Tensor
251
+ event_position: torch.Tensor
252
+ event_track: torch.Tensor
253
+ event_target: torch.Tensor
254
+ event_weight: torch.Tensor
255
+ pattern: torch.Tensor
256
+
257
+
258
+ def text_layout(frames, text):
259
+ """One window's text layout as numpy arrays; text is per track None (absent) or an
260
+ int array [n, 2] of (frame, id) rows in word order, frames in [0, frames)."""
261
+ present = [j for j in range(TRACKS) if text[j] is not None]
262
+ counts = np.zeros((frames + 1, TRACKS), dtype=np.int64)
263
+ parts = []
264
+ for j in present:
265
+ rows = np.asarray(text[j], dtype=np.int64).reshape(-1, 2)
266
+ if len(rows) and (rows[:, 0].min() < 0 or rows[:, 0].max() >= frames):
267
+ raise ValueError("Text frame outside the window")
268
+ if len(rows) and (rows[:, 1].min() < 0 or rows[:, 1].max() >= TEXT_IDS):
269
+ raise ValueError("Text id outside the vocabulary")
270
+ np.add.at(counts[:, j], rows[:, 0], 1)
271
+ parts.append(
272
+ np.column_stack(
273
+ (rows[:, 0], np.full(len(rows), j), np.arange(len(rows)), rows[:, 1])
274
+ )
275
+ )
276
+ tokens = np.concatenate(parts) if parts else np.zeros((0, 4), dtype=np.int64)
277
+ # Frame, then track, then word order.
278
+ tokens = tokens[np.lexsort((tokens[:, 2], tokens[:, 1], tokens[:, 0]))]
279
+ per_step = counts.sum(1)
280
+ start = np.concatenate(([0], np.cumsum(1 + per_step)[:-1]))
281
+ before = np.concatenate(([0], np.cumsum(per_step)))
282
+ positions = int(frames + 1 + per_step.sum())
283
+ rank = np.arange(len(tokens)) - before[tokens[:, 0]]
284
+ token_position = start[tokens[:, 0]] + 1 + rank
285
+ source = np.arange(frames + 1 + len(tokens))
286
+ order = np.empty(positions, dtype=np.int64)
287
+ order[start] = np.arange(frames + 1)
288
+ order[token_position] = source[frames + 1 :]
289
+ # Events: each token from the input before it; each advance from the last input
290
+ # of its track in the step, or where the track's first token would come from.
291
+ position = [token_position - 1]
292
+ track = [tokens[:, 1]]
293
+ target = [tokens[:, 3]]
294
+ weight = [np.ones(len(tokens))]
295
+ written = np.zeros(frames + 1, dtype=np.int64)
296
+ for j in present:
297
+ written = written + counts[:, j]
298
+ position.append(start + written)
299
+ track.append(np.full(frames + 1, j))
300
+ target.append(np.full(frames + 1, ADVANCE))
301
+ weight.append(np.r_[np.full(frames, ADVANCE_WEIGHT), 0.0])
302
+ position, track, target, weight = (
303
+ np.concatenate(x) for x in (position, track, target, weight)
304
+ )
305
+ # Sampling order: by position, then track.
306
+ events = np.lexsort((track, position))
307
+ return {
308
+ "source": order,
309
+ "token": tokens[:, 3],
310
+ "track": tokens[:, 1],
311
+ "last": start + per_step,
312
+ "event_position": position[events],
313
+ "event_track": track[events],
314
+ "event_target": target[events],
315
+ "event_weight": weight[events].astype(np.float32),
316
+ "pattern": sum(1 << j for j in present),
317
+ }
318
+
319
+
320
+ def collate_text(layouts, frames, positions, device):
321
+ """Pads per-window layouts to [batch, positions] and [batch, frames + 1]."""
322
+ b = len(layouts)
323
+ tokens = max(1, max(len(x["token"]) for x in layouts))
324
+ events = max(1, max(len(x["event_target"]) for x in layouts))
325
+ pad = frames + 1 + tokens
326
+ source = torch.full((b, positions), pad, dtype=torch.long)
327
+ token = torch.zeros((b, tokens), dtype=torch.long)
328
+ track = torch.zeros((b, tokens), dtype=torch.long)
329
+ last = torch.zeros((b, frames + 1), dtype=torch.long)
330
+ event = [torch.zeros((b, events), dtype=torch.long) for _ in range(3)]
331
+ event_weight = torch.zeros((b, events))
332
+ for i, x in enumerate(layouts):
333
+ steps, n, m = len(x["last"]), len(x["token"]), len(x["event_target"])
334
+ if len(x["source"]) > positions or steps > frames + 1:
335
+ raise ValueError("Window exceeds the padded batch")
336
+ s = torch.as_tensor(x["source"])
337
+ # Token sources are rebased past this batch's padded audio steps.
338
+ source[i, : len(s)] = torch.where(s < steps, s, s - steps + frames + 1)
339
+ token[i, :n] = torch.as_tensor(x["token"])
340
+ track[i, :n] = torch.as_tensor(x["track"])
341
+ last[i, :steps] = torch.as_tensor(x["last"])
342
+ for tensor, key in zip(
343
+ event, ("event_position", "event_track", "event_target"), strict=True
344
+ ):
345
+ tensor[i, :m] = torch.as_tensor(x[key])
346
+ event_weight[i, :m] = torch.as_tensor(x["event_weight"])
347
+ pattern = torch.tensor([x["pattern"] for x in layouts])
348
+ return TextLayout(
349
+ source.to(device),
350
+ token.to(device),
351
+ track.to(device),
352
+ last.to(device),
353
+ *(t.to(device) for t in event),
354
+ event_weight.to(device),
355
+ pattern.to(device),
356
+ )
357
+
358
+
359
+ class DuplexModel(nn.Module):
360
+ def __init__(self, config: Config):
361
+ super().__init__()
362
+ self.config = config
363
+ c = config
364
+ q = c.total_codebooks
365
+ # One table per role with codebook offsets: a single gather instead of
366
+ # one per codebook. Each track has its own rows.
367
+ self.frame_embeddings = nn.EmbeddingBag(q * c.cardinality, c.width, mode="sum")
368
+ # A zero BOS stays zero through bias-free blocks, amplifying RMSNorm's
369
+ # derivative at zero until deep models overflow on the first backward.
370
+ self.bos = nn.Parameter(torch.randn(c.width))
371
+ # Text: one token table for both tracks, a vector per track, an embedding of
372
+ # which text tracks are present added to every audio position (patterns 1 to
373
+ # 3; none adds nothing), and a query per track before the shared head.
374
+ self.text_embeddings = nn.Embedding(TEXT_IDS, c.width)
375
+ self.text_tracks = nn.Parameter(torch.zeros(TRACKS, c.width))
376
+ self.text_presence = nn.Parameter(torch.zeros(2**TRACKS - 1, c.width))
377
+ self.text_queries = nn.Parameter(torch.zeros(TRACKS, c.width))
378
+ if c.text_heads == "shared":
379
+ self.text_head = nn.Linear(c.width, TEXT_IDS + 1, bias=False)
380
+ else:
381
+ # [track, events, width], initialised as nn.Linear initialises one head.
382
+ bound = c.width**-0.5
383
+ self.text_head = nn.Parameter(
384
+ torch.empty(TRACKS, TEXT_IDS + 1, c.width).uniform_(-bound, bound)
385
+ )
386
+ self.temporal = nn.ModuleList(
387
+ [Block(c.width, c.heads) for _ in range(c.layers)]
388
+ )
389
+ self.to_depth = nn.Linear(c.width, c.depth_width, bias=False)
390
+ self.depth_embeddings = nn.Embedding((q - 1) * c.cardinality, c.depth_width)
391
+ self.depth_positions = nn.Parameter(torch.zeros(q, c.depth_width))
392
+ self.depth = nn.ModuleList(
393
+ [DepthBlock(q, c.depth_width, c.depth_heads) for _ in range(c.depth_layers)]
394
+ )
395
+ # Per-codebook output heads [codebook, cardinality, depth_width].
396
+ bound = c.depth_width**-0.5
397
+ self.outputs = nn.Parameter(
398
+ torch.empty(q, c.cardinality, c.depth_width).uniform_(-bound, bound)
399
+ )
400
+ self.register_buffer(
401
+ "offsets", torch.arange(q) * c.cardinality, persistent=False
402
+ )
403
+ # Codebook 0 of each track has delay zero; its acoustic codebooks one.
404
+ self.delays = [int(k % c.codebooks != 0) for k in range(q)]
405
+ self.register_buffer(
406
+ "semantic",
407
+ torch.tensor([d == 0 for d in self.delays]),
408
+ persistent=False,
409
+ )
410
+ self.register_buffer(
411
+ "track_of", torch.arange(q) // c.codebooks, persistent=False
412
+ )
413
+ self.register_buffer(
414
+ "temporal_rope",
415
+ # One position more than the context flushes the last acoustic frame.
416
+ rotary(c.context + 1, c.width // c.heads),
417
+ persistent=False,
418
+ )
419
+ self.register_buffer(
420
+ "depth_rope",
421
+ rotary(q, c.depth_width // c.depth_heads),
422
+ persistent=False,
423
+ )
424
+
425
+ def delayed(self, tokens, valid=None):
426
+ """Token ids and presence for sequence steps 0..frames, [batch, frames + 1, codebooks].
427
+
428
+ Step s holds codebook 0 of frame s and the acoustic codebooks of frame
429
+ s - 1 in every track. valid is [batch, frames, tracks]: a token is absent
430
+ before the first frame, after the last, on padded frames and on absent tracks.
431
+ """
432
+ b, t, _ = tokens.shape
433
+ if valid is None:
434
+ valid = tokens.new_ones((b, t, TRACKS), dtype=torch.bool)
435
+ valid = valid[..., self.track_of]
436
+ ids = torch.where(
437
+ self.semantic, F.pad(tokens, (0, 0, 0, 1)), F.pad(tokens, (0, 0, 1, 0))
438
+ )
439
+ present = torch.where(
440
+ self.semantic,
441
+ F.pad(valid, (0, 0, 0, 1)),
442
+ F.pad(valid, (0, 0, 1, 0)),
443
+ )
444
+ return ids, present
445
+
446
+ def audio_steps(self, tokens, valid=None):
447
+ """Temporal inputs of the audio positions, [batch, frames + 1, width]: BOS, then
448
+ the sum of every present token of the step before."""
449
+ b, t, q = tokens.shape
450
+ if q != self.config.total_codebooks:
451
+ raise ValueError("Expected [batch, frames, codebooks x tracks]")
452
+ x = self.bos.expand(b, 1, -1)
453
+ if t:
454
+ ids, present = self.delayed(tokens, valid)
455
+ weights = self.frame_embeddings.weight
456
+ embedded = self.frame_embeddings(
457
+ (ids[:, :t] + self.offsets).reshape(-1, q),
458
+ per_sample_weights=present[:, :t].reshape(-1, q).to(weights.dtype),
459
+ )
460
+ x = torch.cat((x, embedded.reshape(b, t, -1)), dim=1)
461
+ return x
462
+
463
+ def temporal_outputs(self, tokens, valid=None, text=None):
464
+ """Normalised temporal outputs [batch, positions, width]."""
465
+ x = self.audio_steps(tokens, valid)
466
+ if text is not None:
467
+ b, _, w = x.shape
468
+ presence = torch.cat((x.new_zeros(1, w), self.text_presence))
469
+ x = x + presence[text.pattern][:, None]
470
+ words = self.text_embeddings(text.token) + self.text_tracks[text.track]
471
+ table = torch.cat((x, words.to(x.dtype), x.new_zeros(b, 1, w)), dim=1)
472
+ x = table.gather(1, text.source[..., None].expand(-1, -1, w))
473
+ if x.shape[1] > self.config.context + 1:
474
+ raise ValueError("Window exceeds the temporal context")
475
+ x = norm(x)
476
+ for block in self.temporal:
477
+ x = block(x, self.temporal_rope)
478
+ return norm(x)
479
+
480
+ def text_logits(self, hidden, track):
481
+ """Logits [..., TEXT_IDS + 1] of text events from normalised temporal outputs."""
482
+ x = norm(hidden + self.text_queries[track].to(hidden.dtype))
483
+ if self.config.text_heads == "shared":
484
+ return self.text_head(x)
485
+ weights = self.text_head.to(x.dtype)
486
+ track = torch.as_tensor(track, device=x.device)
487
+ if track.dim() == 0:
488
+ return x @ weights[track].T
489
+ # Both heads for every event, each event keeping its own track's: fixed
490
+ # shapes, so the compiled step stays fast (mask selection has
491
+ # data-dependent shapes and trained at about half the step rate).
492
+ return torch.where(track[..., None] == 0, x @ weights[0].T, x @ weights[1].T)
493
+
494
+ def depth_logits(self, context, inputs):
495
+ """Logits [batch, steps, codebooks, cardinality] in delayed step order.
496
+
497
+ inputs is [batch, steps, codebooks - 1, depth_width]: the embedding of
498
+ the token before each depth position, zero where there is none.
499
+ """
500
+ b, s, _ = context.shape
501
+ q = self.config.total_codebooks
502
+ x = context.unsqueeze(2) + self.depth_positions
503
+ x = torch.cat((x[:, :, :1], x[:, :, 1:] + inputs), dim=2)
504
+ x = x.reshape(b * s, q, -1)
505
+ for block in self.depth:
506
+ x = block(x, self.depth_rope)
507
+ # [codebook, steps, width] x [codebook, width, cardinality]
508
+ x = norm(x).transpose(0, 1)
509
+ logits = torch.bmm(x, self.outputs.transpose(1, 2).to(x.dtype))
510
+ return logits.transpose(0, 1).reshape(b, s, q, -1)
511
+
512
+ def forward(self, tokens, valid=None, text=None):
513
+ """Canonical frames in; audio logits aligned to the same [batch, frames,
514
+ codebooks], and text logits [batch, events, TEXT_IDS + 1] or None.
515
+
516
+ valid is the [batch, frames, tracks] mask of real frames of present tracks.
517
+ text is a `TextLayout` or None for audio only.
518
+
519
+ Fillers never appear as targets: codebook 0 of frame t is read from step
520
+ t and its acoustic codebooks from step t + 1, so the final step flushes
521
+ the last frame and every source token is scored once.
522
+ """
523
+ hidden = self.temporal_outputs(tokens, valid, text)
524
+ steps = hidden
525
+ if text is not None:
526
+ w = hidden.shape[-1]
527
+ steps = hidden.gather(1, text.last[..., None].expand(-1, -1, w))
528
+ context = self.to_depth(steps)
529
+ ids, present = self.delayed(tokens, valid)
530
+ # Each depth position after the first takes the token before it in
531
+ # step order. The step that flushes a short row's last frame sees no
532
+ # codebook 0, as in generation, instead of batch padding.
533
+ inputs = self.depth_embeddings(ids[:, :, :-1] + self.offsets[:-1])
534
+ inputs = inputs * present[:, :, :-1, None].to(inputs.dtype)
535
+ logits = self.depth_logits(context, inputs)
536
+ audio = torch.where(self.semantic[:, None], logits[:, :-1], logits[:, 1:])
537
+ if text is None:
538
+ return audio, None
539
+ w = hidden.shape[-1]
540
+ events = hidden.gather(1, text.event_position[..., None].expand(-1, -1, w))
541
+ return audio, self.text_logits(events, text.event_track)
542
+
543
+ @torch.no_grad()
544
+ def generate(self, prefix, frames, temperature=1.0, top_k=250):
545
+ """Reference audio sampler without KV cache or text; returns canonical tokens."""
546
+ if frames < 0 or temperature < 0 or top_k < 0:
547
+ raise ValueError("frames, temperature and top_k must be nonnegative")
548
+ c = self.config
549
+ b, forced, q = prefix.shape
550
+ result = torch.cat((prefix, prefix.new_zeros((b, frames, q))), dim=1)
551
+ if not frames:
552
+ return result
553
+ # One step past the last frame flushes its acoustic codebooks.
554
+ for step in range(forced, forced + frames + 1):
555
+ window = result[:, max(0, step - c.context) : step]
556
+ context = self.to_depth(self.temporal_outputs(window)[:, -1:])
557
+ inputs = context.new_zeros((b, 1, q - 1, c.depth_width))
558
+ for position, delay in enumerate(self.delays):
559
+ frame = step - delay
560
+ if not 0 <= frame < forced + frames:
561
+ continue
562
+ # The prefix keeps its own tokens, including the acoustic
563
+ # codebooks of its last frame; later frames are sampled.
564
+ if frame >= forced:
565
+ logits = self.depth_logits(context, inputs)[:, 0, position].float()
566
+ result[:, frame, position] = sample(
567
+ logits, temperature, top_k
568
+ ).squeeze(-1)
569
+ if position < q - 1:
570
+ inputs[:, 0, position] = self.depth_embeddings(
571
+ result[:, frame, position] + self.offsets[position]
572
+ )
573
+ return result
574
+
575
+
576
+ def sample(logits, temperature, top_k=0, generator=None):
577
+ """One draw per row of [..., classes] logits; top_k 0 keeps every class."""
578
+ if temperature == 0:
579
+ return logits.argmax(-1, keepdim=True)
580
+ if top_k and top_k < logits.shape[-1]:
581
+ floor = logits.topk(top_k).values[..., -1:]
582
+ logits = logits.masked_fill(logits < floor, float("-inf"))
583
+ probabilities = (logits / temperature).softmax(-1)
584
+ return torch.multinomial(
585
+ probabilities.reshape(-1, logits.shape[-1]), 1, generator=generator
586
+ ).reshape(*logits.shape[:-1], 1)
587
+
588
+
589
+ def from_legacy(config, state):
590
+ """A model in this format from a 14 November checkpoint (one or two streams,
591
+ shared depth weights, no text).
592
+
593
+ A two-stream checkpoint converts exactly: the shared depth block weights are
594
+ copied to every position and the new text weights stay unused while no text
595
+ is present. A one-stream checkpoint is widened as two-stream training began:
596
+ each track gets a copy of the stream's embeddings, positions and heads, and
597
+ track 1's codebook 7 feeding track 2's codebook 0 starts at zero.
598
+ """
599
+ config = dict(config)
600
+ config.pop("factorization", None)
601
+ streams = config.pop("streams", 1)
602
+ if streams not in (1, 2):
603
+ raise ValueError("Expected a one- or two-stream checkpoint")
604
+ model = DuplexModel(Config(**config))
605
+ c = model.config
606
+ new = model.state_dict()
607
+ state = dict(state)
608
+ if streams == 1:
609
+ for name in ("frame_embeddings.weight", "depth_positions", "outputs"):
610
+ value = state[name]
611
+ state[name] = value.repeat(TRACKS, *[1] * (value.dim() - 1))
612
+ value = state["depth_embeddings.weight"]
613
+ joint = value.new_zeros((c.cardinality, value.shape[1]))
614
+ state["depth_embeddings.weight"] = torch.cat([value, joint, value])
615
+ names = {
616
+ "attn.c_q": "c_q",
617
+ "attn.c_k": "c_k",
618
+ "attn.c_v": "c_v",
619
+ "attn.c_proj": "c_proj",
620
+ "mlp.c_fc": "c_fc",
621
+ "mlp.c_proj": "c_out",
622
+ }
623
+ for layer in range(c.depth_layers):
624
+ for old, name in names.items():
625
+ weight = state.pop(f"depth.{layer}.{old}.weight")
626
+ state[f"depth.{layer}.{name}"] = weight.repeat(c.total_codebooks, 1, 1)
627
+ for name, value in new.items():
628
+ if name.startswith("text_"):
629
+ state[name] = value
630
+ model.load_state_dict(state, strict=True)
631
+ return model
632
+
633
+
634
+ def with_text(model, text):
635
+ """The same weights under another training option, such as a later stage's."""
636
+ widened = DuplexModel(replace(model.config, text=text)).to(model.bos.device)
637
+ widened.load_state_dict(model.state_dict(), strict=True)
638
+ return widened
duplex/streaming.py ADDED
@@ -0,0 +1,542 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Incremental sampling for one DuplexModel sequence, for serving.
2
+
3
+ `DuplexModel.generate` recomputes the whole temporal context for every sequence step.
4
+ `Sampler` keeps a key/value cache instead, so a step costs its temporal positions
5
+ and the depth positions it needs. It computes the same distribution as training:
6
+ step s runs the audio position, then each present text track's tokens of frame s
7
+ until its advance token, then predicts codebook 0 of frame s and the acoustic
8
+ codebooks of frame s - 1 in every track from the step's last position. The text
9
+ written on track p at step s is the words starting at frame s + the checkpoint's
10
+ `Config.text_shift[p]`, as in training windows: track 2, spoken as, at their start,
11
+ and track 1, listened to, a few frames after it (6 by default).
12
+
13
+ Each audio position of a step is forced to a given token, sampled, left absent, or
14
+ supplied later. A later position must follow every sampled position of its step,
15
+ because no prediction in the step may depend on it; it must be supplied with
16
+ `complete` before the next step. Live serving forces the caller's track this way
17
+ while it arrives, frame by frame. Text never depends on the step's audio.
18
+ """
19
+
20
+ import numpy as np
21
+ import torch
22
+ from torch.nn import functional as F
23
+
24
+ from .network import ADVANCE, MAX_TEXT_PER_FRAME, TRACKS, norm, rotate, sample
25
+
26
+ SAMPLE, LATER, ABSENT = -1, -2, -3
27
+ # The sampling standard: audio at temperature 1 from the 250 most likely codes;
28
+ # text at temperature 1 from every token.
29
+ TEMPERATURE, TOP_K = 1.0, 250
30
+
31
+
32
+ def _project(attention, x, rope, position):
33
+ """Rotated, normalised queries and keys and the values for one position per row.
34
+
35
+ x is [rows, width] and position an int or a [rows] tensor; each result is
36
+ [rows, heads, 1, head_width].
37
+ """
38
+ head_width = x.shape[-1] // attention.heads
39
+ q, k, v = [
40
+ layer(x).reshape(x.shape[0], attention.heads, 1, head_width)
41
+ for layer in (attention.c_q, attention.c_k, attention.c_v)
42
+ ]
43
+ cos = rope[0, position].to(x.dtype)
44
+ sin = rope[1, position].to(x.dtype)
45
+ if cos.dim() == 2:
46
+ cos, sin = cos[:, None, None], sin[:, None, None]
47
+ return norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2, v
48
+
49
+
50
+ class _Cache:
51
+ """Keys and values of one transformer stack, [layers, rows, heads, positions, head_width]."""
52
+
53
+ def __init__(self, layers, heads, width, positions, reference, rows=1):
54
+ shape = (layers, rows, heads, positions, width // heads)
55
+ self.keys = reference.new_zeros(shape)
56
+ self.values = reference.new_zeros(shape)
57
+
58
+ def attend(self, i, q, k, v, position):
59
+ self.keys[i, :, :, position] = k[:, :, 0].to(self.keys.dtype)
60
+ self.values[i, :, :, position] = v[:, :, 0].to(self.values.dtype)
61
+ keys = self.keys[i, :, :, : position + 1].to(q.dtype)
62
+ values = self.values[i, :, :, : position + 1].to(q.dtype)
63
+ return F.scaled_dot_product_attention(q, keys, values)
64
+
65
+ def run(self, blocks, x, rope, position):
66
+ """One temporal position through every block, attending to it and those before."""
67
+ for i, block in enumerate(blocks):
68
+ q, k, v = _project(block.attn, norm(x), rope, position)
69
+ y = self.attend(i, q, k, v, position)
70
+ x = x + block.attn.c_proj(y.reshape(x.shape))
71
+ x = x + block.mlp(norm(x))
72
+ return x
73
+
74
+ def run_rows(self, blocks, x, rope, positions, length):
75
+ """One temporal position per row, each at its own position [rows], attending to
76
+ its own earlier positions; length bounds every row's position + 1."""
77
+ rows = torch.arange(x.shape[0], device=x.device)
78
+ mask = torch.arange(length, device=x.device) <= positions[:, None]
79
+ mask = mask[:, None, None]
80
+ for i, block in enumerate(blocks):
81
+ q, k, v = _project(block.attn, norm(x), rope, positions)
82
+ self.keys[i, rows, :, positions] = k[:, :, 0].to(self.keys.dtype)
83
+ self.values[i, rows, :, positions] = v[:, :, 0].to(self.values.dtype)
84
+ keys = self.keys[i, :, :, :length].to(q.dtype)
85
+ values = self.values[i, :, :, :length].to(q.dtype)
86
+ y = F.scaled_dot_product_attention(q, keys, values, attn_mask=mask)
87
+ x = x + block.attn.c_proj(y.reshape(x.shape))
88
+ x = x + block.mlp(norm(x))
89
+ return x
90
+
91
+ def run_depth(self, blocks, x, rope, position):
92
+ """One depth position through every block, with that position's own weights."""
93
+ for i, block in enumerate(blocks):
94
+ heads = block.heads
95
+ d = x.shape[-1] // heads
96
+ h = norm(x)
97
+ q, k, v = [
98
+ (h @ m[position].T.to(h.dtype)).reshape(x.shape[0], heads, 1, d)
99
+ for m in (block.c_q, block.c_k, block.c_v)
100
+ ]
101
+ cos = rope[0, position].to(x.dtype)
102
+ sin = rope[1, position].to(x.dtype)
103
+ q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
104
+ y = self.attend(i, q, k, v, position).reshape(x.shape)
105
+ x = x + y @ block.c_proj[position].T.to(y.dtype)
106
+ h = F.relu(norm(x) @ block.c_fc[position].T.to(x.dtype)).square()
107
+ x = x + h @ block.c_out[position].T.to(h.dtype)
108
+ return x
109
+
110
+
111
+ class Sampler:
112
+ """One sequence, one step at a time. Up to `context` + 1 positions; no rollover.
113
+
114
+ text names the present text tracks (0 for track 1, 1 for track 2); the default
115
+ is the checkpoint's training option. With `record`, `log` collects the log
116
+ probability of every text event and every audio token the steps produce or
117
+ are given, in sampling order: ("text", track, id, logp) and ("audio", position,
118
+ id, logp).
119
+ """
120
+
121
+ def __init__(
122
+ self,
123
+ model,
124
+ temperature=TEMPERATURE,
125
+ generator=None,
126
+ top_k=TOP_K,
127
+ text=None,
128
+ record=False,
129
+ text_sampling=None,
130
+ ):
131
+ if temperature < 0 or top_k < 0:
132
+ raise ValueError("temperature and top_k must be nonnegative")
133
+ self.model = model
134
+ self.temperature = temperature
135
+ # 0 samples from every code; k samples from the k most likely at each position.
136
+ self.top_k = top_k
137
+ # Text's (temperature, top_k); default the audio temperature from every token.
138
+ self.text_sampling = text_sampling or (temperature, 0)
139
+ self.generator = generator
140
+ c = model.config
141
+ self.text = tuple(model.config.text_tracks if text is None else text)
142
+ if any(j not in range(TRACKS) for j in self.text):
143
+ raise ValueError("Text tracks are 0 and 1")
144
+ self.pattern = sum(1 << j for j in self.text)
145
+ self.q = c.total_codebooks
146
+ weights = model.bos.detach()
147
+ self.temporal = _Cache(c.layers, c.heads, c.width, c.context + 1, weights)
148
+ self.steps = 0
149
+ self.positions = 0
150
+ # Tokens of the previous step in delayed order; ABSENT where none.
151
+ self.previous = None
152
+ self.log = [] if record else None
153
+
154
+ def room(self):
155
+ """Whether one more step fits in the context at the per-frame text limit."""
156
+ need = 1 + len(self.text) * MAX_TEXT_PER_FRAME
157
+ return self.positions + need <= self.model.config.context + 1
158
+
159
+ def _position(self, x):
160
+ if self.positions > self.model.config.context:
161
+ raise ValueError("Sequence exceeds the model context")
162
+ m = self.model
163
+ h = self.temporal.run(m.temporal, norm(x), m.temporal_rope, self.positions)
164
+ self.positions += 1
165
+ return norm(h)
166
+
167
+ @torch.no_grad()
168
+ def step(self, forced, text=None):
169
+ """Run the next step. forced has one entry per codebook position: a token,
170
+ SAMPLE, LATER or ABSENT. Acoustic positions of step 0 are absent whatever is
171
+ passed. text has one entry per present text track: SAMPLE, or the list of
172
+ that frame's given token ids (the advance is implied); default all SAMPLE.
173
+
174
+ Returns the step's audio tokens, with LATER where still to be supplied, and
175
+ the text tokens written on each present track."""
176
+ m = self.model
177
+ if self.previous is not None and LATER in self.previous:
178
+ raise ValueError("Complete the previous step first")
179
+ tokens = [int(t) for t in forced]
180
+ if len(tokens) != self.q:
181
+ raise ValueError(f"Expected {self.q} positions")
182
+ if self.steps == 0:
183
+ tokens = [t if d == 0 else ABSENT for t, d in zip(tokens, m.delays)]
184
+ text = [SAMPLE] * len(self.text) if text is None else list(text)
185
+ if len(text) != len(self.text):
186
+ raise ValueError("One text entry per present text track")
187
+ sampled = [k for k, t in enumerate(tokens) if t == SAMPLE]
188
+ later = [k for k, t in enumerate(tokens) if t == LATER]
189
+ if later and sampled and min(later) < max(sampled):
190
+ raise ValueError("A later position precedes a sampled one")
191
+
192
+ if self.previous is None:
193
+ x = m.bos[None]
194
+ else:
195
+ present = [k for k, t in enumerate(self.previous) if t >= 0]
196
+ ids = torch.tensor([self.previous[k] for k in present], device=m.bos.device)
197
+ rows = m.frame_embeddings.weight[ids + m.offsets[present]]
198
+ x = rows.sum(0, keepdim=True).to(m.bos.dtype)
199
+ if self.pattern:
200
+ x = x + m.text_presence[self.pattern - 1].to(x.dtype)
201
+ h = self._position(x)
202
+
203
+ written = []
204
+ for track, given in zip(self.text, text, strict=True):
205
+ out = []
206
+ while True:
207
+ logits = m.text_logits(h, torch.tensor(track, device=h.device))[0]
208
+ logits = logits.float()
209
+ if given == SAMPLE:
210
+ if len(out) == MAX_TEXT_PER_FRAME:
211
+ token = ADVANCE
212
+ else:
213
+ token = int(sample(logits, *self.text_sampling, self.generator))
214
+ else:
215
+ token = given[len(out)] if len(out) < len(given) else ADVANCE
216
+ if self.log is not None:
217
+ self.log.append(
218
+ ("text", track, token, float(logits.log_softmax(-1)[token]))
219
+ )
220
+ if token == ADVANCE:
221
+ break
222
+ out.append(token)
223
+ e = m.text_embeddings.weight[token] + m.text_tracks[track]
224
+ h = self._position(e[None].to(h.dtype))
225
+ written.append(out)
226
+
227
+ context = m.to_depth(h)
228
+ scored = [k for k, t in enumerate(tokens) if t >= 0 or t == SAMPLE]
229
+ last = max(scored if self.log is not None else sampled, default=-1)
230
+ if last >= 0:
231
+ c = m.config
232
+ depth = _Cache(
233
+ c.depth_layers, c.depth_heads, c.depth_width, self.q, context
234
+ )
235
+ for position in range(last + 1):
236
+ x = context + m.depth_positions[position]
237
+ if position and tokens[position - 1] >= 0:
238
+ x = x + m.depth_embeddings.weight[
239
+ tokens[position - 1] + m.offsets[position - 1]
240
+ ].to(x.dtype)
241
+ x = depth.run_depth(m.depth, x, m.depth_rope, position)
242
+ if tokens[position] != SAMPLE and (
243
+ self.log is None or tokens[position] < 0
244
+ ):
245
+ continue
246
+ logits = (norm(x) @ m.outputs[position].T.to(x.dtype))[0].float()
247
+ if tokens[position] == SAMPLE:
248
+ tokens[position] = int(
249
+ sample(logits, self.temperature, self.top_k, self.generator)
250
+ )
251
+ if self.log is not None:
252
+ self.log.append(
253
+ (
254
+ "audio",
255
+ position,
256
+ tokens[position],
257
+ float(logits.log_softmax(-1)[tokens[position]]),
258
+ )
259
+ )
260
+ self.previous = tokens
261
+ self.steps += 1
262
+ return list(tokens), written
263
+
264
+ def complete(self, supplied):
265
+ """Supply the LATER positions of the last step: {position: token or ABSENT}.
266
+
267
+ Positions the step made absent, the acoustics of step 0, are skipped."""
268
+ for position, token in supplied.items():
269
+ if self.previous[position] == ABSENT:
270
+ continue
271
+ if self.previous[position] != LATER:
272
+ raise ValueError(f"Position {position} was not left for later")
273
+ self.previous[position] = int(token)
274
+
275
+
276
+ class Duplex:
277
+ """Live two-track sampling: one caller frame in, one model frame out per call.
278
+
279
+ With `model_stream` 2 the caller is track 1 and the model hears each caller
280
+ frame in the step that samples its own. Its frame s is complete one step later,
281
+ so the first call returns None. With `model_stream` 1 the model samples each step
282
+ before the caller's frame arrives and returns the frame of the same index.
283
+
284
+ The model writes the text of its own track when its training option has it.
285
+ Under the `two` option the caller's text track is written by the model too, as
286
+ its reading of the caller. `text` holds the latest text tokens by track.
287
+ """
288
+
289
+ def __init__(
290
+ self, model, model_stream, temperature=TEMPERATURE, generator=None, top_k=TOP_K
291
+ ):
292
+ if model_stream not in (1, 2):
293
+ raise ValueError("model_stream is 1 or 2")
294
+ own = model_stream - 1
295
+ tracks = model.config.text_tracks
296
+ # Track 2 is the one trained to speak with its own text at its start frames.
297
+ if tracks and own != 1:
298
+ raise ValueError("A checkpoint with text speaks only as track 2")
299
+ self.model_stream = model_stream
300
+ self.sampler = Sampler(model, temperature, generator, top_k)
301
+ self.previous = None # the caller's last frame
302
+ self.semantic = None # the model's codebook 0 awaiting its acoustics
303
+ self.text = {}
304
+ if model_stream == 1:
305
+ tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
306
+ self.semantic = tokens[0]
307
+ self.text = dict(zip(tracks, written, strict=True))
308
+
309
+ def full(self):
310
+ return not self.sampler.room()
311
+
312
+ def push(self, caller):
313
+ """The caller's next frame, 8 codes; returns the model's next frame or None."""
314
+ caller = [int(c) for c in caller]
315
+ given = [caller[0]] + (self.previous[1:] if self.previous else [ABSENT] * 7)
316
+ self.previous = caller
317
+ tracks = self.sampler.text
318
+ if self.model_stream == 1:
319
+ self.sampler.complete(dict(enumerate(given, start=8)))
320
+ tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
321
+ frame, self.semantic = [self.semantic] + tokens[1:8], tokens[0]
322
+ self.text = dict(zip(tracks, written, strict=True))
323
+ return frame
324
+ tokens, written = self.sampler.step(given + [SAMPLE] * 8)
325
+ self.text = dict(zip(tracks, written, strict=True))
326
+ frame = None if self.semantic is None else [self.semantic] + tokens[9:]
327
+ self.semantic = tokens[8]
328
+ return frame
329
+
330
+
331
+ def generate(
332
+ model,
333
+ frames,
334
+ temperature=TEMPERATURE,
335
+ generator=None,
336
+ top_k=TOP_K,
337
+ text=None,
338
+ given=None,
339
+ text_sampling=None,
340
+ ):
341
+ """Fill the SAMPLE entries of canonical frames [frames, codebooks] in delayed order.
342
+
343
+ A prefix of given frames followed by SAMPLE rows matches `DuplexModel.generate`
344
+ when no text track is present. Any other mix forces some positions while
345
+ sampling the rest, such as one track given and the other generated; ABSENT
346
+ marks an empty track. text names the present text tracks (default: the
347
+ checkpoint's option), each sampled unless `given` maps it to the token ids
348
+ to write in each frame (an empty list writes only the advance; SAMPLE samples
349
+ that frame). text_sampling is text's (temperature, top_k), by default the
350
+ audio temperature from every token. Returns the
351
+ tokens and, per present text track, the token ids written in each frame.
352
+ """
353
+ given = given or {}
354
+ result = frames.clone()
355
+ total = len(result)
356
+ sampler = Sampler(
357
+ model, temperature, generator, top_k, text, text_sampling=text_sampling
358
+ )
359
+ written = {track: [] for track in sampler.text}
360
+ for step in range(total + 1):
361
+ rows = [step - d for d in model.delays]
362
+ forced = [
363
+ int(result[f, k]) if 0 <= f < total else ABSENT for k, f in enumerate(rows)
364
+ ]
365
+ # The flush step has no frame, so its text tracks write nothing.
366
+ tokens, out = sampler.step(
367
+ forced,
368
+ [given[t][step] if t in given else SAMPLE for t in sampler.text]
369
+ if step < total
370
+ else [[] for _ in sampler.text],
371
+ )
372
+ for track, ids in zip(sampler.text, out, strict=True):
373
+ if step < total:
374
+ written[track].append(ids)
375
+ for k, f in enumerate(rows):
376
+ if 0 <= f < total and int(result[f, k]) != ABSENT:
377
+ result[f, k] = tokens[k]
378
+ return result, written
379
+
380
+
381
+ # Uniform draws per clip and step: each text track's events, then each codebook.
382
+ _TEXT_SLOTS = MAX_TEXT_PER_FRAME + 1
383
+
384
+
385
+ def _draw(logits, temperature, top_k, u):
386
+ """One inverse-CDF draw per row of logits [rows, classes] from uniforms u [rows],
387
+ over the top_k most likely classes (0 keeps every class); argmax at temperature 0."""
388
+ if temperature == 0:
389
+ return logits.argmax(-1)
390
+ k = top_k if top_k and top_k < logits.shape[-1] else logits.shape[-1]
391
+ values, indices = logits.topk(k)
392
+ cdf = (values / temperature).softmax(-1).cumsum(-1)
393
+ target = (u.to(cdf.dtype) * cdf[:, -1]).unsqueeze(-1)
394
+ pick = torch.searchsorted(cdf, target).clamp(max=k - 1)
395
+ return indices.gather(-1, pick).squeeze(-1)
396
+
397
+
398
+ @torch.no_grad()
399
+ def generate_batch(
400
+ model,
401
+ frames,
402
+ seeds,
403
+ temperature=TEMPERATURE,
404
+ top_k=TOP_K,
405
+ text_sampling=None,
406
+ given=None,
407
+ ):
408
+ """`generate` for a batch of clips [rows, frames, codebooks].
409
+
410
+ Text is sampled unless `given[row]` maps a track to its token ids per frame, as
411
+ in `generate` (SAMPLE samples that frame); forced rows ignore their draws.
412
+
413
+ Each row keeps its own temporal position, because rows write different numbers
414
+ of text tokens, and draws from its own uniforms, seeded by `seeds[row]`, so a
415
+ row's output does not depend on the rest of the batch, up to float rounding. The distribution is the
416
+ batch-one sampler's; at temperature 0 the tokens are its tokens. Returns the
417
+ tokens and, per present text track, each row's token ids written in each frame.
418
+ """
419
+ m, c = model, model.config
420
+ device = m.bos.device
421
+ result = frames.to(device).clone()
422
+ rows, total, q = result.shape
423
+ if len(seeds) != rows:
424
+ raise ValueError("One seed per row")
425
+ text_temperature, text_top_k = text_sampling or (temperature, 0)
426
+ tracks = tuple(c.text_tracks)
427
+ uniforms = torch.from_numpy(
428
+ np.stack(
429
+ [
430
+ np.random.default_rng(seed).random(
431
+ (total + 1, len(tracks) * _TEXT_SLOTS + q)
432
+ )
433
+ for seed in seeds
434
+ ]
435
+ )
436
+ ).to(device)
437
+ # One spare position: rows that already advanced write a placeholder there.
438
+ cache = _Cache(c.layers, c.heads, c.width, c.context + 2, m.bos.detach(), rows)
439
+ positions = torch.zeros(rows, dtype=torch.long, device=device)
440
+ furthest = 0
441
+ written = {track: [[] for _ in range(rows)] for track in tracks}
442
+ delays = torch.tensor(m.delays, device=device)
443
+ codebooks = torch.arange(q, device=device)
444
+ previous = None
445
+
446
+ def advance(x, moving):
447
+ nonlocal furthest
448
+ h = cache.run_rows(
449
+ m.temporal, norm(x), m.temporal_rope, positions, furthest + 1
450
+ )
451
+ positions.add_(moving.long())
452
+ furthest = int(positions.max())
453
+ if furthest > c.context + 1:
454
+ raise ValueError("Sequence exceeds the model context")
455
+ return norm(h)
456
+
457
+ given = given or [{} for _ in range(rows)]
458
+ if len(given) != rows:
459
+ raise ValueError("One given map per row")
460
+ everyone = torch.ones(rows, dtype=torch.bool, device=device)
461
+ for step in range(total + 1):
462
+ frame = step - delays
463
+ inside = (frame >= 0) & (frame < total)
464
+ forced = result[:, frame.clamp(0, total - 1), codebooks]
465
+ forced = torch.where(inside, forced, torch.full_like(forced, ABSENT))
466
+ if previous is None:
467
+ x = m.bos.expand(rows, -1)
468
+ else:
469
+ present = previous >= 0
470
+ embedded = m.frame_embeddings.weight[previous.clamp(min=0) + m.offsets]
471
+ x = (embedded * present[..., None].to(embedded.dtype)).sum(1)
472
+ x = x.to(m.bos.dtype)
473
+ if tracks:
474
+ x = x + m.text_presence[sum(1 << j for j in tracks) - 1].to(x.dtype)
475
+ h = advance(x, everyone)
476
+ # The flush step has no frame, so its text tracks write nothing.
477
+ for slot, track in enumerate(tracks if step < total else ()):
478
+ active = everyone
479
+ forced_text = [
480
+ g[track][step] if track in g and g[track][step] != SAMPLE else None
481
+ for g in given
482
+ ]
483
+ longest = max((len(t) for t in forced_text if t is not None), default=0)
484
+ # A sampled track advances after MAX_TEXT_PER_FRAME tokens without a draw;
485
+ # a forced one after its given tokens.
486
+ for count in range(max(MAX_TEXT_PER_FRAME, longest + 1)):
487
+ logits = m.text_logits(h, torch.tensor(track, device=device)).float()
488
+ u = uniforms[:, step, slot * _TEXT_SLOTS + min(count, _TEXT_SLOTS - 1)]
489
+ token = _draw(logits, text_temperature, text_top_k, u)
490
+ if count >= MAX_TEXT_PER_FRAME:
491
+ token = torch.full_like(token, ADVANCE)
492
+ for row, ids in enumerate(forced_text):
493
+ if ids is not None:
494
+ token[row] = ids[count] if count < len(ids) else ADVANCE
495
+ moving = active & (token != ADVANCE)
496
+ if not bool(moving.any()):
497
+ break
498
+ for row in moving.nonzero()[:, 0].tolist():
499
+ if count == 0:
500
+ written[track][row].append([])
501
+ written[track][row][-1].append(int(token[row]))
502
+ e = m.text_embeddings.weight[token.clamp(max=ADVANCE - 1)]
503
+ e = e + m.text_tracks[track]
504
+ h = torch.where(moving[:, None], advance(e.to(h.dtype), moving), h)
505
+ active = moving
506
+ for row in range(rows):
507
+ if len(written[track][row]) == step:
508
+ written[track][row].append([])
509
+ tokens = forced.clone()
510
+ sampled = forced == SAMPLE
511
+ wanted = sampled.any(0).nonzero()
512
+ if len(wanted):
513
+ context = m.to_depth(h)
514
+ depth = _Cache(
515
+ c.depth_layers, c.depth_heads, c.depth_width, q, context, rows
516
+ )
517
+ for position in range(int(wanted.max()) + 1):
518
+ x = context + m.depth_positions[position]
519
+ if position:
520
+ before = tokens[:, position - 1]
521
+ e = m.depth_embeddings.weight[
522
+ before.clamp(min=0) + m.offsets[position - 1]
523
+ ]
524
+ x = x + (e * (before >= 0)[:, None].to(e.dtype)).to(x.dtype)
525
+ x = depth.run_depth(m.depth, x, m.depth_rope, position)
526
+ if not bool(sampled[:, position].any()):
527
+ continue
528
+ logits = (norm(x) @ m.outputs[position].T.to(x.dtype)).float()
529
+ u = uniforms[:, step, len(tracks) * _TEXT_SLOTS + position]
530
+ draw = _draw(logits, temperature, top_k, u)
531
+ tokens[:, position] = torch.where(
532
+ sampled[:, position], draw, tokens[:, position]
533
+ )
534
+ previous = tokens
535
+ keep = inside & (result[:, frame.clamp(0, total - 1), codebooks] != ABSENT)
536
+ update = torch.where(
537
+ keep, tokens, result[:, frame.clamp(0, total - 1), codebooks]
538
+ )
539
+ result[:, frame.clamp(0, total - 1)[inside], codebooks[inside]] = update[
540
+ :, inside
541
+ ]
542
+ return result, written
examples/prompt1.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45409befa6feaad396a2370213e38caff1adea69804c3a4f6f4f020b34d9fb7d
3
+ size 480044
examples/prompt2.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a21273bb937f6119637c4a754eaa447f09f0b0fc054668e4774f36c4e9406f7
3
+ size 480044
examples/prompt3.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f75656b3877b665e0f317ba9d5a8ffbbc6be24e1eba09dfbaf2d8a14eb47c259
3
+ size 480044
examples/prompt4.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c43ce501f6a4d6f011fe79fcae57ddbd01e136fba1a661b0e6ee813b037b50b
3
+ size 480044
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ torch
2
+ transformers
3
+ sentencepiece
4
+ safetensors
5
+ librosa
6
+ soundfile
7
+ numpy