Safetensors
English
File size: 26,945 Bytes
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d90680b
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d90680b
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d90680b
d52c89c
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
"""Temporal/depth autoregression over two audio tracks and two text tracks.

A frame holds `codebooks` audio codes for each of the two tracks, track 1 then
track 2. In every track the acoustic codebooks run one frame behind codebook 0
inside the model. Callers pass and receive canonical undelayed frames.

Text runs at a variable rate (training readiness owns the design). Sequence step s
is one temporal position for the audio, followed by the text tokens of frame s of
each present text track, track 1 first. A track's text ends with the advance token,
which is only ever a target: each text event is predicted from the last input before
it, and the step's last position conditions the depth transformer.
"""

from dataclasses import dataclass, replace

import numpy as np
import torch
from torch import nn
from torch.nn import functional as F

from .kernels import depth_attention

TRACKS = 2
# Moshi's SentencePiece ids are 0..31999; the advance token follows them.
TEXT_IDS = 32000
ADVANCE = TEXT_IDS
# Sampling forces the advance after this many tokens of one track in one frame.
MAX_TEXT_PER_FRAME = 16
# Moshi's equation 7: codebook 0 of each track against 1 for each acoustic codebook,
# and its padding weight for the advance token.
SEMANTIC_WEIGHT = 100.0
ADVANCE_WEIGHT = 0.5
TEXT_OPTIONS = {"none": (), "one": (1,), "two": (0, 1)}
# Text against audio per track, in 80 ms frames: a word starting at frame f is
# written on track p at frame f - TEXT_SHIFT[p]. Positive puts the text ahead of the
# audio, negative behind it, 0 at the word's start. Track 1 (listened to) is 480 ms
# behind, a transcript; track 2 (spoken as) writes words where they start. The shift
# follows the track after the speaker swap; windows apply it, generation follows it.
# The default of `Config.text_shift`, which each checkpoint records.
TEXT_SHIFT = (-6, 0)
# Text output heads: one shared by both tracks, or one per track.
TEXT_HEADS = ("shared", "per-track")


@dataclass(frozen=True)
class Config:
    layers: int = 12
    width: int = 768
    heads: int = 6
    depth_layers: int = 2
    depth_width: int = 256
    depth_heads: int = 4
    # Temporal positions per window: frames plus text tokens, plus one flush step.
    context: int = 4096
    codebooks: int = 8
    cardinality: int = 2048
    fps: float = 12.5
    # The training option; the parameters are the same for all three.
    text: str = "none"
    # Frames each track's text sits from its word start (see TEXT_SHIFT).
    text_shift: tuple = TEXT_SHIFT
    text_heads: str = "shared"

    @property
    def total_codebooks(self):
        return self.codebooks * TRACKS

    @property
    def text_tracks(self):
        return TEXT_OPTIONS[self.text]

    def __post_init__(self):
        for width, heads in (
            (self.width, self.heads),
            (self.depth_width, self.depth_heads),
        ):
            if heads < 1 or width % heads or (width // heads) % 2:
                raise ValueError("Head dimensions must be even")
        if (
            min(
                self.layers,
                self.depth_layers,
                self.context,
                self.codebooks,
                self.cardinality,
            )
            < 1
        ):
            raise ValueError("Dimensions must be positive")
        if self.fps <= 0:
            raise ValueError("fps must be positive")
        if self.text not in TEXT_OPTIONS:
            raise ValueError(f"text must be one of {sorted(TEXT_OPTIONS)}")
        # A config read from JSON holds a list.
        object.__setattr__(self, "text_shift", tuple(int(x) for x in self.text_shift))
        if len(self.text_shift) != TRACKS:
            raise ValueError("text_shift has one value per track")
        if self.text_heads not in TEXT_HEADS:
            raise ValueError(f"text_heads must be one of {TEXT_HEADS}")


# Sequence order of the codebooks and text. Checkpoints carry it; a model trained
# under one factorization is meaningless under another.
def factorization(config):
    return "two-tracks-delay-0-1-1-1-1-1-1-1-text-advance-shift" + "".join(
        f"{shift:+d}" for shift in config.text_shift
    )


FACTORIZATION = factorization(Config())


def tiny_config(text="none"):
    # Real codec vocabulary, small transformers. No fake-codec checkpoint confusion.
    return Config(
        layers=2,
        width=32,
        heads=2,
        depth_layers=1,
        depth_width=32,
        depth_heads=2,
        context=64,
        text=text,
    )


def norm(x):
    return F.rms_norm(x, (x.shape[-1],))


def rotary(positions, head_width):
    """Cos and sin tables [positions, head_width / 2], built once per model."""
    freq = torch.arange(0, head_width, 2, dtype=torch.float32) / head_width
    angles = torch.outer(torch.arange(positions), 100000.0**-freq)
    return torch.stack((angles.cos(), angles.sin()))


def rotate(y, cos, sin):
    a, b = y.chunk(2, dim=-1)
    return torch.cat((a * cos + b * sin, -a * sin + b * cos), dim=-1)


class Attention(nn.Module):
    def __init__(self, width, heads):
        super().__init__()
        self.heads = heads
        self.c_q = nn.Linear(width, width, bias=False)
        self.c_k = nn.Linear(width, width, bias=False)
        self.c_v = nn.Linear(width, width, bias=False)
        self.c_proj = nn.Linear(width, width, bias=False)

    def forward(self, x, rope):
        b, t, w = x.shape
        d = w // self.heads
        q, k, v = [
            layer(x).reshape(b, t, self.heads, d).transpose(1, 2)
            for layer in (self.c_q, self.c_k, self.c_v)
        ]
        cos, sin = rope[0, :t].to(x.dtype), rope[1, :t].to(x.dtype)
        q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.c_proj(y.transpose(1, 2).reshape(x.shape))


class MLP(nn.Module):
    def __init__(self, width):
        super().__init__()
        self.c_fc = nn.Linear(width, 4 * width, bias=False)
        self.c_proj = nn.Linear(4 * width, width, bias=False)

    def forward(self, x):
        return self.c_proj(F.relu(self.c_fc(x)).square())


class Block(nn.Module):
    def __init__(self, width, heads):
        super().__init__()
        self.attn = Attention(width, heads)
        self.mlp = MLP(width)

    def forward(self, x, rope):
        x = x + self.attn(norm(x), rope)
        return x + self.mlp(norm(x))


def per_position(x, weight):
    """x [rows, positions, in] through weight [positions, out, in]: one matrix per position."""
    y = torch.bmm(x.transpose(0, 1), weight.transpose(1, 2).to(x.dtype))
    return y.transpose(0, 1)


class DepthBlock(nn.Module):
    """A depth block with its own weights at every codebook position.

    Same widths and arithmetic as `Block`; each [positions, out, in] matrix holds
    one nn.Linear weight per position, initialised as nn.Linear initialises.
    """

    def __init__(self, positions, width, heads):
        super().__init__()
        self.heads = heads

        def weight(out, inner):
            bound = inner**-0.5
            return nn.Parameter(
                torch.empty(positions, out, inner).uniform_(-bound, bound)
            )

        self.c_q, self.c_k, self.c_v, self.c_proj = (
            weight(width, width) for _ in range(4)
        )
        self.c_fc = weight(4 * width, width)
        self.c_out = weight(width, 4 * width)

    def forward(self, x, rope):
        n, p, w = x.shape
        d = w // self.heads
        h = norm(x)
        q, k, v = [
            per_position(h, m).reshape(n, p, self.heads, d)
            for m in (self.c_q, self.c_k, self.c_v)
        ]
        cos = rope[0, :p].to(x.dtype).unsqueeze(1)
        sin = rope[1, :p].to(x.dtype).unsqueeze(1)
        q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
        # Rotation and QK norm run at the residual precision; the kernel takes
        # the projection precision.
        y = depth_attention(q.to(v.dtype), k.to(v.dtype), v)
        x = x + per_position(y.reshape(n, p, w), self.c_proj)
        h = F.relu(per_position(norm(x), self.c_fc)).square()
        return x + per_position(h, self.c_out)


@dataclass
class TextLayout:
    """Where text sits in a padded batch, built by `text_layout` and `collate_text`.

    source [batch, positions] indexes the concatenation of the audio steps
    [frames + 1], the text tokens [tokens] and one zero row for padding.
    token and track [batch, tokens] are the text tokens' ids and tracks.
    last [batch, frames + 1] is the position whose output conditions each step's depth.
    event_* [batch, events] are the text predictions: the position predicting it, its
    track, its target id and its loss weight (0 for padding and the flush step).
    pattern [batch] is the bitmask of present text tracks.
    """

    source: torch.Tensor
    token: torch.Tensor
    track: torch.Tensor
    last: torch.Tensor
    event_position: torch.Tensor
    event_track: torch.Tensor
    event_target: torch.Tensor
    event_weight: torch.Tensor
    pattern: torch.Tensor


def text_layout(frames, text):
    """One window's text layout as numpy arrays; text is per track None (absent) or an
    int array [n, 2] of (frame, id) rows in word order, frames in [0, frames)."""
    present = [j for j in range(TRACKS) if text[j] is not None]
    counts = np.zeros((frames + 1, TRACKS), dtype=np.int64)
    parts = []
    for j in present:
        rows = np.asarray(text[j], dtype=np.int64).reshape(-1, 2)
        if len(rows) and (rows[:, 0].min() < 0 or rows[:, 0].max() >= frames):
            raise ValueError("Text frame outside the window")
        if len(rows) and (rows[:, 1].min() < 0 or rows[:, 1].max() >= TEXT_IDS):
            raise ValueError("Text id outside the vocabulary")
        np.add.at(counts[:, j], rows[:, 0], 1)
        parts.append(
            np.column_stack(
                (rows[:, 0], np.full(len(rows), j), np.arange(len(rows)), rows[:, 1])
            )
        )
    tokens = np.concatenate(parts) if parts else np.zeros((0, 4), dtype=np.int64)
    # Frame, then track, then word order.
    tokens = tokens[np.lexsort((tokens[:, 2], tokens[:, 1], tokens[:, 0]))]
    per_step = counts.sum(1)
    start = np.concatenate(([0], np.cumsum(1 + per_step)[:-1]))
    before = np.concatenate(([0], np.cumsum(per_step)))
    positions = int(frames + 1 + per_step.sum())
    rank = np.arange(len(tokens)) - before[tokens[:, 0]]
    token_position = start[tokens[:, 0]] + 1 + rank
    source = np.arange(frames + 1 + len(tokens))
    order = np.empty(positions, dtype=np.int64)
    order[start] = np.arange(frames + 1)
    order[token_position] = source[frames + 1 :]
    # Events: each token from the input before it; each advance from the last input
    # of its track in the step, or where the track's first token would come from.
    position = [token_position - 1]
    track = [tokens[:, 1]]
    target = [tokens[:, 3]]
    weight = [np.ones(len(tokens))]
    written = np.zeros(frames + 1, dtype=np.int64)
    for j in present:
        written = written + counts[:, j]
        position.append(start + written)
        track.append(np.full(frames + 1, j))
        target.append(np.full(frames + 1, ADVANCE))
        weight.append(np.r_[np.full(frames, ADVANCE_WEIGHT), 0.0])
    position, track, target, weight = (
        np.concatenate(x) for x in (position, track, target, weight)
    )
    # Sampling order: by position, then track.
    events = np.lexsort((track, position))
    return {
        "source": order,
        "token": tokens[:, 3],
        "track": tokens[:, 1],
        "last": start + per_step,
        "event_position": position[events],
        "event_track": track[events],
        "event_target": target[events],
        "event_weight": weight[events].astype(np.float32),
        "pattern": sum(1 << j for j in present),
    }


def collate_text(layouts, frames, positions, device):
    """Pads per-window layouts to [batch, positions] and [batch, frames + 1]."""
    b = len(layouts)
    tokens = max(1, max(len(x["token"]) for x in layouts))
    events = max(1, max(len(x["event_target"]) for x in layouts))
    pad = frames + 1 + tokens
    source = torch.full((b, positions), pad, dtype=torch.long)
    token = torch.zeros((b, tokens), dtype=torch.long)
    track = torch.zeros((b, tokens), dtype=torch.long)
    last = torch.zeros((b, frames + 1), dtype=torch.long)
    event = [torch.zeros((b, events), dtype=torch.long) for _ in range(3)]
    event_weight = torch.zeros((b, events))
    for i, x in enumerate(layouts):
        steps, n, m = len(x["last"]), len(x["token"]), len(x["event_target"])
        if len(x["source"]) > positions or steps > frames + 1:
            raise ValueError("Window exceeds the padded batch")
        s = torch.as_tensor(x["source"])
        # Token sources are rebased past this batch's padded audio steps.
        source[i, : len(s)] = torch.where(s < steps, s, s - steps + frames + 1)
        token[i, :n] = torch.as_tensor(x["token"])
        track[i, :n] = torch.as_tensor(x["track"])
        last[i, :steps] = torch.as_tensor(x["last"])
        for tensor, key in zip(
            event, ("event_position", "event_track", "event_target"), strict=True
        ):
            tensor[i, :m] = torch.as_tensor(x[key])
        event_weight[i, :m] = torch.as_tensor(x["event_weight"])
    pattern = torch.tensor([x["pattern"] for x in layouts])
    return TextLayout(
        source.to(device),
        token.to(device),
        track.to(device),
        last.to(device),
        *(t.to(device) for t in event),
        event_weight.to(device),
        pattern.to(device),
    )


class DuplexModel(nn.Module):
    def __init__(self, config: Config):
        super().__init__()
        self.config = config
        c = config
        q = c.total_codebooks
        # One table per role with codebook offsets: a single gather instead of
        # one per codebook. Each track has its own rows.
        self.frame_embeddings = nn.EmbeddingBag(q * c.cardinality, c.width, mode="sum")
        # A zero BOS stays zero through bias-free blocks, amplifying RMSNorm's
        # derivative at zero until deep models overflow on the first backward.
        self.bos = nn.Parameter(torch.randn(c.width))
        # Text: one token table for both tracks, a vector per track, an embedding of
        # which text tracks are present added to every audio position (patterns 1 to
        # 3; none adds nothing), and a query per track before the shared head.
        self.text_embeddings = nn.Embedding(TEXT_IDS, c.width)
        self.text_tracks = nn.Parameter(torch.zeros(TRACKS, c.width))
        self.text_presence = nn.Parameter(torch.zeros(2**TRACKS - 1, c.width))
        self.text_queries = nn.Parameter(torch.zeros(TRACKS, c.width))
        if c.text_heads == "shared":
            self.text_head = nn.Linear(c.width, TEXT_IDS + 1, bias=False)
        else:
            # [track, events, width], initialised as nn.Linear initialises one head.
            bound = c.width**-0.5
            self.text_head = nn.Parameter(
                torch.empty(TRACKS, TEXT_IDS + 1, c.width).uniform_(-bound, bound)
            )
        self.temporal = nn.ModuleList(
            [Block(c.width, c.heads) for _ in range(c.layers)]
        )
        self.to_depth = nn.Linear(c.width, c.depth_width, bias=False)
        self.depth_embeddings = nn.Embedding((q - 1) * c.cardinality, c.depth_width)
        self.depth_positions = nn.Parameter(torch.zeros(q, c.depth_width))
        self.depth = nn.ModuleList(
            [DepthBlock(q, c.depth_width, c.depth_heads) for _ in range(c.depth_layers)]
        )
        # Per-codebook output heads [codebook, cardinality, depth_width].
        bound = c.depth_width**-0.5
        self.outputs = nn.Parameter(
            torch.empty(q, c.cardinality, c.depth_width).uniform_(-bound, bound)
        )
        self.register_buffer(
            "offsets", torch.arange(q) * c.cardinality, persistent=False
        )
        # Codebook 0 of each track has delay zero; its acoustic codebooks one.
        self.delays = [int(k % c.codebooks != 0) for k in range(q)]
        self.register_buffer(
            "semantic",
            torch.tensor([d == 0 for d in self.delays]),
            persistent=False,
        )
        self.register_buffer(
            "track_of", torch.arange(q) // c.codebooks, persistent=False
        )
        self.register_buffer(
            "temporal_rope",
            # One position more than the context flushes the last acoustic frame.
            rotary(c.context + 1, c.width // c.heads),
            persistent=False,
        )
        self.register_buffer(
            "depth_rope",
            rotary(q, c.depth_width // c.depth_heads),
            persistent=False,
        )

    def delayed(self, tokens, valid=None):
        """Token ids and presence for sequence steps 0..frames, [batch, frames + 1, codebooks].

        Step s holds codebook 0 of frame s and the acoustic codebooks of frame
        s - 1 in every track. valid is [batch, frames, tracks]: a token is absent
        before the first frame, after the last, on padded frames and on absent tracks.
        """
        b, t, _ = tokens.shape
        if valid is None:
            valid = tokens.new_ones((b, t, TRACKS), dtype=torch.bool)
        valid = valid[..., self.track_of]
        ids = torch.where(
            self.semantic, F.pad(tokens, (0, 0, 0, 1)), F.pad(tokens, (0, 0, 1, 0))
        )
        present = torch.where(
            self.semantic,
            F.pad(valid, (0, 0, 0, 1)),
            F.pad(valid, (0, 0, 1, 0)),
        )
        return ids, present

    def audio_steps(self, tokens, valid=None):
        """Temporal inputs of the audio positions, [batch, frames + 1, width]: BOS, then
        the sum of every present token of the step before."""
        b, t, q = tokens.shape
        if q != self.config.total_codebooks:
            raise ValueError("Expected [batch, frames, codebooks x tracks]")
        x = self.bos.expand(b, 1, -1)
        if t:
            ids, present = self.delayed(tokens, valid)
            weights = self.frame_embeddings.weight
            embedded = self.frame_embeddings(
                (ids[:, :t] + self.offsets).reshape(-1, q),
                per_sample_weights=present[:, :t].reshape(-1, q).to(weights.dtype),
            )
            x = torch.cat((x, embedded.reshape(b, t, -1)), dim=1)
        return x

    def temporal_outputs(self, tokens, valid=None, text=None):
        """Normalised temporal outputs [batch, positions, width]."""
        x = self.audio_steps(tokens, valid)
        if text is not None:
            b, _, w = x.shape
            presence = torch.cat((x.new_zeros(1, w), self.text_presence))
            x = x + presence[text.pattern][:, None]
            words = self.text_embeddings(text.token) + self.text_tracks[text.track]
            table = torch.cat((x, words.to(x.dtype), x.new_zeros(b, 1, w)), dim=1)
            x = table.gather(1, text.source[..., None].expand(-1, -1, w))
        if x.shape[1] > self.config.context + 1:
            raise ValueError("Window exceeds the temporal context")
        x = norm(x)
        for block in self.temporal:
            x = block(x, self.temporal_rope)
        return norm(x)

    def text_logits(self, hidden, track):
        """Logits [..., TEXT_IDS + 1] of text events from normalised temporal outputs."""
        x = norm(hidden + self.text_queries[track].to(hidden.dtype))
        if self.config.text_heads == "shared":
            return self.text_head(x)
        weights = self.text_head.to(x.dtype)
        track = torch.as_tensor(track, device=x.device)
        if track.dim() == 0:
            return x @ weights[track].T
        # Both heads for every event, each event keeping its own track's: fixed
        # shapes, so the compiled step stays fast (mask selection has
        # data-dependent shapes and trained at about half the step rate).
        return torch.where(track[..., None] == 0, x @ weights[0].T, x @ weights[1].T)

    def depth_logits(self, context, inputs):
        """Logits [batch, steps, codebooks, cardinality] in delayed step order.

        inputs is [batch, steps, codebooks - 1, depth_width]: the embedding of
        the token before each depth position, zero where there is none.
        """
        b, s, _ = context.shape
        q = self.config.total_codebooks
        x = context.unsqueeze(2) + self.depth_positions
        x = torch.cat((x[:, :, :1], x[:, :, 1:] + inputs), dim=2)
        x = x.reshape(b * s, q, -1)
        for block in self.depth:
            x = block(x, self.depth_rope)
        # [codebook, steps, width] x [codebook, width, cardinality]
        x = norm(x).transpose(0, 1)
        logits = torch.bmm(x, self.outputs.transpose(1, 2).to(x.dtype))
        return logits.transpose(0, 1).reshape(b, s, q, -1)

    def forward(self, tokens, valid=None, text=None):
        """Canonical frames in; audio logits aligned to the same [batch, frames,
        codebooks], and text logits [batch, events, TEXT_IDS + 1] or None.

        valid is the [batch, frames, tracks] mask of real frames of present tracks.
        text is a `TextLayout` or None for audio only.

        Fillers never appear as targets: codebook 0 of frame t is read from step
        t and its acoustic codebooks from step t + 1, so the final step flushes
        the last frame and every source token is scored once.
        """
        hidden = self.temporal_outputs(tokens, valid, text)
        steps = hidden
        if text is not None:
            w = hidden.shape[-1]
            steps = hidden.gather(1, text.last[..., None].expand(-1, -1, w))
        context = self.to_depth(steps)
        ids, present = self.delayed(tokens, valid)
        # Each depth position after the first takes the token before it in
        # step order. The step that flushes a short row's last frame sees no
        # codebook 0, as in generation, instead of batch padding.
        inputs = self.depth_embeddings(ids[:, :, :-1] + self.offsets[:-1])
        inputs = inputs * present[:, :, :-1, None].to(inputs.dtype)
        logits = self.depth_logits(context, inputs)
        audio = torch.where(self.semantic[:, None], logits[:, :-1], logits[:, 1:])
        if text is None:
            return audio, None
        w = hidden.shape[-1]
        events = hidden.gather(1, text.event_position[..., None].expand(-1, -1, w))
        return audio, self.text_logits(events, text.event_track)

    @torch.no_grad()
    def generate(self, prefix, frames, temperature=1.0, top_k=250):
        """Reference audio sampler without KV cache or text; returns canonical tokens."""
        if frames < 0 or temperature < 0 or top_k < 0:
            raise ValueError("frames, temperature and top_k must be nonnegative")
        c = self.config
        b, forced, q = prefix.shape
        result = torch.cat((prefix, prefix.new_zeros((b, frames, q))), dim=1)
        if not frames:
            return result
        # One step past the last frame flushes its acoustic codebooks.
        for step in range(forced, forced + frames + 1):
            window = result[:, max(0, step - c.context) : step]
            context = self.to_depth(self.temporal_outputs(window)[:, -1:])
            inputs = context.new_zeros((b, 1, q - 1, c.depth_width))
            for position, delay in enumerate(self.delays):
                frame = step - delay
                if not 0 <= frame < forced + frames:
                    continue
                # The prefix keeps its own tokens, including the acoustic
                # codebooks of its last frame; later frames are sampled.
                if frame >= forced:
                    logits = self.depth_logits(context, inputs)[:, 0, position].float()
                    result[:, frame, position] = sample(
                        logits, temperature, top_k
                    ).squeeze(-1)
                if position < q - 1:
                    inputs[:, 0, position] = self.depth_embeddings(
                        result[:, frame, position] + self.offsets[position]
                    )
        return result


def sample(logits, temperature, top_k=0, generator=None):
    """One draw per row of [..., classes] logits; top_k 0 keeps every class."""
    if temperature == 0:
        return logits.argmax(-1, keepdim=True)
    if top_k and top_k < logits.shape[-1]:
        floor = logits.topk(top_k).values[..., -1:]
        logits = logits.masked_fill(logits < floor, float("-inf"))
    probabilities = (logits / temperature).softmax(-1)
    return torch.multinomial(
        probabilities.reshape(-1, logits.shape[-1]), 1, generator=generator
    ).reshape(*logits.shape[:-1], 1)


def from_legacy(config, state):
    """A model in this format from a 14 November checkpoint (one or two streams,
    shared depth weights, no text).

    A two-stream checkpoint converts exactly: the shared depth block weights are
    copied to every position and the new text weights stay unused while no text
    is present. A one-stream checkpoint is widened as two-stream training began:
    each track gets a copy of the stream's embeddings, positions and heads, and
    track 1's codebook 7 feeding track 2's codebook 0 starts at zero.
    """
    config = dict(config)
    config.pop("factorization", None)
    streams = config.pop("streams", 1)
    if streams not in (1, 2):
        raise ValueError("Expected a one- or two-stream checkpoint")
    model = DuplexModel(Config(**config))
    c = model.config
    new = model.state_dict()
    state = dict(state)
    if streams == 1:
        for name in ("frame_embeddings.weight", "depth_positions", "outputs"):
            value = state[name]
            state[name] = value.repeat(TRACKS, *[1] * (value.dim() - 1))
        value = state["depth_embeddings.weight"]
        joint = value.new_zeros((c.cardinality, value.shape[1]))
        state["depth_embeddings.weight"] = torch.cat([value, joint, value])
    names = {
        "attn.c_q": "c_q",
        "attn.c_k": "c_k",
        "attn.c_v": "c_v",
        "attn.c_proj": "c_proj",
        "mlp.c_fc": "c_fc",
        "mlp.c_proj": "c_out",
    }
    for layer in range(c.depth_layers):
        for old, name in names.items():
            weight = state.pop(f"depth.{layer}.{old}.weight")
            state[f"depth.{layer}.{name}"] = weight.repeat(c.total_codebooks, 1, 1)
    for name, value in new.items():
        if name.startswith("text_"):
            state[name] = value
    model.load_state_dict(state, strict=True)
    return model


def with_text(model, text):
    """The same weights under another training option, such as a later stage's."""
    widened = DuplexModel(replace(model.config, text=text)).to(model.bos.device)
    widened.load_state_dict(model.state_dict(), strict=True)
    return widened