File size: 24,487 Bytes
d20eb3e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
"""Incremental sampling for one DuplexModel sequence, for serving.

`DuplexModel.generate` recomputes the whole temporal context for every sequence step.
`Sampler` keeps a key/value cache instead, so a step costs its temporal positions
and the depth positions it needs. It computes the same distribution as training:
step s runs the audio position, then each present text track's tokens of frame s
until its advance token, then predicts codebook 0 of frame s and the acoustic
codebooks of frame s - 1 in every track from the step's last position. The text
written on track p at step s is the words starting at frame s + the checkpoint's
`Config.text_shift[p]`, as in training windows: track 2, spoken as, at their start,
and track 1, listened to, a few frames after it (6 by default).

Each audio position of a step is forced to a given token, sampled, left absent, or
supplied later. A later position must follow every sampled position of its step,
because no prediction in the step may depend on it; it must be supplied with
`complete` before the next step. Live serving forces the caller's track this way
while it arrives, frame by frame. Text never depends on the step's audio.
"""

import numpy as np
import torch
from torch.nn import functional as F

from .network import ADVANCE, MAX_TEXT_PER_FRAME, TRACKS, norm, rotate, sample

SAMPLE, LATER, ABSENT = -1, -2, -3
# The sampling standard: audio at temperature 1 from the 250 most likely codes;
# text at temperature 1 from every token.
TEMPERATURE, TOP_K = 1.0, 250


def _project(attention, x, rope, position):
    """Rotated, normalised queries and keys and the values for one position per row.

    x is [rows, width] and position an int or a [rows] tensor; each result is
    [rows, heads, 1, head_width].
    """
    head_width = x.shape[-1] // attention.heads
    q, k, v = [
        layer(x).reshape(x.shape[0], attention.heads, 1, head_width)
        for layer in (attention.c_q, attention.c_k, attention.c_v)
    ]
    cos = rope[0, position].to(x.dtype)
    sin = rope[1, position].to(x.dtype)
    if cos.dim() == 2:
        cos, sin = cos[:, None, None], sin[:, None, None]
    return norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2, v


class _Cache:
    """Keys and values of one transformer stack, [layers, rows, heads, positions, head_width]."""

    def __init__(self, layers, heads, width, positions, reference, rows=1):
        shape = (layers, rows, heads, positions, width // heads)
        self.keys = reference.new_zeros(shape)
        self.values = reference.new_zeros(shape)

    def attend(self, i, q, k, v, position):
        self.keys[i, :, :, position] = k[:, :, 0].to(self.keys.dtype)
        self.values[i, :, :, position] = v[:, :, 0].to(self.values.dtype)
        keys = self.keys[i, :, :, : position + 1].to(q.dtype)
        values = self.values[i, :, :, : position + 1].to(q.dtype)
        return F.scaled_dot_product_attention(q, keys, values)

    def run(self, blocks, x, rope, position):
        """One temporal position through every block, attending to it and those before."""
        for i, block in enumerate(blocks):
            q, k, v = _project(block.attn, norm(x), rope, position)
            y = self.attend(i, q, k, v, position)
            x = x + block.attn.c_proj(y.reshape(x.shape))
            x = x + block.mlp(norm(x))
        return x

    def run_rows(self, blocks, x, rope, positions, length):
        """One temporal position per row, each at its own position [rows], attending to
        its own earlier positions; length bounds every row's position + 1."""
        rows = torch.arange(x.shape[0], device=x.device)
        mask = torch.arange(length, device=x.device) <= positions[:, None]
        mask = mask[:, None, None]
        for i, block in enumerate(blocks):
            q, k, v = _project(block.attn, norm(x), rope, positions)
            self.keys[i, rows, :, positions] = k[:, :, 0].to(self.keys.dtype)
            self.values[i, rows, :, positions] = v[:, :, 0].to(self.values.dtype)
            keys = self.keys[i, :, :, :length].to(q.dtype)
            values = self.values[i, :, :, :length].to(q.dtype)
            y = F.scaled_dot_product_attention(q, keys, values, attn_mask=mask)
            x = x + block.attn.c_proj(y.reshape(x.shape))
            x = x + block.mlp(norm(x))
        return x

    def run_depth(self, blocks, x, rope, position):
        """One depth position through every block, with that position's own weights."""
        for i, block in enumerate(blocks):
            heads = block.heads
            d = x.shape[-1] // heads
            h = norm(x)
            q, k, v = [
                (h @ m[position].T.to(h.dtype)).reshape(x.shape[0], heads, 1, d)
                for m in (block.c_q, block.c_k, block.c_v)
            ]
            cos = rope[0, position].to(x.dtype)
            sin = rope[1, position].to(x.dtype)
            q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2
            y = self.attend(i, q, k, v, position).reshape(x.shape)
            x = x + y @ block.c_proj[position].T.to(y.dtype)
            h = F.relu(norm(x) @ block.c_fc[position].T.to(x.dtype)).square()
            x = x + h @ block.c_out[position].T.to(h.dtype)
        return x


class Sampler:
    """One sequence, one step at a time. Up to `context` + 1 positions; no rollover.

    text names the present text tracks (0 for track 1, 1 for track 2); the default
    is the checkpoint's training option. With `record`, `log` collects the log
    probability of every text event and every audio token the steps produce or
    are given, in sampling order: ("text", track, id, logp) and ("audio", position,
    id, logp).
    """

    def __init__(
        self,
        model,
        temperature=TEMPERATURE,
        generator=None,
        top_k=TOP_K,
        text=None,
        record=False,
        text_sampling=None,
    ):
        if temperature < 0 or top_k < 0:
            raise ValueError("temperature and top_k must be nonnegative")
        self.model = model
        self.temperature = temperature
        # 0 samples from every code; k samples from the k most likely at each position.
        self.top_k = top_k
        # Text's (temperature, top_k); default the audio temperature from every token.
        self.text_sampling = text_sampling or (temperature, 0)
        self.generator = generator
        c = model.config
        self.text = tuple(model.config.text_tracks if text is None else text)
        if any(j not in range(TRACKS) for j in self.text):
            raise ValueError("Text tracks are 0 and 1")
        self.pattern = sum(1 << j for j in self.text)
        self.q = c.total_codebooks
        weights = model.bos.detach()
        self.temporal = _Cache(c.layers, c.heads, c.width, c.context + 1, weights)
        self.steps = 0
        self.positions = 0
        # Tokens of the previous step in delayed order; ABSENT where none.
        self.previous = None
        self.log = [] if record else None

    def room(self):
        """Whether one more step fits in the context at the per-frame text limit."""
        need = 1 + len(self.text) * MAX_TEXT_PER_FRAME
        return self.positions + need <= self.model.config.context + 1

    def _position(self, x):
        if self.positions > self.model.config.context:
            raise ValueError("Sequence exceeds the model context")
        m = self.model
        h = self.temporal.run(m.temporal, norm(x), m.temporal_rope, self.positions)
        self.positions += 1
        return norm(h)

    @torch.no_grad()
    def step(self, forced, text=None):
        """Run the next step. forced has one entry per codebook position: a token,
        SAMPLE, LATER or ABSENT. Acoustic positions of step 0 are absent whatever is
        passed. text has one entry per present text track: SAMPLE, or the list of
        that frame's given token ids (the advance is implied); default all SAMPLE.

        Returns the step's audio tokens, with LATER where still to be supplied, and
        the text tokens written on each present track."""
        m = self.model
        if self.previous is not None and LATER in self.previous:
            raise ValueError("Complete the previous step first")
        tokens = [int(t) for t in forced]
        if len(tokens) != self.q:
            raise ValueError(f"Expected {self.q} positions")
        if self.steps == 0:
            tokens = [t if d == 0 else ABSENT for t, d in zip(tokens, m.delays)]
        text = [SAMPLE] * len(self.text) if text is None else list(text)
        if len(text) != len(self.text):
            raise ValueError("One text entry per present text track")
        sampled = [k for k, t in enumerate(tokens) if t == SAMPLE]
        later = [k for k, t in enumerate(tokens) if t == LATER]
        if later and sampled and min(later) < max(sampled):
            raise ValueError("A later position precedes a sampled one")

        if self.previous is None:
            x = m.bos[None]
        else:
            present = [k for k, t in enumerate(self.previous) if t >= 0]
            ids = torch.tensor([self.previous[k] for k in present], device=m.bos.device)
            rows = m.frame_embeddings.weight[ids + m.offsets[present]]
            x = rows.sum(0, keepdim=True).to(m.bos.dtype)
        if self.pattern:
            x = x + m.text_presence[self.pattern - 1].to(x.dtype)
        h = self._position(x)

        written = []
        for track, given in zip(self.text, text, strict=True):
            out = []
            while True:
                logits = m.text_logits(h, torch.tensor(track, device=h.device))[0]
                logits = logits.float()
                if given == SAMPLE:
                    if len(out) == MAX_TEXT_PER_FRAME:
                        token = ADVANCE
                    else:
                        token = int(sample(logits, *self.text_sampling, self.generator))
                else:
                    token = given[len(out)] if len(out) < len(given) else ADVANCE
                if self.log is not None:
                    self.log.append(
                        ("text", track, token, float(logits.log_softmax(-1)[token]))
                    )
                if token == ADVANCE:
                    break
                out.append(token)
                e = m.text_embeddings.weight[token] + m.text_tracks[track]
                h = self._position(e[None].to(h.dtype))
            written.append(out)

        context = m.to_depth(h)
        scored = [k for k, t in enumerate(tokens) if t >= 0 or t == SAMPLE]
        last = max(scored if self.log is not None else sampled, default=-1)
        if last >= 0:
            c = m.config
            depth = _Cache(
                c.depth_layers, c.depth_heads, c.depth_width, self.q, context
            )
            for position in range(last + 1):
                x = context + m.depth_positions[position]
                if position and tokens[position - 1] >= 0:
                    x = x + m.depth_embeddings.weight[
                        tokens[position - 1] + m.offsets[position - 1]
                    ].to(x.dtype)
                x = depth.run_depth(m.depth, x, m.depth_rope, position)
                if tokens[position] != SAMPLE and (
                    self.log is None or tokens[position] < 0
                ):
                    continue
                logits = (norm(x) @ m.outputs[position].T.to(x.dtype))[0].float()
                if tokens[position] == SAMPLE:
                    tokens[position] = int(
                        sample(logits, self.temperature, self.top_k, self.generator)
                    )
                if self.log is not None:
                    self.log.append(
                        (
                            "audio",
                            position,
                            tokens[position],
                            float(logits.log_softmax(-1)[tokens[position]]),
                        )
                    )
        self.previous = tokens
        self.steps += 1
        return list(tokens), written

    def complete(self, supplied):
        """Supply the LATER positions of the last step: {position: token or ABSENT}.

        Positions the step made absent, the acoustics of step 0, are skipped."""
        for position, token in supplied.items():
            if self.previous[position] == ABSENT:
                continue
            if self.previous[position] != LATER:
                raise ValueError(f"Position {position} was not left for later")
            self.previous[position] = int(token)


class Duplex:
    """Live two-track sampling: one caller frame in, one model frame out per call.

    With `model_stream` 2 the caller is track 1 and the model hears each caller
    frame in the step that samples its own. Its frame s is complete one step later,
    so the first call returns None. With `model_stream` 1 the model samples each step
    before the caller's frame arrives and returns the frame of the same index.

    The model writes the text of its own track when its training option has it.
    Under the `two` option the caller's text track is written by the model too, as
    its reading of the caller. `text` holds the latest text tokens by track.
    """

    def __init__(
        self, model, model_stream, temperature=TEMPERATURE, generator=None, top_k=TOP_K
    ):
        if model_stream not in (1, 2):
            raise ValueError("model_stream is 1 or 2")
        own = model_stream - 1
        tracks = model.config.text_tracks
        # Track 2 is the one trained to speak with its own text at its start frames.
        if tracks and own != 1:
            raise ValueError("A checkpoint with text speaks only as track 2")
        self.model_stream = model_stream
        self.sampler = Sampler(model, temperature, generator, top_k)
        self.previous = None  # the caller's last frame
        self.semantic = None  # the model's codebook 0 awaiting its acoustics
        self.text = {}
        if model_stream == 1:
            tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
            self.semantic = tokens[0]
            self.text = dict(zip(tracks, written, strict=True))

    def full(self):
        return not self.sampler.room()

    def push(self, caller):
        """The caller's next frame, 8 codes; returns the model's next frame or None."""
        caller = [int(c) for c in caller]
        given = [caller[0]] + (self.previous[1:] if self.previous else [ABSENT] * 7)
        self.previous = caller
        tracks = self.sampler.text
        if self.model_stream == 1:
            self.sampler.complete(dict(enumerate(given, start=8)))
            tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8)
            frame, self.semantic = [self.semantic] + tokens[1:8], tokens[0]
            self.text = dict(zip(tracks, written, strict=True))
            return frame
        tokens, written = self.sampler.step(given + [SAMPLE] * 8)
        self.text = dict(zip(tracks, written, strict=True))
        frame = None if self.semantic is None else [self.semantic] + tokens[9:]
        self.semantic = tokens[8]
        return frame


def generate(
    model,
    frames,
    temperature=TEMPERATURE,
    generator=None,
    top_k=TOP_K,
    text=None,
    given=None,
    text_sampling=None,
):
    """Fill the SAMPLE entries of canonical frames [frames, codebooks] in delayed order.

    A prefix of given frames followed by SAMPLE rows matches `DuplexModel.generate`
    when no text track is present. Any other mix forces some positions while
    sampling the rest, such as one track given and the other generated; ABSENT
    marks an empty track. text names the present text tracks (default: the
    checkpoint's option), each sampled unless `given` maps it to the token ids
    to write in each frame (an empty list writes only the advance; SAMPLE samples
    that frame). text_sampling is text's (temperature, top_k), by default the
    audio temperature from every token. Returns the
    tokens and, per present text track, the token ids written in each frame.
    """
    given = given or {}
    result = frames.clone()
    total = len(result)
    sampler = Sampler(
        model, temperature, generator, top_k, text, text_sampling=text_sampling
    )
    written = {track: [] for track in sampler.text}
    for step in range(total + 1):
        rows = [step - d for d in model.delays]
        forced = [
            int(result[f, k]) if 0 <= f < total else ABSENT for k, f in enumerate(rows)
        ]
        # The flush step has no frame, so its text tracks write nothing.
        tokens, out = sampler.step(
            forced,
            [given[t][step] if t in given else SAMPLE for t in sampler.text]
            if step < total
            else [[] for _ in sampler.text],
        )
        for track, ids in zip(sampler.text, out, strict=True):
            if step < total:
                written[track].append(ids)
        for k, f in enumerate(rows):
            if 0 <= f < total and int(result[f, k]) != ABSENT:
                result[f, k] = tokens[k]
    return result, written


# Uniform draws per clip and step: each text track's events, then each codebook.
_TEXT_SLOTS = MAX_TEXT_PER_FRAME + 1


def _draw(logits, temperature, top_k, u):
    """One inverse-CDF draw per row of logits [rows, classes] from uniforms u [rows],
    over the top_k most likely classes (0 keeps every class); argmax at temperature 0."""
    if temperature == 0:
        return logits.argmax(-1)
    k = top_k if top_k and top_k < logits.shape[-1] else logits.shape[-1]
    values, indices = logits.topk(k)
    cdf = (values / temperature).softmax(-1).cumsum(-1)
    target = (u.to(cdf.dtype) * cdf[:, -1]).unsqueeze(-1)
    pick = torch.searchsorted(cdf, target).clamp(max=k - 1)
    return indices.gather(-1, pick).squeeze(-1)


@torch.no_grad()
def generate_batch(
    model,
    frames,
    seeds,
    temperature=TEMPERATURE,
    top_k=TOP_K,
    text_sampling=None,
    given=None,
):
    """`generate` for a batch of clips [rows, frames, codebooks].

    Text is sampled unless `given[row]` maps a track to its token ids per frame, as
    in `generate` (SAMPLE samples that frame); forced rows ignore their draws.

    Each row keeps its own temporal position, because rows write different numbers
    of text tokens, and draws from its own uniforms, seeded by `seeds[row]`, so a
    row's output does not depend on the rest of the batch, up to float rounding. The distribution is the
    batch-one sampler's; at temperature 0 the tokens are its tokens. Returns the
    tokens and, per present text track, each row's token ids written in each frame.
    """
    m, c = model, model.config
    device = m.bos.device
    result = frames.to(device).clone()
    rows, total, q = result.shape
    if len(seeds) != rows:
        raise ValueError("One seed per row")
    text_temperature, text_top_k = text_sampling or (temperature, 0)
    tracks = tuple(c.text_tracks)
    uniforms = torch.from_numpy(
        np.stack(
            [
                np.random.default_rng(seed).random(
                    (total + 1, len(tracks) * _TEXT_SLOTS + q)
                )
                for seed in seeds
            ]
        )
    ).to(device)
    # One spare position: rows that already advanced write a placeholder there.
    cache = _Cache(c.layers, c.heads, c.width, c.context + 2, m.bos.detach(), rows)
    positions = torch.zeros(rows, dtype=torch.long, device=device)
    furthest = 0
    written = {track: [[] for _ in range(rows)] for track in tracks}
    delays = torch.tensor(m.delays, device=device)
    codebooks = torch.arange(q, device=device)
    previous = None

    def advance(x, moving):
        nonlocal furthest
        h = cache.run_rows(
            m.temporal, norm(x), m.temporal_rope, positions, furthest + 1
        )
        positions.add_(moving.long())
        furthest = int(positions.max())
        if furthest > c.context + 1:
            raise ValueError("Sequence exceeds the model context")
        return norm(h)

    given = given or [{} for _ in range(rows)]
    if len(given) != rows:
        raise ValueError("One given map per row")
    everyone = torch.ones(rows, dtype=torch.bool, device=device)
    for step in range(total + 1):
        frame = step - delays
        inside = (frame >= 0) & (frame < total)
        forced = result[:, frame.clamp(0, total - 1), codebooks]
        forced = torch.where(inside, forced, torch.full_like(forced, ABSENT))
        if previous is None:
            x = m.bos.expand(rows, -1)
        else:
            present = previous >= 0
            embedded = m.frame_embeddings.weight[previous.clamp(min=0) + m.offsets]
            x = (embedded * present[..., None].to(embedded.dtype)).sum(1)
            x = x.to(m.bos.dtype)
        if tracks:
            x = x + m.text_presence[sum(1 << j for j in tracks) - 1].to(x.dtype)
        h = advance(x, everyone)
        # The flush step has no frame, so its text tracks write nothing.
        for slot, track in enumerate(tracks if step < total else ()):
            active = everyone
            forced_text = [
                g[track][step] if track in g and g[track][step] != SAMPLE else None
                for g in given
            ]
            longest = max((len(t) for t in forced_text if t is not None), default=0)
            # A sampled track advances after MAX_TEXT_PER_FRAME tokens without a draw;
            # a forced one after its given tokens.
            for count in range(max(MAX_TEXT_PER_FRAME, longest + 1)):
                logits = m.text_logits(h, torch.tensor(track, device=device)).float()
                u = uniforms[:, step, slot * _TEXT_SLOTS + min(count, _TEXT_SLOTS - 1)]
                token = _draw(logits, text_temperature, text_top_k, u)
                if count >= MAX_TEXT_PER_FRAME:
                    token = torch.full_like(token, ADVANCE)
                for row, ids in enumerate(forced_text):
                    if ids is not None:
                        token[row] = ids[count] if count < len(ids) else ADVANCE
                moving = active & (token != ADVANCE)
                if not bool(moving.any()):
                    break
                for row in moving.nonzero()[:, 0].tolist():
                    if count == 0:
                        written[track][row].append([])
                    written[track][row][-1].append(int(token[row]))
                e = m.text_embeddings.weight[token.clamp(max=ADVANCE - 1)]
                e = e + m.text_tracks[track]
                h = torch.where(moving[:, None], advance(e.to(h.dtype), moving), h)
                active = moving
            for row in range(rows):
                if len(written[track][row]) == step:
                    written[track][row].append([])
        tokens = forced.clone()
        sampled = forced == SAMPLE
        wanted = sampled.any(0).nonzero()
        if len(wanted):
            context = m.to_depth(h)
            depth = _Cache(
                c.depth_layers, c.depth_heads, c.depth_width, q, context, rows
            )
            for position in range(int(wanted.max()) + 1):
                x = context + m.depth_positions[position]
                if position:
                    before = tokens[:, position - 1]
                    e = m.depth_embeddings.weight[
                        before.clamp(min=0) + m.offsets[position - 1]
                    ]
                    x = x + (e * (before >= 0)[:, None].to(e.dtype)).to(x.dtype)
                x = depth.run_depth(m.depth, x, m.depth_rope, position)
                if not bool(sampled[:, position].any()):
                    continue
                logits = (norm(x) @ m.outputs[position].T.to(x.dtype)).float()
                u = uniforms[:, step, len(tracks) * _TEXT_SLOTS + position]
                draw = _draw(logits, temperature, top_k, u)
                tokens[:, position] = torch.where(
                    sampled[:, position], draw, tokens[:, position]
                )
        previous = tokens
        keep = inside & (result[:, frame.clamp(0, total - 1), codebooks] != ABSENT)
        update = torch.where(
            keep, tokens, result[:, frame.clamp(0, total - 1), codebooks]
        )
        result[:, frame.clamp(0, total - 1)[inside], codebooks[inside]] = update[
            :, inside
        ]
    return result, written