Download scripts/duplex/model/streaming.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 24.5 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/streaming.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/streaming.py
-
curl -L -o streaming.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/streaming.py
24.5 kB
| """Incremental sampling for one DuplexModel sequence, for serving. | |
| `DuplexModel.generate` recomputes the whole temporal context for every sequence step. | |
| `Sampler` keeps a key/value cache instead, so a step costs its temporal positions | |
| and the depth positions it needs. It computes the same distribution as training: | |
| step s runs the audio position, then each present text track's tokens of frame s | |
| until its advance token, then predicts codebook 0 of frame s and the acoustic | |
| codebooks of frame s - 1 in every track from the step's last position. The text | |
| written on track p at step s is the words starting at frame s + the checkpoint's | |
| `Config.text_shift[p]`, as in training windows: track 2, spoken as, at their start, | |
| and track 1, listened to, a few frames after it (6 by default). | |
| Each audio position of a step is forced to a given token, sampled, left absent, or | |
| supplied later. A later position must follow every sampled position of its step, | |
| because no prediction in the step may depend on it; it must be supplied with | |
| `complete` before the next step. Live serving forces the caller's track this way | |
| while it arrives, frame by frame. Text never depends on the step's audio. | |
| """ | |
| import numpy as np | |
| import torch | |
| from torch.nn import functional as F | |
| from .network import ADVANCE, MAX_TEXT_PER_FRAME, TRACKS, norm, rotate, sample | |
| SAMPLE, LATER, ABSENT = -1, -2, -3 | |
| # The sampling standard: audio at temperature 1 from the 250 most likely codes; | |
| # text at temperature 1 from every token. | |
| TEMPERATURE, TOP_K = 1.0, 250 | |
| def _project(attention, x, rope, position): | |
| """Rotated, normalised queries and keys and the values for one position per row. | |
| x is [rows, width] and position an int or a [rows] tensor; each result is | |
| [rows, heads, 1, head_width]. | |
| """ | |
| head_width = x.shape[-1] // attention.heads | |
| q, k, v = [ | |
| layer(x).reshape(x.shape[0], attention.heads, 1, head_width) | |
| for layer in (attention.c_q, attention.c_k, attention.c_v) | |
| ] | |
| cos = rope[0, position].to(x.dtype) | |
| sin = rope[1, position].to(x.dtype) | |
| if cos.dim() == 2: | |
| cos, sin = cos[:, None, None], sin[:, None, None] | |
| return norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2, v | |
| class _Cache: | |
| """Keys and values of one transformer stack, [layers, rows, heads, positions, head_width].""" | |
| def __init__(self, layers, heads, width, positions, reference, rows=1): | |
| shape = (layers, rows, heads, positions, width // heads) | |
| self.keys = reference.new_zeros(shape) | |
| self.values = reference.new_zeros(shape) | |
| def attend(self, i, q, k, v, position): | |
| self.keys[i, :, :, position] = k[:, :, 0].to(self.keys.dtype) | |
| self.values[i, :, :, position] = v[:, :, 0].to(self.values.dtype) | |
| keys = self.keys[i, :, :, : position + 1].to(q.dtype) | |
| values = self.values[i, :, :, : position + 1].to(q.dtype) | |
| return F.scaled_dot_product_attention(q, keys, values) | |
| def run(self, blocks, x, rope, position): | |
| """One temporal position through every block, attending to it and those before.""" | |
| for i, block in enumerate(blocks): | |
| q, k, v = _project(block.attn, norm(x), rope, position) | |
| y = self.attend(i, q, k, v, position) | |
| x = x + block.attn.c_proj(y.reshape(x.shape)) | |
| x = x + block.mlp(norm(x)) | |
| return x | |
| def run_rows(self, blocks, x, rope, positions, length): | |
| """One temporal position per row, each at its own position [rows], attending to | |
| its own earlier positions; length bounds every row's position + 1.""" | |
| rows = torch.arange(x.shape[0], device=x.device) | |
| mask = torch.arange(length, device=x.device) <= positions[:, None] | |
| mask = mask[:, None, None] | |
| for i, block in enumerate(blocks): | |
| q, k, v = _project(block.attn, norm(x), rope, positions) | |
| self.keys[i, rows, :, positions] = k[:, :, 0].to(self.keys.dtype) | |
| self.values[i, rows, :, positions] = v[:, :, 0].to(self.values.dtype) | |
| keys = self.keys[i, :, :, :length].to(q.dtype) | |
| values = self.values[i, :, :, :length].to(q.dtype) | |
| y = F.scaled_dot_product_attention(q, keys, values, attn_mask=mask) | |
| x = x + block.attn.c_proj(y.reshape(x.shape)) | |
| x = x + block.mlp(norm(x)) | |
| return x | |
| def run_depth(self, blocks, x, rope, position): | |
| """One depth position through every block, with that position's own weights.""" | |
| for i, block in enumerate(blocks): | |
| heads = block.heads | |
| d = x.shape[-1] // heads | |
| h = norm(x) | |
| q, k, v = [ | |
| (h @ m[position].T.to(h.dtype)).reshape(x.shape[0], heads, 1, d) | |
| for m in (block.c_q, block.c_k, block.c_v) | |
| ] | |
| cos = rope[0, position].to(x.dtype) | |
| sin = rope[1, position].to(x.dtype) | |
| q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2 | |
| y = self.attend(i, q, k, v, position).reshape(x.shape) | |
| x = x + y @ block.c_proj[position].T.to(y.dtype) | |
| h = F.relu(norm(x) @ block.c_fc[position].T.to(x.dtype)).square() | |
| x = x + h @ block.c_out[position].T.to(h.dtype) | |
| return x | |
| class Sampler: | |
| """One sequence, one step at a time. Up to `context` + 1 positions; no rollover. | |
| text names the present text tracks (0 for track 1, 1 for track 2); the default | |
| is the checkpoint's training option. With `record`, `log` collects the log | |
| probability of every text event and every audio token the steps produce or | |
| are given, in sampling order: ("text", track, id, logp) and ("audio", position, | |
| id, logp). | |
| """ | |
| def __init__( | |
| self, | |
| model, | |
| temperature=TEMPERATURE, | |
| generator=None, | |
| top_k=TOP_K, | |
| text=None, | |
| record=False, | |
| text_sampling=None, | |
| ): | |
| if temperature < 0 or top_k < 0: | |
| raise ValueError("temperature and top_k must be nonnegative") | |
| self.model = model | |
| self.temperature = temperature | |
| # 0 samples from every code; k samples from the k most likely at each position. | |
| self.top_k = top_k | |
| # Text's (temperature, top_k); default the audio temperature from every token. | |
| self.text_sampling = text_sampling or (temperature, 0) | |
| self.generator = generator | |
| c = model.config | |
| self.text = tuple(model.config.text_tracks if text is None else text) | |
| if any(j not in range(TRACKS) for j in self.text): | |
| raise ValueError("Text tracks are 0 and 1") | |
| self.pattern = sum(1 << j for j in self.text) | |
| self.q = c.total_codebooks | |
| weights = model.bos.detach() | |
| self.temporal = _Cache(c.layers, c.heads, c.width, c.context + 1, weights) | |
| self.steps = 0 | |
| self.positions = 0 | |
| # Tokens of the previous step in delayed order; ABSENT where none. | |
| self.previous = None | |
| self.log = [] if record else None | |
| def room(self): | |
| """Whether one more step fits in the context at the per-frame text limit.""" | |
| need = 1 + len(self.text) * MAX_TEXT_PER_FRAME | |
| return self.positions + need <= self.model.config.context + 1 | |
| def _position(self, x): | |
| if self.positions > self.model.config.context: | |
| raise ValueError("Sequence exceeds the model context") | |
| m = self.model | |
| h = self.temporal.run(m.temporal, norm(x), m.temporal_rope, self.positions) | |
| self.positions += 1 | |
| return norm(h) | |
| def step(self, forced, text=None): | |
| """Run the next step. forced has one entry per codebook position: a token, | |
| SAMPLE, LATER or ABSENT. Acoustic positions of step 0 are absent whatever is | |
| passed. text has one entry per present text track: SAMPLE, or the list of | |
| that frame's given token ids (the advance is implied); default all SAMPLE. | |
| Returns the step's audio tokens, with LATER where still to be supplied, and | |
| the text tokens written on each present track.""" | |
| m = self.model | |
| if self.previous is not None and LATER in self.previous: | |
| raise ValueError("Complete the previous step first") | |
| tokens = [int(t) for t in forced] | |
| if len(tokens) != self.q: | |
| raise ValueError(f"Expected {self.q} positions") | |
| if self.steps == 0: | |
| tokens = [t if d == 0 else ABSENT for t, d in zip(tokens, m.delays)] | |
| text = [SAMPLE] * len(self.text) if text is None else list(text) | |
| if len(text) != len(self.text): | |
| raise ValueError("One text entry per present text track") | |
| sampled = [k for k, t in enumerate(tokens) if t == SAMPLE] | |
| later = [k for k, t in enumerate(tokens) if t == LATER] | |
| if later and sampled and min(later) < max(sampled): | |
| raise ValueError("A later position precedes a sampled one") | |
| if self.previous is None: | |
| x = m.bos[None] | |
| else: | |
| present = [k for k, t in enumerate(self.previous) if t >= 0] | |
| ids = torch.tensor([self.previous[k] for k in present], device=m.bos.device) | |
| rows = m.frame_embeddings.weight[ids + m.offsets[present]] | |
| x = rows.sum(0, keepdim=True).to(m.bos.dtype) | |
| if self.pattern: | |
| x = x + m.text_presence[self.pattern - 1].to(x.dtype) | |
| h = self._position(x) | |
| written = [] | |
| for track, given in zip(self.text, text, strict=True): | |
| out = [] | |
| while True: | |
| logits = m.text_logits(h, torch.tensor(track, device=h.device))[0] | |
| logits = logits.float() | |
| if given == SAMPLE: | |
| if len(out) == MAX_TEXT_PER_FRAME: | |
| token = ADVANCE | |
| else: | |
| token = int(sample(logits, *self.text_sampling, self.generator)) | |
| else: | |
| token = given[len(out)] if len(out) < len(given) else ADVANCE | |
| if self.log is not None: | |
| self.log.append( | |
| ("text", track, token, float(logits.log_softmax(-1)[token])) | |
| ) | |
| if token == ADVANCE: | |
| break | |
| out.append(token) | |
| e = m.text_embeddings.weight[token] + m.text_tracks[track] | |
| h = self._position(e[None].to(h.dtype)) | |
| written.append(out) | |
| context = m.to_depth(h) | |
| scored = [k for k, t in enumerate(tokens) if t >= 0 or t == SAMPLE] | |
| last = max(scored if self.log is not None else sampled, default=-1) | |
| if last >= 0: | |
| c = m.config | |
| depth = _Cache( | |
| c.depth_layers, c.depth_heads, c.depth_width, self.q, context | |
| ) | |
| for position in range(last + 1): | |
| x = context + m.depth_positions[position] | |
| if position and tokens[position - 1] >= 0: | |
| x = x + m.depth_embeddings.weight[ | |
| tokens[position - 1] + m.offsets[position - 1] | |
| ].to(x.dtype) | |
| x = depth.run_depth(m.depth, x, m.depth_rope, position) | |
| if tokens[position] != SAMPLE and ( | |
| self.log is None or tokens[position] < 0 | |
| ): | |
| continue | |
| logits = (norm(x) @ m.outputs[position].T.to(x.dtype))[0].float() | |
| if tokens[position] == SAMPLE: | |
| tokens[position] = int( | |
| sample(logits, self.temperature, self.top_k, self.generator) | |
| ) | |
| if self.log is not None: | |
| self.log.append( | |
| ( | |
| "audio", | |
| position, | |
| tokens[position], | |
| float(logits.log_softmax(-1)[tokens[position]]), | |
| ) | |
| ) | |
| self.previous = tokens | |
| self.steps += 1 | |
| return list(tokens), written | |
| def complete(self, supplied): | |
| """Supply the LATER positions of the last step: {position: token or ABSENT}. | |
| Positions the step made absent, the acoustics of step 0, are skipped.""" | |
| for position, token in supplied.items(): | |
| if self.previous[position] == ABSENT: | |
| continue | |
| if self.previous[position] != LATER: | |
| raise ValueError(f"Position {position} was not left for later") | |
| self.previous[position] = int(token) | |
| class Duplex: | |
| """Live two-track sampling: one caller frame in, one model frame out per call. | |
| With `model_stream` 2 the caller is track 1 and the model hears each caller | |
| frame in the step that samples its own. Its frame s is complete one step later, | |
| so the first call returns None. With `model_stream` 1 the model samples each step | |
| before the caller's frame arrives and returns the frame of the same index. | |
| The model writes the text of its own track when its training option has it. | |
| Under the `two` option the caller's text track is written by the model too, as | |
| its reading of the caller. `text` holds the latest text tokens by track. | |
| """ | |
| def __init__( | |
| self, model, model_stream, temperature=TEMPERATURE, generator=None, top_k=TOP_K | |
| ): | |
| if model_stream not in (1, 2): | |
| raise ValueError("model_stream is 1 or 2") | |
| own = model_stream - 1 | |
| tracks = model.config.text_tracks | |
| # Track 2 is the one trained to speak with its own text at its start frames. | |
| if tracks and own != 1: | |
| raise ValueError("A checkpoint with text speaks only as track 2") | |
| self.model_stream = model_stream | |
| self.sampler = Sampler(model, temperature, generator, top_k) | |
| self.previous = None # the caller's last frame | |
| self.semantic = None # the model's codebook 0 awaiting its acoustics | |
| self.text = {} | |
| if model_stream == 1: | |
| tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8) | |
| self.semantic = tokens[0] | |
| self.text = dict(zip(tracks, written, strict=True)) | |
| def full(self): | |
| return not self.sampler.room() | |
| def push(self, caller): | |
| """The caller's next frame, 8 codes; returns the model's next frame or None.""" | |
| caller = [int(c) for c in caller] | |
| given = [caller[0]] + (self.previous[1:] if self.previous else [ABSENT] * 7) | |
| self.previous = caller | |
| tracks = self.sampler.text | |
| if self.model_stream == 1: | |
| self.sampler.complete(dict(enumerate(given, start=8))) | |
| tokens, written = self.sampler.step([SAMPLE] * 8 + [LATER] * 8) | |
| frame, self.semantic = [self.semantic] + tokens[1:8], tokens[0] | |
| self.text = dict(zip(tracks, written, strict=True)) | |
| return frame | |
| tokens, written = self.sampler.step(given + [SAMPLE] * 8) | |
| self.text = dict(zip(tracks, written, strict=True)) | |
| frame = None if self.semantic is None else [self.semantic] + tokens[9:] | |
| self.semantic = tokens[8] | |
| return frame | |
| def generate( | |
| model, | |
| frames, | |
| temperature=TEMPERATURE, | |
| generator=None, | |
| top_k=TOP_K, | |
| text=None, | |
| given=None, | |
| text_sampling=None, | |
| ): | |
| """Fill the SAMPLE entries of canonical frames [frames, codebooks] in delayed order. | |
| A prefix of given frames followed by SAMPLE rows matches `DuplexModel.generate` | |
| when no text track is present. Any other mix forces some positions while | |
| sampling the rest, such as one track given and the other generated; ABSENT | |
| marks an empty track. text names the present text tracks (default: the | |
| checkpoint's option), each sampled unless `given` maps it to the token ids | |
| to write in each frame (an empty list writes only the advance; SAMPLE samples | |
| that frame). text_sampling is text's (temperature, top_k), by default the | |
| audio temperature from every token. Returns the | |
| tokens and, per present text track, the token ids written in each frame. | |
| """ | |
| given = given or {} | |
| result = frames.clone() | |
| total = len(result) | |
| sampler = Sampler( | |
| model, temperature, generator, top_k, text, text_sampling=text_sampling | |
| ) | |
| written = {track: [] for track in sampler.text} | |
| for step in range(total + 1): | |
| rows = [step - d for d in model.delays] | |
| forced = [ | |
| int(result[f, k]) if 0 <= f < total else ABSENT for k, f in enumerate(rows) | |
| ] | |
| # The flush step has no frame, so its text tracks write nothing. | |
| tokens, out = sampler.step( | |
| forced, | |
| [given[t][step] if t in given else SAMPLE for t in sampler.text] | |
| if step < total | |
| else [[] for _ in sampler.text], | |
| ) | |
| for track, ids in zip(sampler.text, out, strict=True): | |
| if step < total: | |
| written[track].append(ids) | |
| for k, f in enumerate(rows): | |
| if 0 <= f < total and int(result[f, k]) != ABSENT: | |
| result[f, k] = tokens[k] | |
| return result, written | |
| # Uniform draws per clip and step: each text track's events, then each codebook. | |
| _TEXT_SLOTS = MAX_TEXT_PER_FRAME + 1 | |
| def _draw(logits, temperature, top_k, u): | |
| """One inverse-CDF draw per row of logits [rows, classes] from uniforms u [rows], | |
| over the top_k most likely classes (0 keeps every class); argmax at temperature 0.""" | |
| if temperature == 0: | |
| return logits.argmax(-1) | |
| k = top_k if top_k and top_k < logits.shape[-1] else logits.shape[-1] | |
| values, indices = logits.topk(k) | |
| cdf = (values / temperature).softmax(-1).cumsum(-1) | |
| target = (u.to(cdf.dtype) * cdf[:, -1]).unsqueeze(-1) | |
| pick = torch.searchsorted(cdf, target).clamp(max=k - 1) | |
| return indices.gather(-1, pick).squeeze(-1) | |
| def generate_batch( | |
| model, | |
| frames, | |
| seeds, | |
| temperature=TEMPERATURE, | |
| top_k=TOP_K, | |
| text_sampling=None, | |
| given=None, | |
| ): | |
| """`generate` for a batch of clips [rows, frames, codebooks]. | |
| Text is sampled unless `given[row]` maps a track to its token ids per frame, as | |
| in `generate` (SAMPLE samples that frame); forced rows ignore their draws. | |
| Each row keeps its own temporal position, because rows write different numbers | |
| of text tokens, and draws from its own uniforms, seeded by `seeds[row]`, so a | |
| row's output does not depend on the rest of the batch, up to float rounding. The distribution is the | |
| batch-one sampler's; at temperature 0 the tokens are its tokens. Returns the | |
| tokens and, per present text track, each row's token ids written in each frame. | |
| """ | |
| m, c = model, model.config | |
| device = m.bos.device | |
| result = frames.to(device).clone() | |
| rows, total, q = result.shape | |
| if len(seeds) != rows: | |
| raise ValueError("One seed per row") | |
| text_temperature, text_top_k = text_sampling or (temperature, 0) | |
| tracks = tuple(c.text_tracks) | |
| uniforms = torch.from_numpy( | |
| np.stack( | |
| [ | |
| np.random.default_rng(seed).random( | |
| (total + 1, len(tracks) * _TEXT_SLOTS + q) | |
| ) | |
| for seed in seeds | |
| ] | |
| ) | |
| ).to(device) | |
| # One spare position: rows that already advanced write a placeholder there. | |
| cache = _Cache(c.layers, c.heads, c.width, c.context + 2, m.bos.detach(), rows) | |
| positions = torch.zeros(rows, dtype=torch.long, device=device) | |
| furthest = 0 | |
| written = {track: [[] for _ in range(rows)] for track in tracks} | |
| delays = torch.tensor(m.delays, device=device) | |
| codebooks = torch.arange(q, device=device) | |
| previous = None | |
| def advance(x, moving): | |
| nonlocal furthest | |
| h = cache.run_rows( | |
| m.temporal, norm(x), m.temporal_rope, positions, furthest + 1 | |
| ) | |
| positions.add_(moving.long()) | |
| furthest = int(positions.max()) | |
| if furthest > c.context + 1: | |
| raise ValueError("Sequence exceeds the model context") | |
| return norm(h) | |
| given = given or [{} for _ in range(rows)] | |
| if len(given) != rows: | |
| raise ValueError("One given map per row") | |
| everyone = torch.ones(rows, dtype=torch.bool, device=device) | |
| for step in range(total + 1): | |
| frame = step - delays | |
| inside = (frame >= 0) & (frame < total) | |
| forced = result[:, frame.clamp(0, total - 1), codebooks] | |
| forced = torch.where(inside, forced, torch.full_like(forced, ABSENT)) | |
| if previous is None: | |
| x = m.bos.expand(rows, -1) | |
| else: | |
| present = previous >= 0 | |
| embedded = m.frame_embeddings.weight[previous.clamp(min=0) + m.offsets] | |
| x = (embedded * present[..., None].to(embedded.dtype)).sum(1) | |
| x = x.to(m.bos.dtype) | |
| if tracks: | |
| x = x + m.text_presence[sum(1 << j for j in tracks) - 1].to(x.dtype) | |
| h = advance(x, everyone) | |
| # The flush step has no frame, so its text tracks write nothing. | |
| for slot, track in enumerate(tracks if step < total else ()): | |
| active = everyone | |
| forced_text = [ | |
| g[track][step] if track in g and g[track][step] != SAMPLE else None | |
| for g in given | |
| ] | |
| longest = max((len(t) for t in forced_text if t is not None), default=0) | |
| # A sampled track advances after MAX_TEXT_PER_FRAME tokens without a draw; | |
| # a forced one after its given tokens. | |
| for count in range(max(MAX_TEXT_PER_FRAME, longest + 1)): | |
| logits = m.text_logits(h, torch.tensor(track, device=device)).float() | |
| u = uniforms[:, step, slot * _TEXT_SLOTS + min(count, _TEXT_SLOTS - 1)] | |
| token = _draw(logits, text_temperature, text_top_k, u) | |
| if count >= MAX_TEXT_PER_FRAME: | |
| token = torch.full_like(token, ADVANCE) | |
| for row, ids in enumerate(forced_text): | |
| if ids is not None: | |
| token[row] = ids[count] if count < len(ids) else ADVANCE | |
| moving = active & (token != ADVANCE) | |
| if not bool(moving.any()): | |
| break | |
| for row in moving.nonzero()[:, 0].tolist(): | |
| if count == 0: | |
| written[track][row].append([]) | |
| written[track][row][-1].append(int(token[row])) | |
| e = m.text_embeddings.weight[token.clamp(max=ADVANCE - 1)] | |
| e = e + m.text_tracks[track] | |
| h = torch.where(moving[:, None], advance(e.to(h.dtype), moving), h) | |
| active = moving | |
| for row in range(rows): | |
| if len(written[track][row]) == step: | |
| written[track][row].append([]) | |
| tokens = forced.clone() | |
| sampled = forced == SAMPLE | |
| wanted = sampled.any(0).nonzero() | |
| if len(wanted): | |
| context = m.to_depth(h) | |
| depth = _Cache( | |
| c.depth_layers, c.depth_heads, c.depth_width, q, context, rows | |
| ) | |
| for position in range(int(wanted.max()) + 1): | |
| x = context + m.depth_positions[position] | |
| if position: | |
| before = tokens[:, position - 1] | |
| e = m.depth_embeddings.weight[ | |
| before.clamp(min=0) + m.offsets[position - 1] | |
| ] | |
| x = x + (e * (before >= 0)[:, None].to(e.dtype)).to(x.dtype) | |
| x = depth.run_depth(m.depth, x, m.depth_rope, position) | |
| if not bool(sampled[:, position].any()): | |
| continue | |
| logits = (norm(x) @ m.outputs[position].T.to(x.dtype)).float() | |
| u = uniforms[:, step, len(tracks) * _TEXT_SLOTS + position] | |
| draw = _draw(logits, temperature, top_k, u) | |
| tokens[:, position] = torch.where( | |
| sampled[:, position], draw, tokens[:, position] | |
| ) | |
| previous = tokens | |
| keep = inside & (result[:, frame.clamp(0, total - 1), codebooks] != ABSENT) | |
| update = torch.where( | |
| keep, tokens, result[:, frame.clamp(0, total - 1), codebooks] | |
| ) | |
| result[:, frame.clamp(0, total - 1)[inside], codebooks[inside]] = update[ | |
| :, inside | |
| ] | |
| return result, written | |