Download scripts/duplex/model/network.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 26.9 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/network.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/network.py
-
curl -L -o network.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/network.py
26.9 kB
| """Temporal/depth autoregression over two audio tracks and two text tracks. | |
| A frame holds `codebooks` audio codes for each of the two tracks, track 1 then | |
| track 2. In every track the acoustic codebooks run one frame behind codebook 0 | |
| inside the model. Callers pass and receive canonical undelayed frames. | |
| Text runs at a variable rate (training readiness owns the design). Sequence step s | |
| is one temporal position for the audio, followed by the text tokens of frame s of | |
| each present text track, track 1 first. A track's text ends with the advance token, | |
| which is only ever a target: each text event is predicted from the last input before | |
| it, and the step's last position conditions the depth transformer. | |
| """ | |
| from dataclasses import dataclass, replace | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from .kernels import depth_attention | |
| TRACKS = 2 | |
| # Moshi's SentencePiece ids are 0..31999; the advance token follows them. | |
| TEXT_IDS = 32000 | |
| ADVANCE = TEXT_IDS | |
| # Sampling forces the advance after this many tokens of one track in one frame. | |
| MAX_TEXT_PER_FRAME = 16 | |
| # Moshi's equation 7: codebook 0 of each track against 1 for each acoustic codebook, | |
| # and its padding weight for the advance token. | |
| SEMANTIC_WEIGHT = 100.0 | |
| ADVANCE_WEIGHT = 0.5 | |
| TEXT_OPTIONS = {"none": (), "one": (1,), "two": (0, 1)} | |
| # Text against audio per track, in 80 ms frames: a word starting at frame f is | |
| # written on track p at frame f - TEXT_SHIFT[p]. Positive puts the text ahead of the | |
| # audio, negative behind it, 0 at the word's start. Track 1 (listened to) is 480 ms | |
| # behind, a transcript; track 2 (spoken as) writes words where they start. The shift | |
| # follows the track after the speaker swap; windows apply it, generation follows it. | |
| # The default of `Config.text_shift`, which each checkpoint records. | |
| TEXT_SHIFT = (-6, 0) | |
| # Text output heads: one shared by both tracks, or one per track. | |
| TEXT_HEADS = ("shared", "per-track") | |
| class Config: | |
| layers: int = 12 | |
| width: int = 768 | |
| heads: int = 6 | |
| depth_layers: int = 2 | |
| depth_width: int = 256 | |
| depth_heads: int = 4 | |
| # Temporal positions per window: frames plus text tokens, plus one flush step. | |
| context: int = 4096 | |
| codebooks: int = 8 | |
| cardinality: int = 2048 | |
| fps: float = 12.5 | |
| # The training option; the parameters are the same for all three. | |
| text: str = "none" | |
| # Frames each track's text sits from its word start (see TEXT_SHIFT). | |
| text_shift: tuple = TEXT_SHIFT | |
| text_heads: str = "shared" | |
| def total_codebooks(self): | |
| return self.codebooks * TRACKS | |
| def text_tracks(self): | |
| return TEXT_OPTIONS[self.text] | |
| def __post_init__(self): | |
| for width, heads in ( | |
| (self.width, self.heads), | |
| (self.depth_width, self.depth_heads), | |
| ): | |
| if heads < 1 or width % heads or (width // heads) % 2: | |
| raise ValueError("Head dimensions must be even") | |
| if ( | |
| min( | |
| self.layers, | |
| self.depth_layers, | |
| self.context, | |
| self.codebooks, | |
| self.cardinality, | |
| ) | |
| < 1 | |
| ): | |
| raise ValueError("Dimensions must be positive") | |
| if self.fps <= 0: | |
| raise ValueError("fps must be positive") | |
| if self.text not in TEXT_OPTIONS: | |
| raise ValueError(f"text must be one of {sorted(TEXT_OPTIONS)}") | |
| # A config read from JSON holds a list. | |
| object.__setattr__(self, "text_shift", tuple(int(x) for x in self.text_shift)) | |
| if len(self.text_shift) != TRACKS: | |
| raise ValueError("text_shift has one value per track") | |
| if self.text_heads not in TEXT_HEADS: | |
| raise ValueError(f"text_heads must be one of {TEXT_HEADS}") | |
| # Sequence order of the codebooks and text. Checkpoints carry it; a model trained | |
| # under one factorization is meaningless under another. | |
| def factorization(config): | |
| return "two-tracks-delay-0-1-1-1-1-1-1-1-text-advance-shift" + "".join( | |
| f"{shift:+d}" for shift in config.text_shift | |
| ) | |
| FACTORIZATION = factorization(Config()) | |
| def tiny_config(text="none"): | |
| # Real codec vocabulary, small transformers. No fake-codec checkpoint confusion. | |
| return Config( | |
| layers=2, | |
| width=32, | |
| heads=2, | |
| depth_layers=1, | |
| depth_width=32, | |
| depth_heads=2, | |
| context=64, | |
| text=text, | |
| ) | |
| def norm(x): | |
| return F.rms_norm(x, (x.shape[-1],)) | |
| def rotary(positions, head_width): | |
| """Cos and sin tables [positions, head_width / 2], built once per model.""" | |
| freq = torch.arange(0, head_width, 2, dtype=torch.float32) / head_width | |
| angles = torch.outer(torch.arange(positions), 100000.0**-freq) | |
| return torch.stack((angles.cos(), angles.sin())) | |
| def rotate(y, cos, sin): | |
| a, b = y.chunk(2, dim=-1) | |
| return torch.cat((a * cos + b * sin, -a * sin + b * cos), dim=-1) | |
| class Attention(nn.Module): | |
| def __init__(self, width, heads): | |
| super().__init__() | |
| self.heads = heads | |
| self.c_q = nn.Linear(width, width, bias=False) | |
| self.c_k = nn.Linear(width, width, bias=False) | |
| self.c_v = nn.Linear(width, width, bias=False) | |
| self.c_proj = nn.Linear(width, width, bias=False) | |
| def forward(self, x, rope): | |
| b, t, w = x.shape | |
| d = w // self.heads | |
| q, k, v = [ | |
| layer(x).reshape(b, t, self.heads, d).transpose(1, 2) | |
| for layer in (self.c_q, self.c_k, self.c_v) | |
| ] | |
| cos, sin = rope[0, :t].to(x.dtype), rope[1, :t].to(x.dtype) | |
| q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2 | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| return self.c_proj(y.transpose(1, 2).reshape(x.shape)) | |
| class MLP(nn.Module): | |
| def __init__(self, width): | |
| super().__init__() | |
| self.c_fc = nn.Linear(width, 4 * width, bias=False) | |
| self.c_proj = nn.Linear(4 * width, width, bias=False) | |
| def forward(self, x): | |
| return self.c_proj(F.relu(self.c_fc(x)).square()) | |
| class Block(nn.Module): | |
| def __init__(self, width, heads): | |
| super().__init__() | |
| self.attn = Attention(width, heads) | |
| self.mlp = MLP(width) | |
| def forward(self, x, rope): | |
| x = x + self.attn(norm(x), rope) | |
| return x + self.mlp(norm(x)) | |
| def per_position(x, weight): | |
| """x [rows, positions, in] through weight [positions, out, in]: one matrix per position.""" | |
| y = torch.bmm(x.transpose(0, 1), weight.transpose(1, 2).to(x.dtype)) | |
| return y.transpose(0, 1) | |
| class DepthBlock(nn.Module): | |
| """A depth block with its own weights at every codebook position. | |
| Same widths and arithmetic as `Block`; each [positions, out, in] matrix holds | |
| one nn.Linear weight per position, initialised as nn.Linear initialises. | |
| """ | |
| def __init__(self, positions, width, heads): | |
| super().__init__() | |
| self.heads = heads | |
| def weight(out, inner): | |
| bound = inner**-0.5 | |
| return nn.Parameter( | |
| torch.empty(positions, out, inner).uniform_(-bound, bound) | |
| ) | |
| self.c_q, self.c_k, self.c_v, self.c_proj = ( | |
| weight(width, width) for _ in range(4) | |
| ) | |
| self.c_fc = weight(4 * width, width) | |
| self.c_out = weight(width, 4 * width) | |
| def forward(self, x, rope): | |
| n, p, w = x.shape | |
| d = w // self.heads | |
| h = norm(x) | |
| q, k, v = [ | |
| per_position(h, m).reshape(n, p, self.heads, d) | |
| for m in (self.c_q, self.c_k, self.c_v) | |
| ] | |
| cos = rope[0, :p].to(x.dtype).unsqueeze(1) | |
| sin = rope[1, :p].to(x.dtype).unsqueeze(1) | |
| q, k = norm(rotate(q, cos, sin)) * 1.2, norm(rotate(k, cos, sin)) * 1.2 | |
| # Rotation and QK norm run at the residual precision; the kernel takes | |
| # the projection precision. | |
| y = depth_attention(q.to(v.dtype), k.to(v.dtype), v) | |
| x = x + per_position(y.reshape(n, p, w), self.c_proj) | |
| h = F.relu(per_position(norm(x), self.c_fc)).square() | |
| return x + per_position(h, self.c_out) | |
| class TextLayout: | |
| """Where text sits in a padded batch, built by `text_layout` and `collate_text`. | |
| source [batch, positions] indexes the concatenation of the audio steps | |
| [frames + 1], the text tokens [tokens] and one zero row for padding. | |
| token and track [batch, tokens] are the text tokens' ids and tracks. | |
| last [batch, frames + 1] is the position whose output conditions each step's depth. | |
| event_* [batch, events] are the text predictions: the position predicting it, its | |
| track, its target id and its loss weight (0 for padding and the flush step). | |
| pattern [batch] is the bitmask of present text tracks. | |
| """ | |
| source: torch.Tensor | |
| token: torch.Tensor | |
| track: torch.Tensor | |
| last: torch.Tensor | |
| event_position: torch.Tensor | |
| event_track: torch.Tensor | |
| event_target: torch.Tensor | |
| event_weight: torch.Tensor | |
| pattern: torch.Tensor | |
| def text_layout(frames, text): | |
| """One window's text layout as numpy arrays; text is per track None (absent) or an | |
| int array [n, 2] of (frame, id) rows in word order, frames in [0, frames).""" | |
| present = [j for j in range(TRACKS) if text[j] is not None] | |
| counts = np.zeros((frames + 1, TRACKS), dtype=np.int64) | |
| parts = [] | |
| for j in present: | |
| rows = np.asarray(text[j], dtype=np.int64).reshape(-1, 2) | |
| if len(rows) and (rows[:, 0].min() < 0 or rows[:, 0].max() >= frames): | |
| raise ValueError("Text frame outside the window") | |
| if len(rows) and (rows[:, 1].min() < 0 or rows[:, 1].max() >= TEXT_IDS): | |
| raise ValueError("Text id outside the vocabulary") | |
| np.add.at(counts[:, j], rows[:, 0], 1) | |
| parts.append( | |
| np.column_stack( | |
| (rows[:, 0], np.full(len(rows), j), np.arange(len(rows)), rows[:, 1]) | |
| ) | |
| ) | |
| tokens = np.concatenate(parts) if parts else np.zeros((0, 4), dtype=np.int64) | |
| # Frame, then track, then word order. | |
| tokens = tokens[np.lexsort((tokens[:, 2], tokens[:, 1], tokens[:, 0]))] | |
| per_step = counts.sum(1) | |
| start = np.concatenate(([0], np.cumsum(1 + per_step)[:-1])) | |
| before = np.concatenate(([0], np.cumsum(per_step))) | |
| positions = int(frames + 1 + per_step.sum()) | |
| rank = np.arange(len(tokens)) - before[tokens[:, 0]] | |
| token_position = start[tokens[:, 0]] + 1 + rank | |
| source = np.arange(frames + 1 + len(tokens)) | |
| order = np.empty(positions, dtype=np.int64) | |
| order[start] = np.arange(frames + 1) | |
| order[token_position] = source[frames + 1 :] | |
| # Events: each token from the input before it; each advance from the last input | |
| # of its track in the step, or where the track's first token would come from. | |
| position = [token_position - 1] | |
| track = [tokens[:, 1]] | |
| target = [tokens[:, 3]] | |
| weight = [np.ones(len(tokens))] | |
| written = np.zeros(frames + 1, dtype=np.int64) | |
| for j in present: | |
| written = written + counts[:, j] | |
| position.append(start + written) | |
| track.append(np.full(frames + 1, j)) | |
| target.append(np.full(frames + 1, ADVANCE)) | |
| weight.append(np.r_[np.full(frames, ADVANCE_WEIGHT), 0.0]) | |
| position, track, target, weight = ( | |
| np.concatenate(x) for x in (position, track, target, weight) | |
| ) | |
| # Sampling order: by position, then track. | |
| events = np.lexsort((track, position)) | |
| return { | |
| "source": order, | |
| "token": tokens[:, 3], | |
| "track": tokens[:, 1], | |
| "last": start + per_step, | |
| "event_position": position[events], | |
| "event_track": track[events], | |
| "event_target": target[events], | |
| "event_weight": weight[events].astype(np.float32), | |
| "pattern": sum(1 << j for j in present), | |
| } | |
| def collate_text(layouts, frames, positions, device): | |
| """Pads per-window layouts to [batch, positions] and [batch, frames + 1].""" | |
| b = len(layouts) | |
| tokens = max(1, max(len(x["token"]) for x in layouts)) | |
| events = max(1, max(len(x["event_target"]) for x in layouts)) | |
| pad = frames + 1 + tokens | |
| source = torch.full((b, positions), pad, dtype=torch.long) | |
| token = torch.zeros((b, tokens), dtype=torch.long) | |
| track = torch.zeros((b, tokens), dtype=torch.long) | |
| last = torch.zeros((b, frames + 1), dtype=torch.long) | |
| event = [torch.zeros((b, events), dtype=torch.long) for _ in range(3)] | |
| event_weight = torch.zeros((b, events)) | |
| for i, x in enumerate(layouts): | |
| steps, n, m = len(x["last"]), len(x["token"]), len(x["event_target"]) | |
| if len(x["source"]) > positions or steps > frames + 1: | |
| raise ValueError("Window exceeds the padded batch") | |
| s = torch.as_tensor(x["source"]) | |
| # Token sources are rebased past this batch's padded audio steps. | |
| source[i, : len(s)] = torch.where(s < steps, s, s - steps + frames + 1) | |
| token[i, :n] = torch.as_tensor(x["token"]) | |
| track[i, :n] = torch.as_tensor(x["track"]) | |
| last[i, :steps] = torch.as_tensor(x["last"]) | |
| for tensor, key in zip( | |
| event, ("event_position", "event_track", "event_target"), strict=True | |
| ): | |
| tensor[i, :m] = torch.as_tensor(x[key]) | |
| event_weight[i, :m] = torch.as_tensor(x["event_weight"]) | |
| pattern = torch.tensor([x["pattern"] for x in layouts]) | |
| return TextLayout( | |
| source.to(device), | |
| token.to(device), | |
| track.to(device), | |
| last.to(device), | |
| *(t.to(device) for t in event), | |
| event_weight.to(device), | |
| pattern.to(device), | |
| ) | |
| class DuplexModel(nn.Module): | |
| def __init__(self, config: Config): | |
| super().__init__() | |
| self.config = config | |
| c = config | |
| q = c.total_codebooks | |
| # One table per role with codebook offsets: a single gather instead of | |
| # one per codebook. Each track has its own rows. | |
| self.frame_embeddings = nn.EmbeddingBag(q * c.cardinality, c.width, mode="sum") | |
| # A zero BOS stays zero through bias-free blocks, amplifying RMSNorm's | |
| # derivative at zero until deep models overflow on the first backward. | |
| self.bos = nn.Parameter(torch.randn(c.width)) | |
| # Text: one token table for both tracks, a vector per track, an embedding of | |
| # which text tracks are present added to every audio position (patterns 1 to | |
| # 3; none adds nothing), and a query per track before the shared head. | |
| self.text_embeddings = nn.Embedding(TEXT_IDS, c.width) | |
| self.text_tracks = nn.Parameter(torch.zeros(TRACKS, c.width)) | |
| self.text_presence = nn.Parameter(torch.zeros(2**TRACKS - 1, c.width)) | |
| self.text_queries = nn.Parameter(torch.zeros(TRACKS, c.width)) | |
| if c.text_heads == "shared": | |
| self.text_head = nn.Linear(c.width, TEXT_IDS + 1, bias=False) | |
| else: | |
| # [track, events, width], initialised as nn.Linear initialises one head. | |
| bound = c.width**-0.5 | |
| self.text_head = nn.Parameter( | |
| torch.empty(TRACKS, TEXT_IDS + 1, c.width).uniform_(-bound, bound) | |
| ) | |
| self.temporal = nn.ModuleList( | |
| [Block(c.width, c.heads) for _ in range(c.layers)] | |
| ) | |
| self.to_depth = nn.Linear(c.width, c.depth_width, bias=False) | |
| self.depth_embeddings = nn.Embedding((q - 1) * c.cardinality, c.depth_width) | |
| self.depth_positions = nn.Parameter(torch.zeros(q, c.depth_width)) | |
| self.depth = nn.ModuleList( | |
| [DepthBlock(q, c.depth_width, c.depth_heads) for _ in range(c.depth_layers)] | |
| ) | |
| # Per-codebook output heads [codebook, cardinality, depth_width]. | |
| bound = c.depth_width**-0.5 | |
| self.outputs = nn.Parameter( | |
| torch.empty(q, c.cardinality, c.depth_width).uniform_(-bound, bound) | |
| ) | |
| self.register_buffer( | |
| "offsets", torch.arange(q) * c.cardinality, persistent=False | |
| ) | |
| # Codebook 0 of each track has delay zero; its acoustic codebooks one. | |
| self.delays = [int(k % c.codebooks != 0) for k in range(q)] | |
| self.register_buffer( | |
| "semantic", | |
| torch.tensor([d == 0 for d in self.delays]), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "track_of", torch.arange(q) // c.codebooks, persistent=False | |
| ) | |
| self.register_buffer( | |
| "temporal_rope", | |
| # One position more than the context flushes the last acoustic frame. | |
| rotary(c.context + 1, c.width // c.heads), | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "depth_rope", | |
| rotary(q, c.depth_width // c.depth_heads), | |
| persistent=False, | |
| ) | |
| def delayed(self, tokens, valid=None): | |
| """Token ids and presence for sequence steps 0..frames, [batch, frames + 1, codebooks]. | |
| Step s holds codebook 0 of frame s and the acoustic codebooks of frame | |
| s - 1 in every track. valid is [batch, frames, tracks]: a token is absent | |
| before the first frame, after the last, on padded frames and on absent tracks. | |
| """ | |
| b, t, _ = tokens.shape | |
| if valid is None: | |
| valid = tokens.new_ones((b, t, TRACKS), dtype=torch.bool) | |
| valid = valid[..., self.track_of] | |
| ids = torch.where( | |
| self.semantic, F.pad(tokens, (0, 0, 0, 1)), F.pad(tokens, (0, 0, 1, 0)) | |
| ) | |
| present = torch.where( | |
| self.semantic, | |
| F.pad(valid, (0, 0, 0, 1)), | |
| F.pad(valid, (0, 0, 1, 0)), | |
| ) | |
| return ids, present | |
| def audio_steps(self, tokens, valid=None): | |
| """Temporal inputs of the audio positions, [batch, frames + 1, width]: BOS, then | |
| the sum of every present token of the step before.""" | |
| b, t, q = tokens.shape | |
| if q != self.config.total_codebooks: | |
| raise ValueError("Expected [batch, frames, codebooks x tracks]") | |
| x = self.bos.expand(b, 1, -1) | |
| if t: | |
| ids, present = self.delayed(tokens, valid) | |
| weights = self.frame_embeddings.weight | |
| embedded = self.frame_embeddings( | |
| (ids[:, :t] + self.offsets).reshape(-1, q), | |
| per_sample_weights=present[:, :t].reshape(-1, q).to(weights.dtype), | |
| ) | |
| x = torch.cat((x, embedded.reshape(b, t, -1)), dim=1) | |
| return x | |
| def temporal_outputs(self, tokens, valid=None, text=None): | |
| """Normalised temporal outputs [batch, positions, width].""" | |
| x = self.audio_steps(tokens, valid) | |
| if text is not None: | |
| b, _, w = x.shape | |
| presence = torch.cat((x.new_zeros(1, w), self.text_presence)) | |
| x = x + presence[text.pattern][:, None] | |
| words = self.text_embeddings(text.token) + self.text_tracks[text.track] | |
| table = torch.cat((x, words.to(x.dtype), x.new_zeros(b, 1, w)), dim=1) | |
| x = table.gather(1, text.source[..., None].expand(-1, -1, w)) | |
| if x.shape[1] > self.config.context + 1: | |
| raise ValueError("Window exceeds the temporal context") | |
| x = norm(x) | |
| for block in self.temporal: | |
| x = block(x, self.temporal_rope) | |
| return norm(x) | |
| def text_logits(self, hidden, track): | |
| """Logits [..., TEXT_IDS + 1] of text events from normalised temporal outputs.""" | |
| x = norm(hidden + self.text_queries[track].to(hidden.dtype)) | |
| if self.config.text_heads == "shared": | |
| return self.text_head(x) | |
| weights = self.text_head.to(x.dtype) | |
| track = torch.as_tensor(track, device=x.device) | |
| if track.dim() == 0: | |
| return x @ weights[track].T | |
| # Both heads for every event, each event keeping its own track's: fixed | |
| # shapes, so the compiled step stays fast (mask selection has | |
| # data-dependent shapes and trained at about half the step rate). | |
| return torch.where(track[..., None] == 0, x @ weights[0].T, x @ weights[1].T) | |
| def depth_logits(self, context, inputs): | |
| """Logits [batch, steps, codebooks, cardinality] in delayed step order. | |
| inputs is [batch, steps, codebooks - 1, depth_width]: the embedding of | |
| the token before each depth position, zero where there is none. | |
| """ | |
| b, s, _ = context.shape | |
| q = self.config.total_codebooks | |
| x = context.unsqueeze(2) + self.depth_positions | |
| x = torch.cat((x[:, :, :1], x[:, :, 1:] + inputs), dim=2) | |
| x = x.reshape(b * s, q, -1) | |
| for block in self.depth: | |
| x = block(x, self.depth_rope) | |
| # [codebook, steps, width] x [codebook, width, cardinality] | |
| x = norm(x).transpose(0, 1) | |
| logits = torch.bmm(x, self.outputs.transpose(1, 2).to(x.dtype)) | |
| return logits.transpose(0, 1).reshape(b, s, q, -1) | |
| def forward(self, tokens, valid=None, text=None): | |
| """Canonical frames in; audio logits aligned to the same [batch, frames, | |
| codebooks], and text logits [batch, events, TEXT_IDS + 1] or None. | |
| valid is the [batch, frames, tracks] mask of real frames of present tracks. | |
| text is a `TextLayout` or None for audio only. | |
| Fillers never appear as targets: codebook 0 of frame t is read from step | |
| t and its acoustic codebooks from step t + 1, so the final step flushes | |
| the last frame and every source token is scored once. | |
| """ | |
| hidden = self.temporal_outputs(tokens, valid, text) | |
| steps = hidden | |
| if text is not None: | |
| w = hidden.shape[-1] | |
| steps = hidden.gather(1, text.last[..., None].expand(-1, -1, w)) | |
| context = self.to_depth(steps) | |
| ids, present = self.delayed(tokens, valid) | |
| # Each depth position after the first takes the token before it in | |
| # step order. The step that flushes a short row's last frame sees no | |
| # codebook 0, as in generation, instead of batch padding. | |
| inputs = self.depth_embeddings(ids[:, :, :-1] + self.offsets[:-1]) | |
| inputs = inputs * present[:, :, :-1, None].to(inputs.dtype) | |
| logits = self.depth_logits(context, inputs) | |
| audio = torch.where(self.semantic[:, None], logits[:, :-1], logits[:, 1:]) | |
| if text is None: | |
| return audio, None | |
| w = hidden.shape[-1] | |
| events = hidden.gather(1, text.event_position[..., None].expand(-1, -1, w)) | |
| return audio, self.text_logits(events, text.event_track) | |
| def generate(self, prefix, frames, temperature=1.0, top_k=250): | |
| """Reference audio sampler without KV cache or text; returns canonical tokens.""" | |
| if frames < 0 or temperature < 0 or top_k < 0: | |
| raise ValueError("frames, temperature and top_k must be nonnegative") | |
| c = self.config | |
| b, forced, q = prefix.shape | |
| result = torch.cat((prefix, prefix.new_zeros((b, frames, q))), dim=1) | |
| if not frames: | |
| return result | |
| # One step past the last frame flushes its acoustic codebooks. | |
| for step in range(forced, forced + frames + 1): | |
| window = result[:, max(0, step - c.context) : step] | |
| context = self.to_depth(self.temporal_outputs(window)[:, -1:]) | |
| inputs = context.new_zeros((b, 1, q - 1, c.depth_width)) | |
| for position, delay in enumerate(self.delays): | |
| frame = step - delay | |
| if not 0 <= frame < forced + frames: | |
| continue | |
| # The prefix keeps its own tokens, including the acoustic | |
| # codebooks of its last frame; later frames are sampled. | |
| if frame >= forced: | |
| logits = self.depth_logits(context, inputs)[:, 0, position].float() | |
| result[:, frame, position] = sample( | |
| logits, temperature, top_k | |
| ).squeeze(-1) | |
| if position < q - 1: | |
| inputs[:, 0, position] = self.depth_embeddings( | |
| result[:, frame, position] + self.offsets[position] | |
| ) | |
| return result | |
| def sample(logits, temperature, top_k=0, generator=None): | |
| """One draw per row of [..., classes] logits; top_k 0 keeps every class.""" | |
| if temperature == 0: | |
| return logits.argmax(-1, keepdim=True) | |
| if top_k and top_k < logits.shape[-1]: | |
| floor = logits.topk(top_k).values[..., -1:] | |
| logits = logits.masked_fill(logits < floor, float("-inf")) | |
| probabilities = (logits / temperature).softmax(-1) | |
| return torch.multinomial( | |
| probabilities.reshape(-1, logits.shape[-1]), 1, generator=generator | |
| ).reshape(*logits.shape[:-1], 1) | |
| def from_legacy(config, state): | |
| """A model in this format from a 14 November checkpoint (one or two streams, | |
| shared depth weights, no text). | |
| A two-stream checkpoint converts exactly: the shared depth block weights are | |
| copied to every position and the new text weights stay unused while no text | |
| is present. A one-stream checkpoint is widened as two-stream training began: | |
| each track gets a copy of the stream's embeddings, positions and heads, and | |
| track 1's codebook 7 feeding track 2's codebook 0 starts at zero. | |
| """ | |
| config = dict(config) | |
| config.pop("factorization", None) | |
| streams = config.pop("streams", 1) | |
| if streams not in (1, 2): | |
| raise ValueError("Expected a one- or two-stream checkpoint") | |
| model = DuplexModel(Config(**config)) | |
| c = model.config | |
| new = model.state_dict() | |
| state = dict(state) | |
| if streams == 1: | |
| for name in ("frame_embeddings.weight", "depth_positions", "outputs"): | |
| value = state[name] | |
| state[name] = value.repeat(TRACKS, *[1] * (value.dim() - 1)) | |
| value = state["depth_embeddings.weight"] | |
| joint = value.new_zeros((c.cardinality, value.shape[1])) | |
| state["depth_embeddings.weight"] = torch.cat([value, joint, value]) | |
| names = { | |
| "attn.c_q": "c_q", | |
| "attn.c_k": "c_k", | |
| "attn.c_v": "c_v", | |
| "attn.c_proj": "c_proj", | |
| "mlp.c_fc": "c_fc", | |
| "mlp.c_proj": "c_out", | |
| } | |
| for layer in range(c.depth_layers): | |
| for old, name in names.items(): | |
| weight = state.pop(f"depth.{layer}.{old}.weight") | |
| state[f"depth.{layer}.{name}"] = weight.repeat(c.total_codebooks, 1, 1) | |
| for name, value in new.items(): | |
| if name.startswith("text_"): | |
| state[name] = value | |
| model.load_state_dict(state, strict=True) | |
| return model | |
| def with_text(model, text): | |
| """The same weights under another training option, such as a later stage's.""" | |
| widened = DuplexModel(replace(model.config, text=text)).to(model.bos.device) | |
| widened.load_state_dict(model.state_dict(), strict=True) | |
| return widened | |