"""EmbeddingGemma 2 audio tower + learned layer mix + transformer reader + CTC head (+ optional RNN-T head). Recipe follows QUAN-Transcribe (Vietnamese, EG2): fine-tune the top audio layers, mix all layer outputs, read them with a small transformer, train with CTC over a reduced Gemma vocabulary. The reader is trained with dynamic chunk masks so the same weights run offline and streaming. Stage 5 adds a transducer head on the same reader output; both heads share the vocabulary. Push 8 adds an optional code-mix fork: mode=codemix's own post-tower stack on the shared tower (add_codemix_fork, encode_cm). """ from __future__ import annotations import copy import random from pathlib import Path import torch import torch.nn.functional as F from torch import nn from torch.utils.checkpoint import checkpoint from transformers import AutoConfig, AutoModel from transformers.masking_utils import create_bidirectional_mask from transformers.models.gemma4.modeling_gemma4 import sliding_window_mask_function from .tower_ctx import set_tower_context # PACKAGE CHANGE (stage7-final): the tower is built from EmbeddingGemma 2's config.json in a local folder, with no Hugging Face # download and no base weights: the release weights file holds every tensor, and transcribe.py loads it strictly. Default: # safetensors/eg2 of this release; transcribe.py points it at the folder it uses (for a GGUF, one written from the file's metadata). BASE_MODEL = str(Path(__file__).resolve().parents[2] / "safetensors" / "eg2") CODEMIX_HEADS = {"transducer_cm": "transducer", "ctc_head_cm": "ctc_head"} # code-mix head -> the Tamil head it starts as FORK = "fork_cm" # state-dict prefix of the code-mix fork (add_codemix_fork) FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm") # the post-tower stack, encode()'s order def codemix_heads_in(state: dict) -> dict[str, bool]: """Which code-mix heads a checkpoint's state dict holds, by key prefix: {"transducer_cm": bool, "ctc_head_cm": bool}.""" return {h: any(k.startswith(h + ".") for k in state) for h in CODEMIX_HEADS} def codemix_fork_in(state) -> bool: """Whether a checkpoint's state dict (or any collection of its keys) holds the code-mix fork: a key fork_cm.*.""" return any(k.startswith(FORK + ".") for k in state) def chunk_mask(lengths: torch.Tensor, max_len: int, chunk: int | None) -> torch.Tensor: """Boolean attention mask (B, 1, T, T); True = may attend. chunk=None gives full attention over valid frames. Otherwise a frame sees every frame in its own chunk and all earlier chunks (block-causal), which is what streaming inference can compute. """ idx = torch.arange(max_len, device=lengths.device) valid = idx[None, :] < lengths[:, None] mask = valid[:, None, None, :].expand(-1, 1, max_len, -1) if chunk is not None: q_chunk = idx[:, None] // chunk k_chunk = idx[None, :] // chunk mask = mask & (k_chunk <= q_chunk)[None, None] return mask class CausalConvPosition(nn.Module): """Depthwise causal conv that adds relative position information (streaming-safe).""" def __init__(self, dim: int, kernel: int = 15): super().__init__() self.pad = kernel - 1 self.conv = nn.Conv1d(dim, dim, kernel, groups=dim) def forward(self, x: torch.Tensor) -> torch.Tensor: y = F.pad(x.transpose(1, 2), (self.pad, 0)) return x + F.gelu(self.conv(y)).transpose(1, 2) class Transducer(nn.Module): """RNN-T head: LSTM prediction network + additive joint over the reader output. Index 0 is the blank and also the start symbol, as in the CTC vocabulary, so both heads share the tokenizer and either one decodes on its own. The prediction network only sees past tokens and the joint is per frame, so the head streams whenever the reader does. styles > 0 adds a register embedding: a per-clip id summed into the prediction network's output. Id 0 is neutral and its row stays zero; the other rows start at zero, so a checkpoint without them decodes as before until they train. With the default styles = 0 there is no parameter and no extra step. delay_penalty and first_only are loss settings from the same config block (B5, see loss_and_grads). They hold no weights and decoding never reads them: train.py passes them to loss_and_grads. They are taken here only so that every script that builds the head from a checkpoint's cfg (evaluate, eval_dump, pick_ckpt, exports) still builds it. """ def __init__(self, vocab_size: int, enc_dim: int, pred_dim: int = 640, joint_dim: int = 640, dropout: float = 0.1, styles: int = 0, delay_penalty: float = 0.0, first_only: bool = False): super().__init__() del delay_penalty, first_only # loss settings only, read by train.py (see the docstring) self.embed = nn.Embedding(vocab_size, pred_dim) self.lstm = nn.LSTM(pred_dim, pred_dim, batch_first=True) self.drop = nn.Dropout(dropout) self.enc_proj = nn.Linear(enc_dim, joint_dim) self.pred_proj = nn.Linear(pred_dim, joint_dim) self.out = nn.Linear(joint_dim, vocab_size) # from_pretrained draws no random numbers, so every other weight starts the same with or without styles. self.style = nn.Embedding.from_pretrained(torch.zeros(styles, pred_dim), freeze=False, padding_idx=0) if styles else None def predict(self, tokens: torch.Tensor, state=None, style: torch.Tensor | None = None): """Previous tokens (B, U) -> prediction features in joint space (B, U, J) and the LSTM state. style (B,): register ids.""" y, state = self.lstm(self.drop(self.embed(tokens)), state) if style is not None: if self.style is None: raise ValueError("a style id was given, but this head has no style embedding (transducer: styles in the config)") y = y + self.style(style)[:, None] return self.pred_proj(self.drop(y)), state def joint(self, enc: torch.Tensor, pred: torch.Tensor) -> torch.Tensor: return self.out(torch.tanh(enc + pred)) def loss_and_grads(self, x, lengths, targets, target_lengths, scale: float = 1.0, max_cells: int = 100_000, style: torch.Tensor | None = None, delay_penalty: float = 0.0, first_only: bool = False, fused: bool | None = None): """RNN-T loss per utterance / its target length, averaged over the batch (like F.ctc_loss "mean"). The (B, T, U+1, V) joint tensor is built a few utterances at a time (≤ max_cells of T × (U+1)) from detached copies of the encoder and prediction features, and each slice is backpropagated at once (`scale` × loss), which bounds memory. The joint's own weights get their gradients here; the caller passes the returned (features, gradients) to torch.autograd.backward together with the CTC loss. Call outside autocast. style (B,): register ids, as in predict. delay_penalty λ > 0, the delay-penalized transducer (Kang et al. 2023; B5, methods scout 2026-11-09 §2.1): the log-softmax runs here instead of inside the loss, λ((T−1)/2 − t) is added to every non-blank log-prob of frame t (T: that utterance's frame count), and the loss reads those log-probs as they are (fused_log_softmax=False). A token emitted before mid-clip gains, a later one pays. first_only: the offset goes on the first token's row (u = 0) only, so only the start of emission is pushed. The returned loss includes the penalty: it does not compare with a run without one. The unfused path holds one more fp32 copy of each slice's joint tensor (cells × V × 4 bytes). 0 (default) keeps torchaudio's fused path, unchanged. fused=False takes the unfused path at λ = 0 too, for scripts/test_delay_penalty.py; None (default) is fused unless λ is set. """ from torchaudio.functional import rnnt_loss if fused and delay_penalty: raise ValueError("a delay penalty needs the unfused path: it goes onto normalised log-probs") if first_only and not delay_penalty: raise ValueError("first_only without delay_penalty changes nothing") unfused = bool(delay_penalty) or fused is False with torch.autocast("cuda", dtype=torch.bfloat16): enc = self.enc_proj(x) pred, _ = self.predict(F.pad(targets, (1, 0)), style=style) # start symbol = blank e, p = enc.detach().requires_grad_(), pred.detach().requires_grad_() total, n = 0.0, x.shape[0] for idx in _slices(lengths, target_lengths, max_cells): T, U = int(lengths[idx].max()), int(target_lengths[idx].max()) with torch.autocast("cuda", dtype=torch.bfloat16): logits = self.joint(e[idx, :T, None], p[idx, None, : U + 1]).float() # the bf16 copy is freed here if unfused: logits = logits.log_softmax(-1) # rebinding frees the fp32 logits: log_softmax keeps only its output if delay_penalty: logits = _delay_offset(logits, lengths[idx], delay_penalty, first_only) loss = rnnt_loss(logits, targets[idx, :U].int(), lengths[idx].int(), target_lengths[idx].int(), blank=0, reduction="none", fused_log_softmax=not unfused) del logits # RnntLoss keeps only its gradients; drop the fp32 copy before backward (K3 OOM fix, 2026-11-08) loss = (loss / target_lengths[idx]).sum() / n (scale * loss).backward() total += loss.item() return total, (enc, pred), (e.grad, p.grad) @torch.no_grad() def greedy(self, x: torch.Tensor, lengths: torch.Tensor, max_symbols: int = 5, allowed: torch.Tensor | None = None, swap: bool = False, start_penalty: float = 0.0, cue: torch.Tensor | None = None, style: int | None = None, all_frame_penalty: float = 0.0, frames_out: list | None = None) -> list[list[int]]: """Batched greedy decoding: at each frame emit tokens until blank (at most max_symbols). allowed (V) or (B, V) bool: the caller's language hint, the pieces that may be emitted (blank must be among them; SPMTokenizer.script_ids() gives the script classes). The others get log-prob -inf, so where the model wants one, blank usually wins and the word is lost. swap: where the unmasked argmax is a forbidden piece, emit the best allowed piece instead of letting blank win; at most one swap per frame, at the frame's first symbol step (later on, a forbidden wish ends the frame). Nothing without allowed. start_penalty: taken off blank's logit while a clip has emitted nothing, for clips that fail to start. cue (B, T) bool, from the caller: the penalty applies only on frames where it is True, e.g. once the CTC head has left blank on that clip. It comes before allowed, so the mask and swap see the penalised blank. 0 (default) is plain greedy, whatever cue says. style: one register id for the whole batch; None (default) is neutral. all_frame_penalty: taken off blank's logit at every symbol step of every clip, started or not; cue does not gate it. It adds to start_penalty and also comes before allowed. 0 (default) changes nothing. frames_out: a list the caller owns; it is extended with one list per clip, the frame of each emitted token (for scripts/decode_ctc_assist.py). None (default) records nothing; the hypotheses are the same either way. """ B, T = x.shape[:2] enc = self.enc_proj(x) style = None if style is None else torch.full((B,), style, device=x.device) pred, state = self.predict(torch.zeros(B, 1, dtype=torch.long, device=x.device), style=style) pred = pred[:, 0] started = torch.zeros(B, dtype=torch.bool, device=x.device) hyps = [[] for _ in range(B)] frames = [[] for _ in range(B)] ban = None if allowed is not None: if not allowed[..., 0].all(): raise ValueError("allowed must keep blank (index 0): without it a clip can never end a frame") ban = ~allowed.to(x.device).expand(B, -1) idx = torch.arange(B, device=x.device) for t in range(T): live = lengths > t for s in range(max_symbols): logits = self.joint(enc[:, t], pred) if start_penalty or all_frame_penalty: waiting = ~started if cue is None else ~started & cue[:, t] logits[:, 0] = logits[:, 0] - (all_frame_penalty + start_penalty * waiting.to(logits.dtype)) if ban is not None: wished_banned = ban[idx, logits.argmax(-1)] logits = logits.masked_fill(ban, float("-inf")) if swap: forced = float("-inf") if s == 0 else float("inf") logits[:, 0] = torch.where(wished_banned, torch.full_like(logits[:, 0], forced), logits[:, 0]) k = logits.argmax(-1) emit = live & (k != 0) if not emit.any(): break started = started | emit new_pred, new_state = self.predict(k[:, None], state, style) pred = torch.where(emit[:, None], new_pred[:, 0], pred) state = tuple(torch.where(emit[None, :, None], n, s) for n, s in zip(new_state, state)) for b, tok in zip(emit.nonzero()[:, 0].tolist(), k[emit].tolist()): hyps[b].append(tok) frames[b].append(t) live = emit if frames_out is not None: frames_out.extend(frames) return hyps def _delay_offset(logp: torch.Tensor, lengths: torch.Tensor, delay_penalty: float, first_only: bool) -> torch.Tensor: """Log-probs (b, T, U+1, V) plus λ((T_i − 1)/2 − t) on each non-blank entry of frame t (blank = 0): on every row u, or with first_only on u = 0 only. T_i is utterance i's frame count; frames past it are never read by the loss. Out of place, since log_softmax keeps logp for its backward. Both forms pass the gradient through unchanged, with no full-size temporary: the offset itself is (b, T, 1, V).""" t = torch.arange(logp.shape[1], device=logp.device, dtype=logp.dtype) off = delay_penalty * ((lengths[:, None].to(logp.dtype) - 1) / 2 - t) # (b, T) off = F.pad(off[:, :, None, None].expand(-1, -1, 1, logp.shape[-1] - 1), (1, 0)) # (b, T, 1, V), 0 on blank if first_only: return logp.index_add(2, torch.zeros(1, dtype=torch.long, device=logp.device), off) return logp + off def _slices(lengths: torch.Tensor, target_lengths: torch.Tensor, max_cells: int) -> list[torch.Tensor]: """Consecutive index groups whose padded joint size B × max T × max (U+1) stays within max_cells.""" out, cur, t_max, u_max = [], [], 0, 0 for i, (t, u) in enumerate(zip(lengths.tolist(), (target_lengths + 1).tolist())): if cur and (len(cur) + 1) * max(t_max, t) * max(u_max, u) > max_cells: out.append(cur) cur, t_max, u_max = [], 0, 0 cur.append(i) t_max, u_max = max(t_max, t), max(u_max, u) out.append(cur) return [torch.tensor(c, device=lengths.device) for c in out] class EG2SpeechModel(nn.Module): def __init__( self, vocab_size: int, train_tower_layers: int = 2, reader_layers: int = 6, reader_dim: int = 768, reader_heads: int = 12, dropout: float = 0.1, stream_chunk_frames: int = 12, stream_prob: float = 0.5, grad_checkpointing: bool = True, zero_init_residual: bool = True, transducer: dict | None = None, offline_right: int = 0, ): super().__init__() self.tower = AutoModel.from_config(AutoConfig.from_pretrained(BASE_MODEL).audio_config, dtype=torch.float32) # PACKAGE CHANGE cfg = self.tower.config n_states = cfg.num_hidden_layers + 1 self.layer_logits = nn.Parameter(torch.zeros(n_states)) self.in_norm = nn.LayerNorm(cfg.hidden_size) self.in_proj = nn.Linear(cfg.hidden_size, reader_dim) self.position = CausalConvPosition(reader_dim) layer = nn.TransformerEncoderLayer( reader_dim, reader_heads, 4 * reader_dim, dropout=dropout, activation="gelu", batch_first=True, norm_first=True, ) self.reader = nn.TransformerEncoder(layer, reader_layers, enable_nested_tensor=False) self.reader_heads = reader_heads if zero_init_residual: # Each reader block starts as identity, so CTC sees the tower features from step 0 # and the reader only learns a correction (avoids a long all-blank plateau). for blk in self.reader.layers: for lin in (blk.self_attn.out_proj, blk.linear2): nn.init.zeros_(lin.weight) nn.init.zeros_(lin.bias) self.out_norm = nn.LayerNorm(reader_dim) self.ctc_head = nn.Linear(reader_dim, vocab_size) # index 0 = CTC blank self.transducer = Transducer(vocab_size, reader_dim, **transducer) if transducer else None self.transducer_cm = None # add_codemix_head() self.ctc_head_cm = None # add_codemix_head(ctc=True) self.fork_cm = None # add_codemix_fork() self.stream_chunk_frames = stream_chunk_frames self.stream_prob = stream_prob self.offline_right = offline_right # dual-mode: tower right context on offline passes; streaming passes always run at right 0 self.grad_checkpointing = grad_checkpointing self.set_trainable_tower_layers(train_tower_layers) # ---- freezing ------------------------------------------------------------------------- def set_trainable_tower_layers(self, n: int | str) -> None: """Train only the top-n conformer layers ("all" = whole tower incl. subsampling).""" total = self.tower.config.num_hidden_layers n = total if n == "all" else int(n) self.first_trainable = total - n for p in self.tower.parameters(): p.requires_grad = False for layer in self.tower.layers[self.first_trainable:]: for p in layer.parameters(): p.requires_grad = True if n == total: for p in self.tower.subsample_conv_projection.parameters(): p.requires_grad = True self.tower.output_proj.requires_grad_(False) # unused by the STT path def add_codemix_head(self, ctc: bool = False) -> None: """Add the mode=codemix RNN-T head, transducer_cm, and with ctc=True the code-mix CTC head, ctc_head_cm: their own weights on the same reader output (encode_cm's on a model with add_codemix_fork), so the encoder and the mode=tamil heads stay as they are. They start as copies of transducer and ctc_head at this moment, so an untrained code-mix head decodes exactly as its Tamil twin. State-dict keys: transducer_cm.* (the transducer's names) and ctc_head_cm.weight, ctc_head_cm.bias. param_groups puts transducer_cm in the transducer group and ctc_head_cm in the head group. Order: call this before load_state_dict, then copy_missing_codemix(missing): a checkpoint with the heads fills them, one without them leaves them missing and copy_missing_codemix copies the just-loaded Tamil heads into them (never the random init of a model built before the load). A model that never calls this ignores transducer_cm.* and ctc_head_cm.* in a checkpoint (unexpected keys under strict=False), so its mode=tamil output cannot change.""" if self.transducer is None: raise ValueError("the code-mix head starts as a copy of the transducer head, and this model has none") if self.transducer_cm is not None or self.ctc_head_cm is not None: raise ValueError("the code-mix heads already exist: a second copy would overwrite their weights") self.transducer_cm = copy.deepcopy(self.transducer) if ctc: self.ctc_head_cm = copy.deepcopy(self.ctc_head) @torch.no_grad() def copy_missing_codemix(self, missing: list[str], log: bool = True) -> list[str]: """After load_state_dict(state, strict=False), with its `missing` list: each code-mix head this model has whose keys are all missing is set to a copy of its Tamil twin, as loaded. Returns the heads copied (and prints one line each), so an old checkpoint gives code-mix heads equal to its Tamil heads. A head with only some keys missing is an error: the checkpoint holds a different head.""" missing, copied = set(missing), [] for name, twin in CODEMIX_HEADS.items(): head = getattr(self, name) if head is None: continue keys = {f"{name}.{k}" for k in head.state_dict()} gone = keys & missing if not gone: continue if gone != keys: raise ValueError(f"the checkpoint holds only part of {name}: missing {sorted(gone)[:5]}") head.load_state_dict(getattr(self, twin).state_dict()) copied.append(name) if log: print(f"{name} not in the checkpoint: starts as a copy of the loaded {twin}", flush=True) return copied def add_codemix_fork(self) -> None: """Add the code-mix fork, fork_cm (push 8 plan section 1): mode=codemix's own copy of the post-tower stack, every module between the tower and the heads (FORK_MODULES: layer_logits, in_norm, in_proj, position, reader with all its layers, out_norm). Deep copies of those modules at this moment, so a fork added after a load is the loaded stack and encode_cm then gives encode()'s output bit for bit. The tower is shared, not copied. State-dict keys: fork_cm. + the stack's own names (81 tensors, 43,330,317 values with the 6-layer 768-d reader on the 1024-d tower). encode() and the Tamil heads never read the fork, so mode=tamil output is the same with or without it; encode_cm reads it, for the code-mix heads. Order: as add_codemix_head, call this before load_state_dict when a checkpoint holds fork_cm.* (codemix_fork_in), so its weights replace the copies. Nothing fills a fork the checkpoint lacks: those keys stay missing, and fork_cm.* keys in a model without the fork stay unexpected; every loader (train.load_weights, scripts/eval_dump.py) refuses both.""" if self.fork_cm is not None: raise ValueError("the code-mix fork already exists: a second copy would overwrite its weights") fork = nn.Module() for name in FORK_MODULES: setattr(fork, name, copy.deepcopy(getattr(self, name))) # a Parameter stays a Parameter, requires_grad as it was self.fork_cm = fork def encode_for(self, head: str): """The encoder a head set reads: encode_cm for head="cm" on a model with the fork, else encode (both heads of a model without the fork read encode(), as before push 8).""" if head not in ("ta", "cm"): raise ValueError(f"head must be 'ta' or 'cm', not {head!r}") return self.encode_cm if head == "cm" and self.fork_cm is not None else self.encode def param_groups(self, lr_tower: float, lr_head: float, lr_reader: float, weight_decay: float, lr_transducer: float | None = None) -> list[dict]: """Three LRs. The from-scratch transformer reader needs a low LR (overfit test: 1e-3 stalls in an all-blank plateau, 5e-5 memorises 32 clips in 200 steps); the linear layers take more. A transducer head, and a code-mix head, get a fourth group (lr_transducer, default lr_head), appended last; the code-mix CTC head (ctc_head_cm) is in the head group with ctc_head. The fork's modules (add_codemix_fork) go where their Tamil twins go: fork_cm.reader.* with reader, the rest of the fork with head.""" groups = {"tower": [], "reader": [], "head": [], "transducer": []} for n, p in self.named_parameters(): if p.requires_grad: groups["tower" if n.startswith("tower.") else "reader" if n.startswith(("reader.", FORK + ".reader.")) else "transducer" if n.startswith(("transducer.", "transducer_cm.")) else "head"].append(p) out = [ {"params": groups["tower"], "lr": lr_tower, "weight_decay": weight_decay}, {"params": groups["head"], "lr": lr_head, "weight_decay": weight_decay}, {"params": groups["reader"], "lr": lr_reader, "weight_decay": weight_decay}, ] if groups["transducer"]: out.append({"params": groups["transducer"], "lr": lr_transducer or lr_head, "weight_decay": weight_decay}) return out # ---- forward -------------------------------------------------------------------------- def encode_tower(self, feats: torch.Tensor, feat_mask: torch.Tensor): """Replicates Gemma4AudioModel.forward but returns every layer and supports checkpointing.""" tower, cfg = self.tower, self.tower.config grad_sub = self.first_trainable == 0 and self.training with torch.set_grad_enabled(grad_sub and torch.is_grad_enabled()): hs, out_mask = tower.subsample_conv_projection(feats, feat_mask) pos = tower.rel_pos_enc(hs) attn = create_bidirectional_mask( config=cfg, inputs_embeds=hs, attention_mask=out_mask, and_mask_function=sliding_window_mask_function( (cfg.attention_context_left - 1, cfg.attention_context_right) ), ) if attn is not None: attn = tower._convert_4d_mask_to_blocked_5d(attn) states = [hs] for i, layer in enumerate(tower.layers): trainable = i >= self.first_trainable and self.training and torch.is_grad_enabled() if not trainable: with torch.no_grad(): hs = layer(hs, attention_mask=attn, position_embeddings=pos) elif self.grad_checkpointing: hs = checkpoint(layer, hs, attn, pos, use_reentrant=False) else: hs = layer(hs, attention_mask=attn, position_embeddings=pos) states.append(hs) return states, out_mask def forward(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): """Returns CTC log-probs (B, T, V) float32 and frame lengths (B,).""" x, lengths = self.encode(feats, feat_mask, streaming) return self.ctc_log_probs(x), lengths def ctc_log_probs(self, x: torch.Tensor, head: str = "ta") -> torch.Tensor: """CTC log-probs (B, T, V) float32 of the Tamil head (head="ta", default) or the code-mix head (head="cm").""" if head == "ta": lin = self.ctc_head elif head == "cm": lin = self.ctc_head_cm if lin is None: raise ValueError("head='cm' needs the code-mix CTC head: add_codemix_head(ctc=True)") else: raise ValueError(f"head must be 'ta' or 'cm', not {head!r}") return lin(x).float().log_softmax(-1) # PACKAGE CHANGE (STT-1.2): encode() is split into mix() + read(), the same operations in the same order, so its output is # bit-identical. A streaming caller keeps mix()'s output and runs read() twice: block-causal for the partials, then once with # full attention at the end of the utterance for the two-pass final (transcribe.py --finals two-pass). def encode(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): """Normalized reader output (B, T, D), which both heads read, and frame lengths (B,): read(mix(...)).""" if streaming is None: streaming = self.training and random.random() < self.stream_prob x, lengths = self.mix(feats, feat_mask, streaming) return self.read(x, lengths, streaming) def mix(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool = False): """The reader's input (B, T, D) and frame lengths (B,): the audio tower, the learned layer mix, in_proj and the causal position conv. With offline_right = 0 (every model but B4's, and every release) nothing here depends on `streaming`, so one mix() output serves both read() passes; with offline_right > 0 the tower differs between the modes, and a two-pass caller must not share it.""" if self.offline_right: # 0, every run but B4: no call, the tower stays as loaded set_tower_context(self.tower, right=0 if streaming else self.offline_right) states, out_mask = self.encode_tower(feats, feat_mask) w = torch.softmax(self.layer_logits, dim=0) mixed = sum(w[i] * s for i, s in enumerate(states)) x = self.in_proj(self.in_norm(mixed)) return self.position(x), out_mask.sum(-1) def read(self, x: torch.Tensor, lengths: torch.Tensor, streaming: bool = False): """The transformer reader over mix()'s output and out_norm: normalized reader output (B, T, D) and the same lengths. streaming: the block-causal chunk_mask (stream_chunk_frames per block), else full attention over the valid frames.""" if len(self.reader.layers): allowed = chunk_mask(lengths, x.shape[1], self.stream_chunk_frames if streaming else None) # nn.TransformerEncoder expects True = masked out, shape (B*heads, T, T) block = (~allowed).repeat_interleave(self.reader_heads, dim=0).squeeze(1) x = self.reader(x, mask=block) return self.out_norm(x), lengths # PACKAGE CHANGE (STT-2): the project's encode_cm() (push 8) is split into mix_cm() + read_cm(), as encode() is split into # mix() + read() above: the same operations in the same order, so its output is bit-identical. The code-mix partials # (read_cm, block-causal) and the two-pass final (read_cm, full attention) share one mix_cm() output. mix(), read() and # encode(), the mode=tamil path, are not changed. def encode_cm(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): """mode=codemix's reader output (B, T, D), which the code-mix heads read, and frame lengths (B,): encode()'s ops in encode()'s order on the same encode_tower() and the same chunk_mask (block-causal at stream_chunk_frames when streaming), with the fork's modules (add_codemix_fork) in place of the Tamil stack. encode() itself is not changed. A caller that runs both on one batch passes streaming: None draws its own random number in training, as encode(). read_cm(mix_cm(...)).""" if self.fork_cm is None: raise ValueError("encode_cm needs the code-mix fork: add_codemix_fork()") if streaming is None: streaming = self.training and random.random() < self.stream_prob x, lengths = self.mix_cm(feats, feat_mask, streaming) return self.read_cm(x, lengths, streaming) def mix_cm(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool = False): """mix() with the fork: the shared audio tower (encode_tower), then the fork's layer mix, in_norm, in_proj and causal position conv. The reader's input (B, T, D) for read_cm() and frame lengths (B,). As in mix(), nothing here depends on `streaming` when offline_right = 0, so one mix_cm() output serves both read_cm() passes.""" fork = self.fork_cm if fork is None: raise ValueError("mix_cm needs the code-mix fork: add_codemix_fork()") if self.offline_right: # as encode(): 0 in every run but B4 set_tower_context(self.tower, right=0 if streaming else self.offline_right) states, out_mask = self.encode_tower(feats, feat_mask) w = torch.softmax(fork.layer_logits, dim=0) mixed = sum(w[i] * s for i, s in enumerate(states)) x = fork.in_proj(fork.in_norm(mixed)) return fork.position(x), out_mask.sum(-1) def read_cm(self, x: torch.Tensor, lengths: torch.Tensor, streaming: bool = False): """read() with the fork: the fork's transformer reader over mix_cm()'s output and the fork's out_norm, with the same chunk_mask (block-causal at stream_chunk_frames when streaming, else full attention over the valid frames).""" fork = self.fork_cm if fork is None: raise ValueError("read_cm needs the code-mix fork: add_codemix_fork()") if len(fork.reader.layers): allowed = chunk_mask(lengths, x.shape[1], self.stream_chunk_frames if streaming else None) block = (~allowed).repeat_interleave(self.reader_heads, dim=0).squeeze(1) # as encode(): True = masked out x = fork.reader(x, mask=block) return fork.out_norm(x), lengths