Automatic Speech Recognition
PyTorch
Safetensors
GGUF
MLX
Tamil
English
speech
asr
streaming
rnnt
ctc
tamil
tanglish
code-switching
indian-english
verisafe
Instructions to use junafinity/VeriSafe-Tanglish-1-STT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use junafinity/VeriSafe-Tanglish-1-STT with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download junafinity/VeriSafe-Tanglish-1-STT --local-dir VeriSafe-Tanglish-1-STT
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download code/indic_stt/model.py from junafinity/VeriSafe-Tanglish-1-STT: direct link, hf CLI and curl.
- Browser
- Download file 33.3 kB
-
https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/indic_stt/model.py
- Command line
-
hf download hf://junafinity/VeriSafe-Tanglish-1-STT/code/indic_stt/model.py
-
curl -L -o model.py https://huggingface.co/junafinity/VeriSafe-Tanglish-1-STT/resolve/main/code/indic_stt/model.py
33.3 kB
| """EmbeddingGemma 2 audio tower + learned layer mix + transformer reader + CTC head (+ optional RNN-T head). | |
| Recipe follows QUAN-Transcribe (Vietnamese, EG2): fine-tune the top audio layers, mix all layer | |
| outputs, read them with a small transformer, train with CTC over a reduced Gemma vocabulary. | |
| The reader is trained with dynamic chunk masks so the same weights run offline and streaming. | |
| Stage 5 adds a transducer head on the same reader output; both heads share the vocabulary. | |
| Push 8 adds an optional code-mix fork: mode=codemix's own post-tower stack on the shared tower (add_codemix_fork, encode_cm). | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import random | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from torch.utils.checkpoint import checkpoint | |
| from transformers import AutoConfig, AutoModel | |
| from transformers.masking_utils import create_bidirectional_mask | |
| from transformers.models.gemma4.modeling_gemma4 import sliding_window_mask_function | |
| from .tower_ctx import set_tower_context | |
| # PACKAGE CHANGE (stage7-final): the tower is built from EmbeddingGemma 2's config.json in a local folder, with no Hugging Face | |
| # download and no base weights: the release weights file holds every tensor, and transcribe.py loads it strictly. Default: | |
| # safetensors/eg2 of this release; transcribe.py points it at the folder it uses (for a GGUF, one written from the file's metadata). | |
| BASE_MODEL = str(Path(__file__).resolve().parents[2] / "safetensors" / "eg2") | |
| CODEMIX_HEADS = {"transducer_cm": "transducer", "ctc_head_cm": "ctc_head"} # code-mix head -> the Tamil head it starts as | |
| FORK = "fork_cm" # state-dict prefix of the code-mix fork (add_codemix_fork) | |
| FORK_MODULES = ("layer_logits", "in_norm", "in_proj", "position", "reader", "out_norm") # the post-tower stack, encode()'s order | |
| def codemix_heads_in(state: dict) -> dict[str, bool]: | |
| """Which code-mix heads a checkpoint's state dict holds, by key prefix: {"transducer_cm": bool, "ctc_head_cm": bool}.""" | |
| return {h: any(k.startswith(h + ".") for k in state) for h in CODEMIX_HEADS} | |
| def codemix_fork_in(state) -> bool: | |
| """Whether a checkpoint's state dict (or any collection of its keys) holds the code-mix fork: a key fork_cm.*.""" | |
| return any(k.startswith(FORK + ".") for k in state) | |
| def chunk_mask(lengths: torch.Tensor, max_len: int, chunk: int | None) -> torch.Tensor: | |
| """Boolean attention mask (B, 1, T, T); True = may attend. | |
| chunk=None gives full attention over valid frames. Otherwise a frame sees every frame in its own | |
| chunk and all earlier chunks (block-causal), which is what streaming inference can compute. | |
| """ | |
| idx = torch.arange(max_len, device=lengths.device) | |
| valid = idx[None, :] < lengths[:, None] | |
| mask = valid[:, None, None, :].expand(-1, 1, max_len, -1) | |
| if chunk is not None: | |
| q_chunk = idx[:, None] // chunk | |
| k_chunk = idx[None, :] // chunk | |
| mask = mask & (k_chunk <= q_chunk)[None, None] | |
| return mask | |
| class CausalConvPosition(nn.Module): | |
| """Depthwise causal conv that adds relative position information (streaming-safe).""" | |
| def __init__(self, dim: int, kernel: int = 15): | |
| super().__init__() | |
| self.pad = kernel - 1 | |
| self.conv = nn.Conv1d(dim, dim, kernel, groups=dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| y = F.pad(x.transpose(1, 2), (self.pad, 0)) | |
| return x + F.gelu(self.conv(y)).transpose(1, 2) | |
| class Transducer(nn.Module): | |
| """RNN-T head: LSTM prediction network + additive joint over the reader output. | |
| Index 0 is the blank and also the start symbol, as in the CTC vocabulary, so both heads share | |
| the tokenizer and either one decodes on its own. The prediction network only sees past tokens | |
| and the joint is per frame, so the head streams whenever the reader does. | |
| styles > 0 adds a register embedding: a per-clip id summed into the prediction network's output. Id 0 is neutral and its | |
| row stays zero; the other rows start at zero, so a checkpoint without them decodes as before until they train. With the | |
| default styles = 0 there is no parameter and no extra step. | |
| delay_penalty and first_only are loss settings from the same config block (B5, see loss_and_grads). They hold no weights | |
| and decoding never reads them: train.py passes them to loss_and_grads. They are taken here only so that every script | |
| that builds the head from a checkpoint's cfg (evaluate, eval_dump, pick_ckpt, exports) still builds it. | |
| """ | |
| def __init__(self, vocab_size: int, enc_dim: int, pred_dim: int = 640, joint_dim: int = 640, | |
| dropout: float = 0.1, styles: int = 0, delay_penalty: float = 0.0, first_only: bool = False): | |
| super().__init__() | |
| del delay_penalty, first_only # loss settings only, read by train.py (see the docstring) | |
| self.embed = nn.Embedding(vocab_size, pred_dim) | |
| self.lstm = nn.LSTM(pred_dim, pred_dim, batch_first=True) | |
| self.drop = nn.Dropout(dropout) | |
| self.enc_proj = nn.Linear(enc_dim, joint_dim) | |
| self.pred_proj = nn.Linear(pred_dim, joint_dim) | |
| self.out = nn.Linear(joint_dim, vocab_size) | |
| # from_pretrained draws no random numbers, so every other weight starts the same with or without styles. | |
| self.style = nn.Embedding.from_pretrained(torch.zeros(styles, pred_dim), freeze=False, padding_idx=0) if styles else None | |
| def predict(self, tokens: torch.Tensor, state=None, style: torch.Tensor | None = None): | |
| """Previous tokens (B, U) -> prediction features in joint space (B, U, J) and the LSTM state. style (B,): register ids.""" | |
| y, state = self.lstm(self.drop(self.embed(tokens)), state) | |
| if style is not None: | |
| if self.style is None: | |
| raise ValueError("a style id was given, but this head has no style embedding (transducer: styles in the config)") | |
| y = y + self.style(style)[:, None] | |
| return self.pred_proj(self.drop(y)), state | |
| def joint(self, enc: torch.Tensor, pred: torch.Tensor) -> torch.Tensor: | |
| return self.out(torch.tanh(enc + pred)) | |
| def loss_and_grads(self, x, lengths, targets, target_lengths, scale: float = 1.0, max_cells: int = 100_000, | |
| style: torch.Tensor | None = None, delay_penalty: float = 0.0, first_only: bool = False, | |
| fused: bool | None = None): | |
| """RNN-T loss per utterance / its target length, averaged over the batch (like F.ctc_loss "mean"). | |
| The (B, T, U+1, V) joint tensor is built a few utterances at a time (≤ max_cells of T × (U+1)) | |
| from detached copies of the encoder and prediction features, and each slice is backpropagated | |
| at once (`scale` × loss), which bounds memory. The joint's own weights get their gradients | |
| here; the caller passes the returned (features, gradients) to torch.autograd.backward | |
| together with the CTC loss. Call outside autocast. style (B,): register ids, as in predict. | |
| delay_penalty λ > 0, the delay-penalized transducer (Kang et al. 2023; B5, methods scout 2026-11-09 §2.1): the | |
| log-softmax runs here instead of inside the loss, λ((T−1)/2 − t) is added to every non-blank log-prob of frame t | |
| (T: that utterance's frame count), and the loss reads those log-probs as they are (fused_log_softmax=False). A token | |
| emitted before mid-clip gains, a later one pays. first_only: the offset goes on the first token's row (u = 0) only, | |
| so only the start of emission is pushed. The returned loss includes the penalty: it does not compare with a run | |
| without one. The unfused path holds one more fp32 copy of each slice's joint tensor (cells × V × 4 bytes). | |
| 0 (default) keeps torchaudio's fused path, unchanged. fused=False takes the unfused path at λ = 0 too, for | |
| scripts/test_delay_penalty.py; None (default) is fused unless λ is set. | |
| """ | |
| from torchaudio.functional import rnnt_loss | |
| if fused and delay_penalty: | |
| raise ValueError("a delay penalty needs the unfused path: it goes onto normalised log-probs") | |
| if first_only and not delay_penalty: | |
| raise ValueError("first_only without delay_penalty changes nothing") | |
| unfused = bool(delay_penalty) or fused is False | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| enc = self.enc_proj(x) | |
| pred, _ = self.predict(F.pad(targets, (1, 0)), style=style) # start symbol = blank | |
| e, p = enc.detach().requires_grad_(), pred.detach().requires_grad_() | |
| total, n = 0.0, x.shape[0] | |
| for idx in _slices(lengths, target_lengths, max_cells): | |
| T, U = int(lengths[idx].max()), int(target_lengths[idx].max()) | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| logits = self.joint(e[idx, :T, None], p[idx, None, : U + 1]).float() # the bf16 copy is freed here | |
| if unfused: | |
| logits = logits.log_softmax(-1) # rebinding frees the fp32 logits: log_softmax keeps only its output | |
| if delay_penalty: | |
| logits = _delay_offset(logits, lengths[idx], delay_penalty, first_only) | |
| loss = rnnt_loss(logits, targets[idx, :U].int(), lengths[idx].int(), | |
| target_lengths[idx].int(), blank=0, reduction="none", fused_log_softmax=not unfused) | |
| del logits # RnntLoss keeps only its gradients; drop the fp32 copy before backward (K3 OOM fix, 2026-11-08) | |
| loss = (loss / target_lengths[idx]).sum() / n | |
| (scale * loss).backward() | |
| total += loss.item() | |
| return total, (enc, pred), (e.grad, p.grad) | |
| def greedy(self, x: torch.Tensor, lengths: torch.Tensor, max_symbols: int = 5, | |
| allowed: torch.Tensor | None = None, swap: bool = False, start_penalty: float = 0.0, | |
| cue: torch.Tensor | None = None, style: int | None = None, | |
| all_frame_penalty: float = 0.0, frames_out: list | None = None) -> list[list[int]]: | |
| """Batched greedy decoding: at each frame emit tokens until blank (at most max_symbols). | |
| allowed (V) or (B, V) bool: the caller's language hint, the pieces that may be emitted (blank must be among them; | |
| SPMTokenizer.script_ids() gives the script classes). The others get log-prob -inf, so where the model wants one, | |
| blank usually wins and the word is lost. | |
| swap: where the unmasked argmax is a forbidden piece, emit the best allowed piece instead of letting blank win; at most | |
| one swap per frame, at the frame's first symbol step (later on, a forbidden wish ends the frame). Nothing without allowed. | |
| start_penalty: taken off blank's logit while a clip has emitted nothing, for clips that fail to start. cue (B, T) bool, from | |
| the caller: the penalty applies only on frames where it is True, e.g. once the CTC head has left blank on that clip. It | |
| comes before allowed, so the mask and swap see the penalised blank. 0 (default) is plain greedy, whatever cue says. | |
| style: one register id for the whole batch; None (default) is neutral. | |
| all_frame_penalty: taken off blank's logit at every symbol step of every clip, started or not; cue does not gate it. It adds | |
| to start_penalty and also comes before allowed. 0 (default) changes nothing. | |
| frames_out: a list the caller owns; it is extended with one list per clip, the frame of each emitted token (for | |
| scripts/decode_ctc_assist.py). None (default) records nothing; the hypotheses are the same either way. | |
| """ | |
| B, T = x.shape[:2] | |
| enc = self.enc_proj(x) | |
| style = None if style is None else torch.full((B,), style, device=x.device) | |
| pred, state = self.predict(torch.zeros(B, 1, dtype=torch.long, device=x.device), style=style) | |
| pred = pred[:, 0] | |
| started = torch.zeros(B, dtype=torch.bool, device=x.device) | |
| hyps = [[] for _ in range(B)] | |
| frames = [[] for _ in range(B)] | |
| ban = None | |
| if allowed is not None: | |
| if not allowed[..., 0].all(): | |
| raise ValueError("allowed must keep blank (index 0): without it a clip can never end a frame") | |
| ban = ~allowed.to(x.device).expand(B, -1) | |
| idx = torch.arange(B, device=x.device) | |
| for t in range(T): | |
| live = lengths > t | |
| for s in range(max_symbols): | |
| logits = self.joint(enc[:, t], pred) | |
| if start_penalty or all_frame_penalty: | |
| waiting = ~started if cue is None else ~started & cue[:, t] | |
| logits[:, 0] = logits[:, 0] - (all_frame_penalty + start_penalty * waiting.to(logits.dtype)) | |
| if ban is not None: | |
| wished_banned = ban[idx, logits.argmax(-1)] | |
| logits = logits.masked_fill(ban, float("-inf")) | |
| if swap: | |
| forced = float("-inf") if s == 0 else float("inf") | |
| logits[:, 0] = torch.where(wished_banned, torch.full_like(logits[:, 0], forced), logits[:, 0]) | |
| k = logits.argmax(-1) | |
| emit = live & (k != 0) | |
| if not emit.any(): | |
| break | |
| started = started | emit | |
| new_pred, new_state = self.predict(k[:, None], state, style) | |
| pred = torch.where(emit[:, None], new_pred[:, 0], pred) | |
| state = tuple(torch.where(emit[None, :, None], n, s) for n, s in zip(new_state, state)) | |
| for b, tok in zip(emit.nonzero()[:, 0].tolist(), k[emit].tolist()): | |
| hyps[b].append(tok) | |
| frames[b].append(t) | |
| live = emit | |
| if frames_out is not None: | |
| frames_out.extend(frames) | |
| return hyps | |
| def _delay_offset(logp: torch.Tensor, lengths: torch.Tensor, delay_penalty: float, first_only: bool) -> torch.Tensor: | |
| """Log-probs (b, T, U+1, V) plus λ((T_i − 1)/2 − t) on each non-blank entry of frame t (blank = 0): on every row u, or | |
| with first_only on u = 0 only. T_i is utterance i's frame count; frames past it are never read by the loss. | |
| Out of place, since log_softmax keeps logp for its backward. Both forms pass the gradient through unchanged, with no | |
| full-size temporary: the offset itself is (b, T, 1, V).""" | |
| t = torch.arange(logp.shape[1], device=logp.device, dtype=logp.dtype) | |
| off = delay_penalty * ((lengths[:, None].to(logp.dtype) - 1) / 2 - t) # (b, T) | |
| off = F.pad(off[:, :, None, None].expand(-1, -1, 1, logp.shape[-1] - 1), (1, 0)) # (b, T, 1, V), 0 on blank | |
| if first_only: | |
| return logp.index_add(2, torch.zeros(1, dtype=torch.long, device=logp.device), off) | |
| return logp + off | |
| def _slices(lengths: torch.Tensor, target_lengths: torch.Tensor, max_cells: int) -> list[torch.Tensor]: | |
| """Consecutive index groups whose padded joint size B × max T × max (U+1) stays within max_cells.""" | |
| out, cur, t_max, u_max = [], [], 0, 0 | |
| for i, (t, u) in enumerate(zip(lengths.tolist(), (target_lengths + 1).tolist())): | |
| if cur and (len(cur) + 1) * max(t_max, t) * max(u_max, u) > max_cells: | |
| out.append(cur) | |
| cur, t_max, u_max = [], 0, 0 | |
| cur.append(i) | |
| t_max, u_max = max(t_max, t), max(u_max, u) | |
| out.append(cur) | |
| return [torch.tensor(c, device=lengths.device) for c in out] | |
| class EG2SpeechModel(nn.Module): | |
| def __init__( | |
| self, | |
| vocab_size: int, | |
| train_tower_layers: int = 2, | |
| reader_layers: int = 6, | |
| reader_dim: int = 768, | |
| reader_heads: int = 12, | |
| dropout: float = 0.1, | |
| stream_chunk_frames: int = 12, | |
| stream_prob: float = 0.5, | |
| grad_checkpointing: bool = True, | |
| zero_init_residual: bool = True, | |
| transducer: dict | None = None, | |
| offline_right: int = 0, | |
| ): | |
| super().__init__() | |
| self.tower = AutoModel.from_config(AutoConfig.from_pretrained(BASE_MODEL).audio_config, dtype=torch.float32) # PACKAGE CHANGE | |
| cfg = self.tower.config | |
| n_states = cfg.num_hidden_layers + 1 | |
| self.layer_logits = nn.Parameter(torch.zeros(n_states)) | |
| self.in_norm = nn.LayerNorm(cfg.hidden_size) | |
| self.in_proj = nn.Linear(cfg.hidden_size, reader_dim) | |
| self.position = CausalConvPosition(reader_dim) | |
| layer = nn.TransformerEncoderLayer( | |
| reader_dim, reader_heads, 4 * reader_dim, dropout=dropout, | |
| activation="gelu", batch_first=True, norm_first=True, | |
| ) | |
| self.reader = nn.TransformerEncoder(layer, reader_layers, enable_nested_tensor=False) | |
| self.reader_heads = reader_heads | |
| if zero_init_residual: | |
| # Each reader block starts as identity, so CTC sees the tower features from step 0 | |
| # and the reader only learns a correction (avoids a long all-blank plateau). | |
| for blk in self.reader.layers: | |
| for lin in (blk.self_attn.out_proj, blk.linear2): | |
| nn.init.zeros_(lin.weight) | |
| nn.init.zeros_(lin.bias) | |
| self.out_norm = nn.LayerNorm(reader_dim) | |
| self.ctc_head = nn.Linear(reader_dim, vocab_size) # index 0 = CTC blank | |
| self.transducer = Transducer(vocab_size, reader_dim, **transducer) if transducer else None | |
| self.transducer_cm = None # add_codemix_head() | |
| self.ctc_head_cm = None # add_codemix_head(ctc=True) | |
| self.fork_cm = None # add_codemix_fork() | |
| self.stream_chunk_frames = stream_chunk_frames | |
| self.stream_prob = stream_prob | |
| self.offline_right = offline_right # dual-mode: tower right context on offline passes; streaming passes always run at right 0 | |
| self.grad_checkpointing = grad_checkpointing | |
| self.set_trainable_tower_layers(train_tower_layers) | |
| # ---- freezing ------------------------------------------------------------------------- | |
| def set_trainable_tower_layers(self, n: int | str) -> None: | |
| """Train only the top-n conformer layers ("all" = whole tower incl. subsampling).""" | |
| total = self.tower.config.num_hidden_layers | |
| n = total if n == "all" else int(n) | |
| self.first_trainable = total - n | |
| for p in self.tower.parameters(): | |
| p.requires_grad = False | |
| for layer in self.tower.layers[self.first_trainable:]: | |
| for p in layer.parameters(): | |
| p.requires_grad = True | |
| if n == total: | |
| for p in self.tower.subsample_conv_projection.parameters(): | |
| p.requires_grad = True | |
| self.tower.output_proj.requires_grad_(False) # unused by the STT path | |
| def add_codemix_head(self, ctc: bool = False) -> None: | |
| """Add the mode=codemix RNN-T head, transducer_cm, and with ctc=True the code-mix CTC head, ctc_head_cm: their own | |
| weights on the same reader output (encode_cm's on a model with add_codemix_fork), so the encoder and the mode=tamil | |
| heads stay as they are. They start as copies of | |
| transducer and ctc_head at this moment, so an untrained code-mix head decodes exactly as its Tamil twin. | |
| State-dict keys: transducer_cm.* (the transducer's names) and ctc_head_cm.weight, ctc_head_cm.bias. param_groups puts | |
| transducer_cm in the transducer group and ctc_head_cm in the head group. | |
| Order: call this before load_state_dict, then copy_missing_codemix(missing): a checkpoint with the heads fills them, one | |
| without them leaves them missing and copy_missing_codemix copies the just-loaded Tamil heads into them (never the | |
| random init of a model built before the load). A model that never calls this ignores transducer_cm.* and | |
| ctc_head_cm.* in a checkpoint (unexpected keys under strict=False), so its mode=tamil output cannot change.""" | |
| if self.transducer is None: | |
| raise ValueError("the code-mix head starts as a copy of the transducer head, and this model has none") | |
| if self.transducer_cm is not None or self.ctc_head_cm is not None: | |
| raise ValueError("the code-mix heads already exist: a second copy would overwrite their weights") | |
| self.transducer_cm = copy.deepcopy(self.transducer) | |
| if ctc: | |
| self.ctc_head_cm = copy.deepcopy(self.ctc_head) | |
| def copy_missing_codemix(self, missing: list[str], log: bool = True) -> list[str]: | |
| """After load_state_dict(state, strict=False), with its `missing` list: each code-mix head this model has whose keys | |
| are all missing is set to a copy of its Tamil twin, as loaded. Returns the heads copied (and prints one line each), so | |
| an old checkpoint gives code-mix heads equal to its Tamil heads. A head with only some keys missing is an error: | |
| the checkpoint holds a different head.""" | |
| missing, copied = set(missing), [] | |
| for name, twin in CODEMIX_HEADS.items(): | |
| head = getattr(self, name) | |
| if head is None: | |
| continue | |
| keys = {f"{name}.{k}" for k in head.state_dict()} | |
| gone = keys & missing | |
| if not gone: | |
| continue | |
| if gone != keys: | |
| raise ValueError(f"the checkpoint holds only part of {name}: missing {sorted(gone)[:5]}") | |
| head.load_state_dict(getattr(self, twin).state_dict()) | |
| copied.append(name) | |
| if log: | |
| print(f"{name} not in the checkpoint: starts as a copy of the loaded {twin}", flush=True) | |
| return copied | |
| def add_codemix_fork(self) -> None: | |
| """Add the code-mix fork, fork_cm (push 8 plan section 1): mode=codemix's own copy of the post-tower stack, every module | |
| between the tower and the heads (FORK_MODULES: layer_logits, in_norm, in_proj, position, reader with all its layers, | |
| out_norm). Deep copies of those modules at this moment, so a fork added after a load is the loaded stack and encode_cm | |
| then gives encode()'s output bit for bit. The tower is shared, not copied. State-dict keys: fork_cm. + the stack's own | |
| names (81 tensors, 43,330,317 values with the 6-layer 768-d reader on the 1024-d tower). encode() and the Tamil heads | |
| never read the fork, so mode=tamil output is the same with or without it; encode_cm reads it, for the code-mix heads. | |
| Order: as add_codemix_head, call this before load_state_dict when a checkpoint holds fork_cm.* (codemix_fork_in), so its | |
| weights replace the copies. Nothing fills a fork the checkpoint lacks: those keys stay missing, and fork_cm.* keys in a | |
| model without the fork stay unexpected; every loader (train.load_weights, scripts/eval_dump.py) refuses both.""" | |
| if self.fork_cm is not None: | |
| raise ValueError("the code-mix fork already exists: a second copy would overwrite its weights") | |
| fork = nn.Module() | |
| for name in FORK_MODULES: | |
| setattr(fork, name, copy.deepcopy(getattr(self, name))) # a Parameter stays a Parameter, requires_grad as it was | |
| self.fork_cm = fork | |
| def encode_for(self, head: str): | |
| """The encoder a head set reads: encode_cm for head="cm" on a model with the fork, else encode (both heads of a model | |
| without the fork read encode(), as before push 8).""" | |
| if head not in ("ta", "cm"): | |
| raise ValueError(f"head must be 'ta' or 'cm', not {head!r}") | |
| return self.encode_cm if head == "cm" and self.fork_cm is not None else self.encode | |
| def param_groups(self, lr_tower: float, lr_head: float, lr_reader: float, weight_decay: float, | |
| lr_transducer: float | None = None) -> list[dict]: | |
| """Three LRs. The from-scratch transformer reader needs a low LR (overfit test: 1e-3 stalls | |
| in an all-blank plateau, 5e-5 memorises 32 clips in 200 steps); the linear layers take more. | |
| A transducer head, and a code-mix head, get a fourth group (lr_transducer, default lr_head), appended last; the code-mix | |
| CTC head (ctc_head_cm) is in the head group with ctc_head. The fork's modules (add_codemix_fork) go where their Tamil | |
| twins go: fork_cm.reader.* with reader, the rest of the fork with head.""" | |
| groups = {"tower": [], "reader": [], "head": [], "transducer": []} | |
| for n, p in self.named_parameters(): | |
| if p.requires_grad: | |
| groups["tower" if n.startswith("tower.") else "reader" if n.startswith(("reader.", FORK + ".reader.")) | |
| else "transducer" if n.startswith(("transducer.", "transducer_cm.")) else "head"].append(p) | |
| out = [ | |
| {"params": groups["tower"], "lr": lr_tower, "weight_decay": weight_decay}, | |
| {"params": groups["head"], "lr": lr_head, "weight_decay": weight_decay}, | |
| {"params": groups["reader"], "lr": lr_reader, "weight_decay": weight_decay}, | |
| ] | |
| if groups["transducer"]: | |
| out.append({"params": groups["transducer"], "lr": lr_transducer or lr_head, "weight_decay": weight_decay}) | |
| return out | |
| # ---- forward -------------------------------------------------------------------------- | |
| def encode_tower(self, feats: torch.Tensor, feat_mask: torch.Tensor): | |
| """Replicates Gemma4AudioModel.forward but returns every layer and supports checkpointing.""" | |
| tower, cfg = self.tower, self.tower.config | |
| grad_sub = self.first_trainable == 0 and self.training | |
| with torch.set_grad_enabled(grad_sub and torch.is_grad_enabled()): | |
| hs, out_mask = tower.subsample_conv_projection(feats, feat_mask) | |
| pos = tower.rel_pos_enc(hs) | |
| attn = create_bidirectional_mask( | |
| config=cfg, inputs_embeds=hs, attention_mask=out_mask, | |
| and_mask_function=sliding_window_mask_function( | |
| (cfg.attention_context_left - 1, cfg.attention_context_right) | |
| ), | |
| ) | |
| if attn is not None: | |
| attn = tower._convert_4d_mask_to_blocked_5d(attn) | |
| states = [hs] | |
| for i, layer in enumerate(tower.layers): | |
| trainable = i >= self.first_trainable and self.training and torch.is_grad_enabled() | |
| if not trainable: | |
| with torch.no_grad(): | |
| hs = layer(hs, attention_mask=attn, position_embeddings=pos) | |
| elif self.grad_checkpointing: | |
| hs = checkpoint(layer, hs, attn, pos, use_reentrant=False) | |
| else: | |
| hs = layer(hs, attention_mask=attn, position_embeddings=pos) | |
| states.append(hs) | |
| return states, out_mask | |
| def forward(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): | |
| """Returns CTC log-probs (B, T, V) float32 and frame lengths (B,).""" | |
| x, lengths = self.encode(feats, feat_mask, streaming) | |
| return self.ctc_log_probs(x), lengths | |
| def ctc_log_probs(self, x: torch.Tensor, head: str = "ta") -> torch.Tensor: | |
| """CTC log-probs (B, T, V) float32 of the Tamil head (head="ta", default) or the code-mix head (head="cm").""" | |
| if head == "ta": | |
| lin = self.ctc_head | |
| elif head == "cm": | |
| lin = self.ctc_head_cm | |
| if lin is None: | |
| raise ValueError("head='cm' needs the code-mix CTC head: add_codemix_head(ctc=True)") | |
| else: | |
| raise ValueError(f"head must be 'ta' or 'cm', not {head!r}") | |
| return lin(x).float().log_softmax(-1) | |
| # PACKAGE CHANGE (STT-1.2): encode() is split into mix() + read(), the same operations in the same order, so its output is | |
| # bit-identical. A streaming caller keeps mix()'s output and runs read() twice: block-causal for the partials, then once with | |
| # full attention at the end of the utterance for the two-pass final (transcribe.py --finals two-pass). | |
| def encode(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): | |
| """Normalized reader output (B, T, D), which both heads read, and frame lengths (B,): read(mix(...)).""" | |
| if streaming is None: | |
| streaming = self.training and random.random() < self.stream_prob | |
| x, lengths = self.mix(feats, feat_mask, streaming) | |
| return self.read(x, lengths, streaming) | |
| def mix(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool = False): | |
| """The reader's input (B, T, D) and frame lengths (B,): the audio tower, the learned layer mix, in_proj and the causal | |
| position conv. With offline_right = 0 (every model but B4's, and every release) nothing here depends on `streaming`, so | |
| one mix() output serves both read() passes; with offline_right > 0 the tower differs between the modes, and a two-pass | |
| caller must not share it.""" | |
| if self.offline_right: # 0, every run but B4: no call, the tower stays as loaded | |
| set_tower_context(self.tower, right=0 if streaming else self.offline_right) | |
| states, out_mask = self.encode_tower(feats, feat_mask) | |
| w = torch.softmax(self.layer_logits, dim=0) | |
| mixed = sum(w[i] * s for i, s in enumerate(states)) | |
| x = self.in_proj(self.in_norm(mixed)) | |
| return self.position(x), out_mask.sum(-1) | |
| def read(self, x: torch.Tensor, lengths: torch.Tensor, streaming: bool = False): | |
| """The transformer reader over mix()'s output and out_norm: normalized reader output (B, T, D) and the same lengths. | |
| streaming: the block-causal chunk_mask (stream_chunk_frames per block), else full attention over the valid frames.""" | |
| if len(self.reader.layers): | |
| allowed = chunk_mask(lengths, x.shape[1], self.stream_chunk_frames if streaming else None) | |
| # nn.TransformerEncoder expects True = masked out, shape (B*heads, T, T) | |
| block = (~allowed).repeat_interleave(self.reader_heads, dim=0).squeeze(1) | |
| x = self.reader(x, mask=block) | |
| return self.out_norm(x), lengths | |
| # PACKAGE CHANGE (STT-2): the project's encode_cm() (push 8) is split into mix_cm() + read_cm(), as encode() is split into | |
| # mix() + read() above: the same operations in the same order, so its output is bit-identical. The code-mix partials | |
| # (read_cm, block-causal) and the two-pass final (read_cm, full attention) share one mix_cm() output. mix(), read() and | |
| # encode(), the mode=tamil path, are not changed. | |
| def encode_cm(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool | None = None): | |
| """mode=codemix's reader output (B, T, D), which the code-mix heads read, and frame lengths (B,): encode()'s ops in | |
| encode()'s order on the same encode_tower() and the same chunk_mask (block-causal at stream_chunk_frames when | |
| streaming), with the fork's modules (add_codemix_fork) in place of the Tamil stack. encode() itself is not changed. | |
| A caller that runs both on one batch passes streaming: None draws its own random number in training, as encode(). | |
| read_cm(mix_cm(...)).""" | |
| if self.fork_cm is None: | |
| raise ValueError("encode_cm needs the code-mix fork: add_codemix_fork()") | |
| if streaming is None: | |
| streaming = self.training and random.random() < self.stream_prob | |
| x, lengths = self.mix_cm(feats, feat_mask, streaming) | |
| return self.read_cm(x, lengths, streaming) | |
| def mix_cm(self, feats: torch.Tensor, feat_mask: torch.Tensor, streaming: bool = False): | |
| """mix() with the fork: the shared audio tower (encode_tower), then the fork's layer mix, in_norm, in_proj and causal | |
| position conv. The reader's input (B, T, D) for read_cm() and frame lengths (B,). As in mix(), nothing here depends on | |
| `streaming` when offline_right = 0, so one mix_cm() output serves both read_cm() passes.""" | |
| fork = self.fork_cm | |
| if fork is None: | |
| raise ValueError("mix_cm needs the code-mix fork: add_codemix_fork()") | |
| if self.offline_right: # as encode(): 0 in every run but B4 | |
| set_tower_context(self.tower, right=0 if streaming else self.offline_right) | |
| states, out_mask = self.encode_tower(feats, feat_mask) | |
| w = torch.softmax(fork.layer_logits, dim=0) | |
| mixed = sum(w[i] * s for i, s in enumerate(states)) | |
| x = fork.in_proj(fork.in_norm(mixed)) | |
| return fork.position(x), out_mask.sum(-1) | |
| def read_cm(self, x: torch.Tensor, lengths: torch.Tensor, streaming: bool = False): | |
| """read() with the fork: the fork's transformer reader over mix_cm()'s output and the fork's out_norm, with the same | |
| chunk_mask (block-causal at stream_chunk_frames when streaming, else full attention over the valid frames).""" | |
| fork = self.fork_cm | |
| if fork is None: | |
| raise ValueError("read_cm needs the code-mix fork: add_codemix_fork()") | |
| if len(fork.reader.layers): | |
| allowed = chunk_mask(lengths, x.shape[1], self.stream_chunk_frames if streaming else None) | |
| block = (~allowed).repeat_interleave(self.reader_heads, dim=0).squeeze(1) # as encode(): True = masked out | |
| x = fork.reader(x, mask=block) | |
| return fork.out_norm(x), lengths | |