#!/usr/bin/env python3 """FA-EG2 Persian reader: transformer body over frozen-encoder soft tokens. Architecture ``a`` (the released Persian reader): a 6-layer pre-LayerNorm transformer encoder built from scratch over the frozen EmbeddingGemma 2 audio soft tokens ``[T, 512]`` (25 tokens/s, 40 ms each). Per token the body emits a 768-d unit vector ``h_t``; a single blank logit is produced by a learned vector. Word logits are logits[w] = scale * cos(h_t, word_table[w]) (scale = exp(log_scale)) logits[blank] = + blank.bias so the reader is trained with ``torch.nn.functional.ctc_loss`` over ``N + 1`` classes (N Persian words + blank). Greedy CTC decoding merges repeated word ids and drops blanks. The module is state-dict compatible with ``checkpoints/persian_reader.pt`` (a plain ``OrderedDict`` of tensors): ``log_scale``, ``body.*`` and ``blank.*``. """ from __future__ import annotations import math import torch import torch.nn as nn import torch.nn.functional as F BLANK = 0 # class index of the CTC blank is not used here; blank is a separate logit # --------------------------------------------------------------------------- # # Rotary positional embeddings # --------------------------------------------------------------------------- # def build_rope_cache(seq_len: int, dim: int, base: float = 10000.0, device=None, dtype=torch.float32): """Return (cos, sin) each [seq_len, dim]. dim must be even.""" inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) t = torch.arange(seq_len, dtype=torch.float32, device=device) freqs = torch.outer(t, inv_freq) # [T, dim/2] emb = torch.cat([freqs, freqs], dim=-1) # [T, dim] return emb.cos().to(dtype), emb.sin().to(dtype) def apply_rope(x, cos, sin): """x: [B, H, T, Dh]; cos/sin: [T, Dh].""" d = x.shape[-1] x1, x2 = x[..., : d // 2], x[..., d // 2:] rotated = torch.cat([-x2, x1], dim=-1) cos = cos[None, None, :, :] sin = sin[None, None, :, :] return x * cos + rotated * sin class Block(nn.Module): """Pre-LayerNorm transformer block with MHSA (RoPE) + GELU MLP.""" def __init__(self, d_model=512, heads=8, ffn=2048, rope_base=10000.0): super().__init__() assert d_model % heads == 0 self.d_model = d_model self.heads = heads self.head_dim = d_model // heads self.rope_base = rope_base self.ln1 = nn.LayerNorm(d_model) self.qkv = nn.Linear(d_model, 3 * d_model) self.proj = nn.Linear(d_model, d_model) self.ln2 = nn.LayerNorm(d_model) self.ff1 = nn.Linear(d_model, ffn) self.ff2 = nn.Linear(ffn, d_model) def forward(self, x, mask=None, rope=None): B, T, D = x.shape h = self.ln1(x) qkv = self.qkv(h).view(B, T, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] # [B, H, T, Dh] if rope is not None: cos, sin = rope q = apply_rope(q, cos, sin) k = apply_rope(k, cos, sin) attn = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim) if mask is not None: # mask: [B, T] True = valid neg = torch.finfo(attn.dtype).min attn = attn.masked_fill(~mask[:, None, None, :], neg) attn = attn.softmax(dim=-1) out = (attn @ v).transpose(1, 2).reshape(B, T, D) x = x + self.proj(out) h = self.ln2(x) x = x + self.ff2(F.gelu(self.ff1(h))) return x class ReaderBody(nn.Module): """LayerNorm -> Linear 512->512 -> N transformer blocks -> LayerNorm -> Linear 512->768.""" def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0): super().__init__() self.d_model = d_model self.layers = layers self.heads = heads self.ffn = ffn self.out_dim = out_dim self.rope_base = rope_base self.ln_in = nn.LayerNorm(d_model) self.inp = nn.Linear(d_model, d_model) self.blocks = nn.ModuleList(Block(d_model, heads, ffn, rope_base) for _ in range(layers)) self.ln_out = nn.LayerNorm(d_model) self.out = nn.Linear(d_model, out_dim) def forward(self, x, mask=None): # x: [B, T, d_model] soft tokens x = self.inp(self.ln_in(x)) T = x.shape[1] cos, sin = build_rope_cache(T, x.shape[-1] // self.heads, self.rope_base, x.device, x.dtype) rope = (cos, sin) for blk in self.blocks: x = blk(x, mask=mask, rope=rope) return self.out(self.ln_out(x)) class PersianReader(nn.Module): """Body + blank logit; logits over ``N + 1`` classes (N = word-table size).""" def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0): super().__init__() self.body = ReaderBody(d_model, layers, heads, ffn, out_dim, rope_base) self.blank = nn.Linear(out_dim, 1) self.log_scale = nn.Parameter(torch.tensor(math.log(64.0))) def forward(self, soft_tokens, mask, table): """soft_tokens: [B,T,512]; mask: [B,T] bool; table: [N,768] float32 (L2-normalized). Returns logits [B, T, N+1] where index 0..N-1 are words and index N is blank. Differentiable (no internal ``torch.no_grad``); inference callers wrap it in ``torch.no_grad()``. """ h = self.body(soft_tokens, mask) # [B,T,768] raw hn = F.normalize(h, dim=-1) scale = self.log_scale.exp() word_logits = scale * torch.einsum("btd,nd->btn", hn, table) # [B,T,N] blank_logit = self.blank(h) # [B,T,1] from raw body output return torch.cat([word_logits, blank_logit], dim=-1) def build_reader_from_config(cfg: dict) -> PersianReader: arch = cfg.get("architecture", {}) return PersianReader( d_model=arch.get("d_model", 512), layers=arch.get("layers", 6), heads=arch.get("heads", 8), ffn=arch.get("ffn", 2048), out_dim=arch.get("out_dim", 768), ) def load_reader(checkpoint_path, table, device="cpu") -> PersianReader: """Load ``checkpoints/persian_reader.pt`` (a plain state dict) into a PersianReader. The architecture (layers, d_model, ffn, out_dim) is inferred from the state dict.""" state = torch.load(checkpoint_path, map_location="cpu", weights_only=False) if isinstance(state, dict) and "state_dict" in state and "body.inp.weight" not in state: state = state["state_dict"] layers = 1 + max(int(k.split(".")[2]) for k in state if k.startswith("body.blocks.")) reader = PersianReader( d_model=state["body.inp.weight"].shape[0], layers=layers, ffn=state["body.blocks.0.ff1.weight"].shape[0], out_dim=state["body.out.weight"].shape[0], ) missing, unexpected = reader.load_state_dict(state, strict=False) if missing or unexpected: raise RuntimeError(f"checkpoint mismatch: missing={missing} unexpected={unexpected}") reader.eval() return reader.to(device) @torch.no_grad() def greedy_decode(logits, table_ids_to_words=None): """logits: [T, N+1] -> list of class ids (word ids, blank merged away). Blank is the last class (index N). Repeated ids are merged, then blanks dropped. """ ids = logits.argmax(dim=-1).tolist() n_classes = logits.shape[-1] blank_id = n_classes - 1 out = [] prev = None for i in ids: if i != prev: if i != blank_id: out.append(i) prev = i return out # --------------------------------------------------------------------------- # # Full reader = frozen encoder + reader, for end-to-end transcription # --------------------------------------------------------------------------- # class FAEG2Transcriber(nn.Module): """Frozen EmbeddingGemma 2 audio encoder + trained Persian reader. ``transcribe(wav_16k)`` returns the greedy-decoded Persian string. Pass a ``feature_extractor`` to compute log-mel features exactly as in caching. """ def __init__(self, model_dir, checkpoint_path, word_table, vocab, device="cpu"): super().__init__() import numpy as np from transformers import EmbeddingGemma2Model, Gemma4AudioFeatureExtractor self.device = device self.fe = Gemma4AudioFeatureExtractor.from_pretrained(model_dir) self.encoder = EmbeddingGemma2Model.from_pretrained( model_dir, dtype=torch.float32, vision_config=None ).to(device).eval() for p in self.encoder.parameters(): p.requires_grad_(False) table = np.load(word_table) if isinstance(word_table, str) else word_table self.table = torch.from_numpy(table.astype("float32")).to(device) self.vocab = ( [w for w in open(vocab, encoding="utf-8").read().splitlines() if w] if isinstance(vocab, str) else vocab ) self.reader = load_reader(checkpoint_path, table, device) @torch.no_grad() def soft_tokens(self, wav): import numpy as np out = self.fe([wav], return_tensors="pt", sampling_rate=16000) x = out["input_features"].to(self.device) m = out["input_features_mask"].to(self.device).bool() res = self.encoder.get_audio_features(x, m, return_dict=True) pooled, am = res.pooler_output, res.attention_mask return pooled[0][am[0]].unsqueeze(0).float() # [1, T, 512] @torch.no_grad() def transcribe(self, wav): import numpy as np if not isinstance(wav, np.ndarray): wav = np.asarray(wav, dtype="float32") wav = np.ascontiguousarray(wav, dtype="float32") soft = self.soft_tokens(wav) mask = torch.ones(soft.shape[:2], dtype=torch.bool, device=self.device) logits = self.reader(soft, mask, self.table)[0] ids = greedy_decode(logits) return " ".join(self.vocab[i] for i in ids)