Download fa_reader_model.py from karim23657/fa-eg2-persian: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/fa_reader_model.py
- Command line
-
hf download hf://karim23657/fa-eg2-persian/fa_reader_model.py
-
curl -L -o fa_reader_model.py https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/fa_reader_model.py
10.1 kB
| #!/usr/bin/env python3 | |
| """FA-EG2 Persian reader: transformer body over frozen-encoder soft tokens. | |
| Architecture ``a`` (the released Persian reader): a 6-layer pre-LayerNorm transformer | |
| encoder built from scratch over the frozen EmbeddingGemma 2 audio soft tokens | |
| ``[T, 512]`` (25 tokens/s, 40 ms each). Per token the body emits a 768-d unit vector | |
| ``h_t``; a single blank logit is produced by a learned vector. Word logits are | |
| logits[w] = scale * cos(h_t, word_table[w]) (scale = exp(log_scale)) | |
| logits[blank] = <blank.weight, h_t> + blank.bias | |
| so the reader is trained with ``torch.nn.functional.ctc_loss`` over ``N + 1`` | |
| classes (N Persian words + blank). Greedy CTC decoding merges repeated word ids and | |
| drops blanks. | |
| The module is state-dict compatible with ``checkpoints/persian_reader.pt`` | |
| (a plain ``OrderedDict`` of tensors): ``log_scale``, ``body.*`` and ``blank.*``. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| BLANK = 0 # class index of the CTC blank is not used here; blank is a separate logit | |
| # --------------------------------------------------------------------------- # | |
| # Rotary positional embeddings | |
| # --------------------------------------------------------------------------- # | |
| def build_rope_cache(seq_len: int, dim: int, base: float = 10000.0, device=None, dtype=torch.float32): | |
| """Return (cos, sin) each [seq_len, dim]. dim must be even.""" | |
| inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) | |
| t = torch.arange(seq_len, dtype=torch.float32, device=device) | |
| freqs = torch.outer(t, inv_freq) # [T, dim/2] | |
| emb = torch.cat([freqs, freqs], dim=-1) # [T, dim] | |
| return emb.cos().to(dtype), emb.sin().to(dtype) | |
| def apply_rope(x, cos, sin): | |
| """x: [B, H, T, Dh]; cos/sin: [T, Dh].""" | |
| d = x.shape[-1] | |
| x1, x2 = x[..., : d // 2], x[..., d // 2:] | |
| rotated = torch.cat([-x2, x1], dim=-1) | |
| cos = cos[None, None, :, :] | |
| sin = sin[None, None, :, :] | |
| return x * cos + rotated * sin | |
| class Block(nn.Module): | |
| """Pre-LayerNorm transformer block with MHSA (RoPE) + GELU MLP.""" | |
| def __init__(self, d_model=512, heads=8, ffn=2048, rope_base=10000.0): | |
| super().__init__() | |
| assert d_model % heads == 0 | |
| self.d_model = d_model | |
| self.heads = heads | |
| self.head_dim = d_model // heads | |
| self.rope_base = rope_base | |
| self.ln1 = nn.LayerNorm(d_model) | |
| self.qkv = nn.Linear(d_model, 3 * d_model) | |
| self.proj = nn.Linear(d_model, d_model) | |
| self.ln2 = nn.LayerNorm(d_model) | |
| self.ff1 = nn.Linear(d_model, ffn) | |
| self.ff2 = nn.Linear(ffn, d_model) | |
| def forward(self, x, mask=None, rope=None): | |
| B, T, D = x.shape | |
| h = self.ln1(x) | |
| qkv = self.qkv(h).view(B, T, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4) | |
| q, k, v = qkv[0], qkv[1], qkv[2] # [B, H, T, Dh] | |
| if rope is not None: | |
| cos, sin = rope | |
| q = apply_rope(q, cos, sin) | |
| k = apply_rope(k, cos, sin) | |
| attn = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim) | |
| if mask is not None: | |
| # mask: [B, T] True = valid | |
| neg = torch.finfo(attn.dtype).min | |
| attn = attn.masked_fill(~mask[:, None, None, :], neg) | |
| attn = attn.softmax(dim=-1) | |
| out = (attn @ v).transpose(1, 2).reshape(B, T, D) | |
| x = x + self.proj(out) | |
| h = self.ln2(x) | |
| x = x + self.ff2(F.gelu(self.ff1(h))) | |
| return x | |
| class ReaderBody(nn.Module): | |
| """LayerNorm -> Linear 512->512 -> N transformer blocks -> LayerNorm -> Linear 512->768.""" | |
| def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.layers = layers | |
| self.heads = heads | |
| self.ffn = ffn | |
| self.out_dim = out_dim | |
| self.rope_base = rope_base | |
| self.ln_in = nn.LayerNorm(d_model) | |
| self.inp = nn.Linear(d_model, d_model) | |
| self.blocks = nn.ModuleList(Block(d_model, heads, ffn, rope_base) for _ in range(layers)) | |
| self.ln_out = nn.LayerNorm(d_model) | |
| self.out = nn.Linear(d_model, out_dim) | |
| def forward(self, x, mask=None): | |
| # x: [B, T, d_model] soft tokens | |
| x = self.inp(self.ln_in(x)) | |
| T = x.shape[1] | |
| cos, sin = build_rope_cache(T, x.shape[-1] // self.heads, self.rope_base, x.device, x.dtype) | |
| rope = (cos, sin) | |
| for blk in self.blocks: | |
| x = blk(x, mask=mask, rope=rope) | |
| return self.out(self.ln_out(x)) | |
| class PersianReader(nn.Module): | |
| """Body + blank logit; logits over ``N + 1`` classes (N = word-table size).""" | |
| def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0): | |
| super().__init__() | |
| self.body = ReaderBody(d_model, layers, heads, ffn, out_dim, rope_base) | |
| self.blank = nn.Linear(out_dim, 1) | |
| self.log_scale = nn.Parameter(torch.tensor(math.log(64.0))) | |
| def forward(self, soft_tokens, mask, table): | |
| """soft_tokens: [B,T,512]; mask: [B,T] bool; table: [N,768] float32 (L2-normalized). | |
| Returns logits [B, T, N+1] where index 0..N-1 are words and index N is blank. | |
| Differentiable (no internal ``torch.no_grad``); inference callers wrap it in | |
| ``torch.no_grad()``. | |
| """ | |
| h = self.body(soft_tokens, mask) # [B,T,768] raw | |
| hn = F.normalize(h, dim=-1) | |
| scale = self.log_scale.exp() | |
| word_logits = scale * torch.einsum("btd,nd->btn", hn, table) # [B,T,N] | |
| blank_logit = self.blank(h) # [B,T,1] from raw body output | |
| return torch.cat([word_logits, blank_logit], dim=-1) | |
| def build_reader_from_config(cfg: dict) -> PersianReader: | |
| arch = cfg.get("architecture", {}) | |
| return PersianReader( | |
| d_model=arch.get("d_model", 512), | |
| layers=arch.get("layers", 6), | |
| heads=arch.get("heads", 8), | |
| ffn=arch.get("ffn", 2048), | |
| out_dim=arch.get("out_dim", 768), | |
| ) | |
| def load_reader(checkpoint_path, table, device="cpu") -> PersianReader: | |
| """Load ``checkpoints/persian_reader.pt`` (a plain state dict) into a PersianReader. | |
| The architecture (layers, d_model, ffn, out_dim) is inferred from the state dict.""" | |
| state = torch.load(checkpoint_path, map_location="cpu", weights_only=False) | |
| if isinstance(state, dict) and "state_dict" in state and "body.inp.weight" not in state: | |
| state = state["state_dict"] | |
| layers = 1 + max(int(k.split(".")[2]) for k in state if k.startswith("body.blocks.")) | |
| reader = PersianReader( | |
| d_model=state["body.inp.weight"].shape[0], | |
| layers=layers, | |
| ffn=state["body.blocks.0.ff1.weight"].shape[0], | |
| out_dim=state["body.out.weight"].shape[0], | |
| ) | |
| missing, unexpected = reader.load_state_dict(state, strict=False) | |
| if missing or unexpected: | |
| raise RuntimeError(f"checkpoint mismatch: missing={missing} unexpected={unexpected}") | |
| reader.eval() | |
| return reader.to(device) | |
| def greedy_decode(logits, table_ids_to_words=None): | |
| """logits: [T, N+1] -> list of class ids (word ids, blank merged away). | |
| Blank is the last class (index N). Repeated ids are merged, then blanks dropped. | |
| """ | |
| ids = logits.argmax(dim=-1).tolist() | |
| n_classes = logits.shape[-1] | |
| blank_id = n_classes - 1 | |
| out = [] | |
| prev = None | |
| for i in ids: | |
| if i != prev: | |
| if i != blank_id: | |
| out.append(i) | |
| prev = i | |
| return out | |
| # --------------------------------------------------------------------------- # | |
| # Full reader = frozen encoder + reader, for end-to-end transcription | |
| # --------------------------------------------------------------------------- # | |
| class FAEG2Transcriber(nn.Module): | |
| """Frozen EmbeddingGemma 2 audio encoder + trained Persian reader. | |
| ``transcribe(wav_16k)`` returns the greedy-decoded Persian string. Pass a | |
| ``feature_extractor`` to compute log-mel features exactly as in caching. | |
| """ | |
| def __init__(self, model_dir, checkpoint_path, word_table, vocab, device="cpu"): | |
| super().__init__() | |
| import numpy as np | |
| from transformers import EmbeddingGemma2Model, Gemma4AudioFeatureExtractor | |
| self.device = device | |
| self.fe = Gemma4AudioFeatureExtractor.from_pretrained(model_dir) | |
| self.encoder = EmbeddingGemma2Model.from_pretrained( | |
| model_dir, dtype=torch.float32, vision_config=None | |
| ).to(device).eval() | |
| for p in self.encoder.parameters(): | |
| p.requires_grad_(False) | |
| table = np.load(word_table) if isinstance(word_table, str) else word_table | |
| self.table = torch.from_numpy(table.astype("float32")).to(device) | |
| self.vocab = ( | |
| [w for w in open(vocab, encoding="utf-8").read().splitlines() if w] | |
| if isinstance(vocab, str) | |
| else vocab | |
| ) | |
| self.reader = load_reader(checkpoint_path, table, device) | |
| def soft_tokens(self, wav): | |
| import numpy as np | |
| out = self.fe([wav], return_tensors="pt", sampling_rate=16000) | |
| x = out["input_features"].to(self.device) | |
| m = out["input_features_mask"].to(self.device).bool() | |
| res = self.encoder.get_audio_features(x, m, return_dict=True) | |
| pooled, am = res.pooler_output, res.attention_mask | |
| return pooled[0][am[0]].unsqueeze(0).float() # [1, T, 512] | |
| def transcribe(self, wav): | |
| import numpy as np | |
| if not isinstance(wav, np.ndarray): | |
| wav = np.asarray(wav, dtype="float32") | |
| wav = np.ascontiguousarray(wav, dtype="float32") | |
| soft = self.soft_tokens(wav) | |
| mask = torch.ones(soft.shape[:2], dtype=torch.bool, device=self.device) | |
| logits = self.reader(soft, mask, self.table)[0] | |
| ids = greedy_decode(logits) | |
| return " ".join(self.vocab[i] for i in ids) | |