fa-eg2-persian / fa_reader_model.py
karim23657's picture
FA-EG2: model + full source + technical results
d1597be verified
Raw History Blame Contribute Delete
10.1 kB
#!/usr/bin/env python3
"""FA-EG2 Persian reader: transformer body over frozen-encoder soft tokens.
Architecture ``a`` (the released Persian reader): a 6-layer pre-LayerNorm transformer
encoder built from scratch over the frozen EmbeddingGemma 2 audio soft tokens
``[T, 512]`` (25 tokens/s, 40 ms each). Per token the body emits a 768-d unit vector
``h_t``; a single blank logit is produced by a learned vector. Word logits are
logits[w] = scale * cos(h_t, word_table[w]) (scale = exp(log_scale))
logits[blank] = <blank.weight, h_t> + blank.bias
so the reader is trained with ``torch.nn.functional.ctc_loss`` over ``N + 1``
classes (N Persian words + blank). Greedy CTC decoding merges repeated word ids and
drops blanks.
The module is state-dict compatible with ``checkpoints/persian_reader.pt``
(a plain ``OrderedDict`` of tensors): ``log_scale``, ``body.*`` and ``blank.*``.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
BLANK = 0 # class index of the CTC blank is not used here; blank is a separate logit
# --------------------------------------------------------------------------- #
# Rotary positional embeddings
# --------------------------------------------------------------------------- #
def build_rope_cache(seq_len: int, dim: int, base: float = 10000.0, device=None, dtype=torch.float32):
"""Return (cos, sin) each [seq_len, dim]. dim must be even."""
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))
t = torch.arange(seq_len, dtype=torch.float32, device=device)
freqs = torch.outer(t, inv_freq) # [T, dim/2]
emb = torch.cat([freqs, freqs], dim=-1) # [T, dim]
return emb.cos().to(dtype), emb.sin().to(dtype)
def apply_rope(x, cos, sin):
"""x: [B, H, T, Dh]; cos/sin: [T, Dh]."""
d = x.shape[-1]
x1, x2 = x[..., : d // 2], x[..., d // 2:]
rotated = torch.cat([-x2, x1], dim=-1)
cos = cos[None, None, :, :]
sin = sin[None, None, :, :]
return x * cos + rotated * sin
class Block(nn.Module):
"""Pre-LayerNorm transformer block with MHSA (RoPE) + GELU MLP."""
def __init__(self, d_model=512, heads=8, ffn=2048, rope_base=10000.0):
super().__init__()
assert d_model % heads == 0
self.d_model = d_model
self.heads = heads
self.head_dim = d_model // heads
self.rope_base = rope_base
self.ln1 = nn.LayerNorm(d_model)
self.qkv = nn.Linear(d_model, 3 * d_model)
self.proj = nn.Linear(d_model, d_model)
self.ln2 = nn.LayerNorm(d_model)
self.ff1 = nn.Linear(d_model, ffn)
self.ff2 = nn.Linear(ffn, d_model)
def forward(self, x, mask=None, rope=None):
B, T, D = x.shape
h = self.ln1(x)
qkv = self.qkv(h).view(B, T, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0], qkv[1], qkv[2] # [B, H, T, Dh]
if rope is not None:
cos, sin = rope
q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
attn = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
if mask is not None:
# mask: [B, T] True = valid
neg = torch.finfo(attn.dtype).min
attn = attn.masked_fill(~mask[:, None, None, :], neg)
attn = attn.softmax(dim=-1)
out = (attn @ v).transpose(1, 2).reshape(B, T, D)
x = x + self.proj(out)
h = self.ln2(x)
x = x + self.ff2(F.gelu(self.ff1(h)))
return x
class ReaderBody(nn.Module):
"""LayerNorm -> Linear 512->512 -> N transformer blocks -> LayerNorm -> Linear 512->768."""
def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0):
super().__init__()
self.d_model = d_model
self.layers = layers
self.heads = heads
self.ffn = ffn
self.out_dim = out_dim
self.rope_base = rope_base
self.ln_in = nn.LayerNorm(d_model)
self.inp = nn.Linear(d_model, d_model)
self.blocks = nn.ModuleList(Block(d_model, heads, ffn, rope_base) for _ in range(layers))
self.ln_out = nn.LayerNorm(d_model)
self.out = nn.Linear(d_model, out_dim)
def forward(self, x, mask=None):
# x: [B, T, d_model] soft tokens
x = self.inp(self.ln_in(x))
T = x.shape[1]
cos, sin = build_rope_cache(T, x.shape[-1] // self.heads, self.rope_base, x.device, x.dtype)
rope = (cos, sin)
for blk in self.blocks:
x = blk(x, mask=mask, rope=rope)
return self.out(self.ln_out(x))
class PersianReader(nn.Module):
"""Body + blank logit; logits over ``N + 1`` classes (N = word-table size)."""
def __init__(self, d_model=512, layers=6, heads=8, ffn=2048, out_dim=768, rope_base=10000.0):
super().__init__()
self.body = ReaderBody(d_model, layers, heads, ffn, out_dim, rope_base)
self.blank = nn.Linear(out_dim, 1)
self.log_scale = nn.Parameter(torch.tensor(math.log(64.0)))
def forward(self, soft_tokens, mask, table):
"""soft_tokens: [B,T,512]; mask: [B,T] bool; table: [N,768] float32 (L2-normalized).
Returns logits [B, T, N+1] where index 0..N-1 are words and index N is blank.
Differentiable (no internal ``torch.no_grad``); inference callers wrap it in
``torch.no_grad()``.
"""
h = self.body(soft_tokens, mask) # [B,T,768] raw
hn = F.normalize(h, dim=-1)
scale = self.log_scale.exp()
word_logits = scale * torch.einsum("btd,nd->btn", hn, table) # [B,T,N]
blank_logit = self.blank(h) # [B,T,1] from raw body output
return torch.cat([word_logits, blank_logit], dim=-1)
def build_reader_from_config(cfg: dict) -> PersianReader:
arch = cfg.get("architecture", {})
return PersianReader(
d_model=arch.get("d_model", 512),
layers=arch.get("layers", 6),
heads=arch.get("heads", 8),
ffn=arch.get("ffn", 2048),
out_dim=arch.get("out_dim", 768),
)
def load_reader(checkpoint_path, table, device="cpu") -> PersianReader:
"""Load ``checkpoints/persian_reader.pt`` (a plain state dict) into a PersianReader.
The architecture (layers, d_model, ffn, out_dim) is inferred from the state dict."""
state = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
if isinstance(state, dict) and "state_dict" in state and "body.inp.weight" not in state:
state = state["state_dict"]
layers = 1 + max(int(k.split(".")[2]) for k in state if k.startswith("body.blocks."))
reader = PersianReader(
d_model=state["body.inp.weight"].shape[0],
layers=layers,
ffn=state["body.blocks.0.ff1.weight"].shape[0],
out_dim=state["body.out.weight"].shape[0],
)
missing, unexpected = reader.load_state_dict(state, strict=False)
if missing or unexpected:
raise RuntimeError(f"checkpoint mismatch: missing={missing} unexpected={unexpected}")
reader.eval()
return reader.to(device)
@torch.no_grad()
def greedy_decode(logits, table_ids_to_words=None):
"""logits: [T, N+1] -> list of class ids (word ids, blank merged away).
Blank is the last class (index N). Repeated ids are merged, then blanks dropped.
"""
ids = logits.argmax(dim=-1).tolist()
n_classes = logits.shape[-1]
blank_id = n_classes - 1
out = []
prev = None
for i in ids:
if i != prev:
if i != blank_id:
out.append(i)
prev = i
return out
# --------------------------------------------------------------------------- #
# Full reader = frozen encoder + reader, for end-to-end transcription
# --------------------------------------------------------------------------- #
class FAEG2Transcriber(nn.Module):
"""Frozen EmbeddingGemma 2 audio encoder + trained Persian reader.
``transcribe(wav_16k)`` returns the greedy-decoded Persian string. Pass a
``feature_extractor`` to compute log-mel features exactly as in caching.
"""
def __init__(self, model_dir, checkpoint_path, word_table, vocab, device="cpu"):
super().__init__()
import numpy as np
from transformers import EmbeddingGemma2Model, Gemma4AudioFeatureExtractor
self.device = device
self.fe = Gemma4AudioFeatureExtractor.from_pretrained(model_dir)
self.encoder = EmbeddingGemma2Model.from_pretrained(
model_dir, dtype=torch.float32, vision_config=None
).to(device).eval()
for p in self.encoder.parameters():
p.requires_grad_(False)
table = np.load(word_table) if isinstance(word_table, str) else word_table
self.table = torch.from_numpy(table.astype("float32")).to(device)
self.vocab = (
[w for w in open(vocab, encoding="utf-8").read().splitlines() if w]
if isinstance(vocab, str)
else vocab
)
self.reader = load_reader(checkpoint_path, table, device)
@torch.no_grad()
def soft_tokens(self, wav):
import numpy as np
out = self.fe([wav], return_tensors="pt", sampling_rate=16000)
x = out["input_features"].to(self.device)
m = out["input_features_mask"].to(self.device).bool()
res = self.encoder.get_audio_features(x, m, return_dict=True)
pooled, am = res.pooler_output, res.attention_mask
return pooled[0][am[0]].unsqueeze(0).float() # [1, T, 512]
@torch.no_grad()
def transcribe(self, wav):
import numpy as np
if not isinstance(wav, np.ndarray):
wav = np.asarray(wav, dtype="float32")
wav = np.ascontiguousarray(wav, dtype="float32")
soft = self.soft_tokens(wav)
mask = torch.ones(soft.shape[:2], dtype=torch.bool, device=self.device)
logits = self.reader(soft, mask, self.table)[0]
ids = greedy_decode(logits)
return " ".join(self.vocab[i] for i in ids)