#!/usr/bin/env python3 """Train the FA-EG2 Persian reader (frozen encoder, CTC over the word table). The EmbeddingGemma 2 audio encoder is frozen: its soft tokens are pre-computed by ``cache_soft_tokens_fa.py`` and the reader is trained directly on them. The reader body (6-layer transformer, 19.6 M params) is trained with CTC against the Persian word table; per epoch the validation WER is measured and the best checkpoint is kept (early stopping). Config: ``config/persian.yaml``. Example:: python train_reader_fa.py --config config/persian.yaml """ import argparse import json import os import time import numpy as np import torch import torch.nn.functional as F import yaml import persian_text as PT from data_fa import SoftTokenCache, make_batches, collate from fa_reader_model import PersianReader, greedy_decode def log(msg): print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) def cosine_lr(step, total, warmup, base_lr, min_lr=0.0): if step < warmup: return base_lr * (step + 1) / max(1, warmup) p = (step - warmup) / max(1, total - warmup) return min_lr + 0.5 * (base_lr - min_lr) * (1 + np.cos(np.pi * min(p, 1.0))) @torch.no_grad() def evaluate_wer(reader, table, vocab, cache, device, batch_size=32, max_utts=0): reader.eval() items = cache.items[: max_utts] if max_utts else cache.items hyps, refs = [], [] order = sorted(range(len(items)), key=lambda i: items[i][1]) for a in range(0, len(order), batch_size): idx = order[a:a + batch_size] soft, mask, _tgt, _tl, _il = collate(cache, idx, device) logits = reader(soft, mask, table) for j, i in enumerate(idx): t = cache.soft(cache.items[i][0]).shape[0] ids = greedy_decode(logits[j, :t]) hyps.append(PT.normalize(" ".join(vocab[k] for k in ids))) refs.extend("" for _ in idx) # rebuild refs in the same order refs = [PT.normalize(cache.index_full[i][4]) for i in order] import jiwer return jiwer.wer([r or " " for r in refs], [h or " " for h in hyps]) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--config", required=True) args = ap.parse_args() cfg = yaml.safe_load(open(args.config, encoding="utf-8")) device = cfg.get("device") or ("cuda" if torch.cuda.is_available() else "cpu") cache_dir = cfg["cache"] out_dir = cfg["out_dir"] os.makedirs(out_dir, exist_ok=True) def _resolve(path, name): """Use the configured path, else fall back to the same basename at the package root.""" if path and os.path.exists(path): return path alt = os.path.join(os.path.dirname(os.path.abspath(args.config)), "..", os.path.basename(path or name)) alt = os.path.normpath(alt) if path and os.path.exists(os.path.join(os.getcwd(), os.path.basename(path))): return os.path.join(os.getcwd(), os.path.basename(path)) if os.path.exists(alt): return alt return path vocab_path = _resolve(cfg["vocab"], "vocab_fa.txt") table_path = _resolve(cfg["word_table"], "word_table_fa.npy") vocab = [w for w in open(vocab_path, encoding="utf-8").read().splitlines() if w] table = torch.from_numpy(np.load(table_path).astype("float32")).to(device) log(f"vocab {len(vocab)} ({vocab_path}), table {tuple(table.shape)} ({table_path}) on {device}") train = SoftTokenCache(cache_dir, cfg.get("train_split", "train"), vocab) dev = SoftTokenCache(cache_dir, cfg.get("dev_split", "validation"), vocab) dev.index_full = dev.index # attach full index rows to train too (for reference text) train.index_full = train.index log(f"train {len(train)} utts (OOV {100 * train.oov_rate:.2f} %), dev {len(dev)} utts") reader = PersianReader( d_model=cfg.get("d_model", 512), layers=cfg.get("layers", 6), heads=cfg.get("heads", 8), ffn=cfg.get("ffn", 2048), ).to(device) epochs = cfg.get("epochs", 12) batch_size = cfg.get("batch_size", 16) base_lr = cfg.get("lr", 3e-4) warmup = cfg.get("warmup_steps", 500) clip = cfg.get("grad_clip", 1.0) patience = cfg.get("patience", 3) from torch.optim import AdamW opt = AdamW(reader.parameters(), lr=base_lr, betas=(0.9, 0.98), weight_decay=cfg.get("weight_decay", 0.01)) batches_per_epoch = max(1, len(make_batches(train.items, batch_size))) total_steps = epochs * batches_per_epoch if device == "cuda": torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True step = 0 best_wer = float("inf") best_epoch = -1 no_improve = 0 metrics = [] for epoch in range(1, epochs + 1): reader.train() t0 = time.time() run = 0.0 nb = 0 for batch in make_batches(train.items, batch_size, shuffle=True, seed=epoch): soft, mask, targets, tlen, ilen = collate(train, batch, device) lr = cosine_lr(step, total_steps, warmup, base_lr) for g in opt.param_groups: g["lr"] = lr logits = reader(soft, mask, table) # [B,T,N+1] lp = F.log_softmax(logits, dim=-1).transpose(0, 1) # [T,B,N+1] loss = F.ctc_loss(lp, targets, ilen, tlen, blank=table.shape[0], reduction="mean", zero_infinity=True) opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(reader.parameters(), clip) opt.step() run += float(loss.item()) nb += 1 step += 1 if nb % 100 == 0: log(f"epoch {epoch} step {nb}/{batches_per_epoch} loss {run / nb:.3f} lr {lr:.2e}") train_loss = run / max(1, nb) wer = evaluate_wer(reader, table, vocab, dev, device, max_utts=cfg.get("dev_max_utts", 0)) log(f"epoch {epoch}: train loss {train_loss:.3f}, val WER {100 * wer:.2f} % ({time.time() - t0:.0f} s)") metrics.append({"epoch": epoch, "train_loss": train_loss, "val_wer": wer, "lr": lr}) torch.save(reader.state_dict(), os.path.join(out_dir, "last_model.pt")) if wer < best_wer: best_wer = wer best_epoch = epoch no_improve = 0 torch.save(reader.state_dict(), os.path.join(out_dir, "best_model.pt")) else: no_improve += 1 if no_improve >= patience: log(f"early stop at epoch {epoch} (best {100 * best_wer:.2f} % @ epoch {best_epoch})") break with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: json.dump({"best_epoch": best_epoch, "best_val_wer": best_wer, "history": metrics}, f, indent=1) log(f"done. best val WER {100 * best_wer:.2f} % @ epoch {best_epoch}; checkpoints in {out_dir}") if __name__ == "__main__": main()