fa-eg2-persian / train_reader_fa.py
karim23657's picture
FA-EG2: model + full source + technical results
d1597be verified
Raw History Blame Contribute Delete
6.99 kB
#!/usr/bin/env python3
"""Train the FA-EG2 Persian reader (frozen encoder, CTC over the word table).
The EmbeddingGemma 2 audio encoder is frozen: its soft tokens are pre-computed by
``cache_soft_tokens_fa.py`` and the reader is trained directly on them. The reader body
(6-layer transformer, 19.6 M params) is trained with CTC against the Persian word table;
per epoch the validation WER is measured and the best checkpoint is kept (early stopping).
Config: ``config/persian.yaml``. Example::
python train_reader_fa.py --config config/persian.yaml
"""
import argparse
import json
import os
import time
import numpy as np
import torch
import torch.nn.functional as F
import yaml
import persian_text as PT
from data_fa import SoftTokenCache, make_batches, collate
from fa_reader_model import PersianReader, greedy_decode
def log(msg):
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
def cosine_lr(step, total, warmup, base_lr, min_lr=0.0):
if step < warmup:
return base_lr * (step + 1) / max(1, warmup)
p = (step - warmup) / max(1, total - warmup)
return min_lr + 0.5 * (base_lr - min_lr) * (1 + np.cos(np.pi * min(p, 1.0)))
@torch.no_grad()
def evaluate_wer(reader, table, vocab, cache, device, batch_size=32, max_utts=0):
reader.eval()
items = cache.items[: max_utts] if max_utts else cache.items
hyps, refs = [], []
order = sorted(range(len(items)), key=lambda i: items[i][1])
for a in range(0, len(order), batch_size):
idx = order[a:a + batch_size]
soft, mask, _tgt, _tl, _il = collate(cache, idx, device)
logits = reader(soft, mask, table)
for j, i in enumerate(idx):
t = cache.soft(cache.items[i][0]).shape[0]
ids = greedy_decode(logits[j, :t])
hyps.append(PT.normalize(" ".join(vocab[k] for k in ids)))
refs.extend("" for _ in idx)
# rebuild refs in the same order
refs = [PT.normalize(cache.index_full[i][4]) for i in order]
import jiwer
return jiwer.wer([r or " " for r in refs], [h or " " for h in hyps])
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--config", required=True)
args = ap.parse_args()
cfg = yaml.safe_load(open(args.config, encoding="utf-8"))
device = cfg.get("device") or ("cuda" if torch.cuda.is_available() else "cpu")
cache_dir = cfg["cache"]
out_dir = cfg["out_dir"]
os.makedirs(out_dir, exist_ok=True)
def _resolve(path, name):
"""Use the configured path, else fall back to the same basename at the package root."""
if path and os.path.exists(path):
return path
alt = os.path.join(os.path.dirname(os.path.abspath(args.config)), "..", os.path.basename(path or name))
alt = os.path.normpath(alt)
if path and os.path.exists(os.path.join(os.getcwd(), os.path.basename(path))):
return os.path.join(os.getcwd(), os.path.basename(path))
if os.path.exists(alt):
return alt
return path
vocab_path = _resolve(cfg["vocab"], "vocab_fa.txt")
table_path = _resolve(cfg["word_table"], "word_table_fa.npy")
vocab = [w for w in open(vocab_path, encoding="utf-8").read().splitlines() if w]
table = torch.from_numpy(np.load(table_path).astype("float32")).to(device)
log(f"vocab {len(vocab)} ({vocab_path}), table {tuple(table.shape)} ({table_path}) on {device}")
train = SoftTokenCache(cache_dir, cfg.get("train_split", "train"), vocab)
dev = SoftTokenCache(cache_dir, cfg.get("dev_split", "validation"), vocab)
dev.index_full = dev.index
# attach full index rows to train too (for reference text)
train.index_full = train.index
log(f"train {len(train)} utts (OOV {100 * train.oov_rate:.2f} %), dev {len(dev)} utts")
reader = PersianReader(
d_model=cfg.get("d_model", 512), layers=cfg.get("layers", 6),
heads=cfg.get("heads", 8), ffn=cfg.get("ffn", 2048),
).to(device)
epochs = cfg.get("epochs", 12)
batch_size = cfg.get("batch_size", 16)
base_lr = cfg.get("lr", 3e-4)
warmup = cfg.get("warmup_steps", 500)
clip = cfg.get("grad_clip", 1.0)
patience = cfg.get("patience", 3)
from torch.optim import AdamW
opt = AdamW(reader.parameters(), lr=base_lr, betas=(0.9, 0.98), weight_decay=cfg.get("weight_decay", 0.01))
batches_per_epoch = max(1, len(make_batches(train.items, batch_size)))
total_steps = epochs * batches_per_epoch
if device == "cuda":
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
step = 0
best_wer = float("inf")
best_epoch = -1
no_improve = 0
metrics = []
for epoch in range(1, epochs + 1):
reader.train()
t0 = time.time()
run = 0.0
nb = 0
for batch in make_batches(train.items, batch_size, shuffle=True, seed=epoch):
soft, mask, targets, tlen, ilen = collate(train, batch, device)
lr = cosine_lr(step, total_steps, warmup, base_lr)
for g in opt.param_groups:
g["lr"] = lr
logits = reader(soft, mask, table) # [B,T,N+1]
lp = F.log_softmax(logits, dim=-1).transpose(0, 1) # [T,B,N+1]
loss = F.ctc_loss(lp, targets, ilen, tlen, blank=table.shape[0], reduction="mean",
zero_infinity=True)
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(reader.parameters(), clip)
opt.step()
run += float(loss.item())
nb += 1
step += 1
if nb % 100 == 0:
log(f"epoch {epoch} step {nb}/{batches_per_epoch} loss {run / nb:.3f} lr {lr:.2e}")
train_loss = run / max(1, nb)
wer = evaluate_wer(reader, table, vocab, dev, device, max_utts=cfg.get("dev_max_utts", 0))
log(f"epoch {epoch}: train loss {train_loss:.3f}, val WER {100 * wer:.2f} % ({time.time() - t0:.0f} s)")
metrics.append({"epoch": epoch, "train_loss": train_loss, "val_wer": wer, "lr": lr})
torch.save(reader.state_dict(), os.path.join(out_dir, "last_model.pt"))
if wer < best_wer:
best_wer = wer
best_epoch = epoch
no_improve = 0
torch.save(reader.state_dict(), os.path.join(out_dir, "best_model.pt"))
else:
no_improve += 1
if no_improve >= patience:
log(f"early stop at epoch {epoch} (best {100 * best_wer:.2f} % @ epoch {best_epoch})")
break
with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f:
json.dump({"best_epoch": best_epoch, "best_val_wer": best_wer, "history": metrics}, f, indent=1)
log(f"done. best val WER {100 * best_wer:.2f} % @ epoch {best_epoch}; checkpoints in {out_dir}")
if __name__ == "__main__":
main()