Download train_reader_fa.py from karim23657/fa-eg2-persian: direct link, hf CLI and curl.
- Browser
- Download file 6.99 kB
-
https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/train_reader_fa.py
- Command line
-
hf download hf://karim23657/fa-eg2-persian/train_reader_fa.py
-
curl -L -o train_reader_fa.py https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/train_reader_fa.py
6.99 kB
| #!/usr/bin/env python3 | |
| """Train the FA-EG2 Persian reader (frozen encoder, CTC over the word table). | |
| The EmbeddingGemma 2 audio encoder is frozen: its soft tokens are pre-computed by | |
| ``cache_soft_tokens_fa.py`` and the reader is trained directly on them. The reader body | |
| (6-layer transformer, 19.6 M params) is trained with CTC against the Persian word table; | |
| per epoch the validation WER is measured and the best checkpoint is kept (early stopping). | |
| Config: ``config/persian.yaml``. Example:: | |
| python train_reader_fa.py --config config/persian.yaml | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| import yaml | |
| import persian_text as PT | |
| from data_fa import SoftTokenCache, make_batches, collate | |
| from fa_reader_model import PersianReader, greedy_decode | |
| def log(msg): | |
| print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) | |
| def cosine_lr(step, total, warmup, base_lr, min_lr=0.0): | |
| if step < warmup: | |
| return base_lr * (step + 1) / max(1, warmup) | |
| p = (step - warmup) / max(1, total - warmup) | |
| return min_lr + 0.5 * (base_lr - min_lr) * (1 + np.cos(np.pi * min(p, 1.0))) | |
| def evaluate_wer(reader, table, vocab, cache, device, batch_size=32, max_utts=0): | |
| reader.eval() | |
| items = cache.items[: max_utts] if max_utts else cache.items | |
| hyps, refs = [], [] | |
| order = sorted(range(len(items)), key=lambda i: items[i][1]) | |
| for a in range(0, len(order), batch_size): | |
| idx = order[a:a + batch_size] | |
| soft, mask, _tgt, _tl, _il = collate(cache, idx, device) | |
| logits = reader(soft, mask, table) | |
| for j, i in enumerate(idx): | |
| t = cache.soft(cache.items[i][0]).shape[0] | |
| ids = greedy_decode(logits[j, :t]) | |
| hyps.append(PT.normalize(" ".join(vocab[k] for k in ids))) | |
| refs.extend("" for _ in idx) | |
| # rebuild refs in the same order | |
| refs = [PT.normalize(cache.index_full[i][4]) for i in order] | |
| import jiwer | |
| return jiwer.wer([r or " " for r in refs], [h or " " for h in hyps]) | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--config", required=True) | |
| args = ap.parse_args() | |
| cfg = yaml.safe_load(open(args.config, encoding="utf-8")) | |
| device = cfg.get("device") or ("cuda" if torch.cuda.is_available() else "cpu") | |
| cache_dir = cfg["cache"] | |
| out_dir = cfg["out_dir"] | |
| os.makedirs(out_dir, exist_ok=True) | |
| def _resolve(path, name): | |
| """Use the configured path, else fall back to the same basename at the package root.""" | |
| if path and os.path.exists(path): | |
| return path | |
| alt = os.path.join(os.path.dirname(os.path.abspath(args.config)), "..", os.path.basename(path or name)) | |
| alt = os.path.normpath(alt) | |
| if path and os.path.exists(os.path.join(os.getcwd(), os.path.basename(path))): | |
| return os.path.join(os.getcwd(), os.path.basename(path)) | |
| if os.path.exists(alt): | |
| return alt | |
| return path | |
| vocab_path = _resolve(cfg["vocab"], "vocab_fa.txt") | |
| table_path = _resolve(cfg["word_table"], "word_table_fa.npy") | |
| vocab = [w for w in open(vocab_path, encoding="utf-8").read().splitlines() if w] | |
| table = torch.from_numpy(np.load(table_path).astype("float32")).to(device) | |
| log(f"vocab {len(vocab)} ({vocab_path}), table {tuple(table.shape)} ({table_path}) on {device}") | |
| train = SoftTokenCache(cache_dir, cfg.get("train_split", "train"), vocab) | |
| dev = SoftTokenCache(cache_dir, cfg.get("dev_split", "validation"), vocab) | |
| dev.index_full = dev.index | |
| # attach full index rows to train too (for reference text) | |
| train.index_full = train.index | |
| log(f"train {len(train)} utts (OOV {100 * train.oov_rate:.2f} %), dev {len(dev)} utts") | |
| reader = PersianReader( | |
| d_model=cfg.get("d_model", 512), layers=cfg.get("layers", 6), | |
| heads=cfg.get("heads", 8), ffn=cfg.get("ffn", 2048), | |
| ).to(device) | |
| epochs = cfg.get("epochs", 12) | |
| batch_size = cfg.get("batch_size", 16) | |
| base_lr = cfg.get("lr", 3e-4) | |
| warmup = cfg.get("warmup_steps", 500) | |
| clip = cfg.get("grad_clip", 1.0) | |
| patience = cfg.get("patience", 3) | |
| from torch.optim import AdamW | |
| opt = AdamW(reader.parameters(), lr=base_lr, betas=(0.9, 0.98), weight_decay=cfg.get("weight_decay", 0.01)) | |
| batches_per_epoch = max(1, len(make_batches(train.items, batch_size))) | |
| total_steps = epochs * batches_per_epoch | |
| if device == "cuda": | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| step = 0 | |
| best_wer = float("inf") | |
| best_epoch = -1 | |
| no_improve = 0 | |
| metrics = [] | |
| for epoch in range(1, epochs + 1): | |
| reader.train() | |
| t0 = time.time() | |
| run = 0.0 | |
| nb = 0 | |
| for batch in make_batches(train.items, batch_size, shuffle=True, seed=epoch): | |
| soft, mask, targets, tlen, ilen = collate(train, batch, device) | |
| lr = cosine_lr(step, total_steps, warmup, base_lr) | |
| for g in opt.param_groups: | |
| g["lr"] = lr | |
| logits = reader(soft, mask, table) # [B,T,N+1] | |
| lp = F.log_softmax(logits, dim=-1).transpose(0, 1) # [T,B,N+1] | |
| loss = F.ctc_loss(lp, targets, ilen, tlen, blank=table.shape[0], reduction="mean", | |
| zero_infinity=True) | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(reader.parameters(), clip) | |
| opt.step() | |
| run += float(loss.item()) | |
| nb += 1 | |
| step += 1 | |
| if nb % 100 == 0: | |
| log(f"epoch {epoch} step {nb}/{batches_per_epoch} loss {run / nb:.3f} lr {lr:.2e}") | |
| train_loss = run / max(1, nb) | |
| wer = evaluate_wer(reader, table, vocab, dev, device, max_utts=cfg.get("dev_max_utts", 0)) | |
| log(f"epoch {epoch}: train loss {train_loss:.3f}, val WER {100 * wer:.2f} % ({time.time() - t0:.0f} s)") | |
| metrics.append({"epoch": epoch, "train_loss": train_loss, "val_wer": wer, "lr": lr}) | |
| torch.save(reader.state_dict(), os.path.join(out_dir, "last_model.pt")) | |
| if wer < best_wer: | |
| best_wer = wer | |
| best_epoch = epoch | |
| no_improve = 0 | |
| torch.save(reader.state_dict(), os.path.join(out_dir, "best_model.pt")) | |
| else: | |
| no_improve += 1 | |
| if no_improve >= patience: | |
| log(f"early stop at epoch {epoch} (best {100 * best_wer:.2f} % @ epoch {best_epoch})") | |
| break | |
| with open(os.path.join(out_dir, "metrics.json"), "w", encoding="utf-8") as f: | |
| json.dump({"best_epoch": best_epoch, "best_val_wer": best_wer, "history": metrics}, f, indent=1) | |
| log(f"done. best val WER {100 * best_wer:.2f} % @ epoch {best_epoch}; checkpoints in {out_dir}") | |
| if __name__ == "__main__": | |
| main() | |