Download export_reader_fa.py from karim23657/fa-eg2-persian: direct link, hf CLI and curl.
- Browser
- Download file 3.48 kB
-
https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/export_reader_fa.py
- Command line
-
hf download hf://karim23657/fa-eg2-persian/export_reader_fa.py
-
curl -L -o export_reader_fa.py https://huggingface.co/karim23657/fa-eg2-persian/resolve/main/export_reader_fa.py
3.48 kB
| #!/usr/bin/env python3 | |
| """Export a trained reader run to the released checkpoint layout. | |
| Copies ``best_model.pt`` from a training run to ``checkpoints/persian_reader.pt``, writes | |
| ``checkpoints/persian_reader.config.json`` and (optionally) a root-level ``reader.pt`` alias. | |
| The word table and vocabulary live next to the checkpoint as ``word_table_fa.npy`` / | |
| ``vocab_fa.txt``. | |
| Example:: | |
| python export_reader_fa.py --from-run checkpoints/persian_run \ | |
| --vocab word_tables/vocab_fa.txt --table word_tables/word_table_fa.npy --out-dir checkpoints | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import shutil | |
| import torch | |
| from fa_reader_model import PersianReader | |
| CONFIG = { | |
| "architecture": { | |
| "name": "a", | |
| "kind": "transformer encoder from scratch", | |
| "layers": 6, | |
| "d_model": 512, | |
| "heads": 8, | |
| "ffn": 2048, | |
| "activation": "gelu", | |
| "norm": "pre-LayerNorm", | |
| "positions": "rotary (base 10000)", | |
| "input": "LayerNorm -> Linear 512->512", | |
| "output": "LayerNorm -> Linear 512->768", | |
| }, | |
| "params": 19573762, | |
| "vocab": "vocab_fa.txt", | |
| "table": {"file": "word_table_fa.npy"}, | |
| "language": "fa", | |
| } | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--from-run", required=True, help="training run dir holding best_model.pt") | |
| ap.add_argument("--vocab", required=True) | |
| ap.add_argument("--table", required=True) | |
| ap.add_argument("--out-dir", default="checkpoints") | |
| ap.add_argument("--name", default="persian_reader") | |
| args = ap.parse_args() | |
| os.makedirs(args.out_dir, exist_ok=True) | |
| src = os.path.join(args.from_run, "best_model.pt") | |
| if not os.path.exists(src): | |
| src = os.path.join(args.from_run, "last_model.pt") | |
| dst = os.path.join(args.out_dir, f"{args.name}.pt") | |
| state = torch.load(src, map_location="cpu", weights_only=False) | |
| if isinstance(state, dict) and "state_dict" in state and "body.inp.weight" not in state: | |
| state = state["state_dict"] | |
| # infer the architecture from the state dict so any run exports correctly | |
| layers = 1 + max(int(k.split(".")[2]) for k in state if k.startswith("body.blocks.")) | |
| d_model = state["body.inp.weight"].shape[0] | |
| ffn = state["body.blocks.0.ff1.weight"].shape[0] | |
| out_dim = state["body.out.weight"].shape[0] | |
| reader = PersianReader(d_model=d_model, layers=layers, ffn=ffn, out_dim=out_dim) | |
| reader.load_state_dict(state, strict=True) | |
| torch.save(reader.state_dict(), dst) | |
| cfg = json.loads(json.dumps(CONFIG)) | |
| cfg["architecture"]["layers"] = layers | |
| cfg["architecture"]["d_model"] = d_model | |
| cfg["architecture"]["ffn"] = ffn | |
| cfg["params"] = sum(p.numel() for p in reader.parameters()) | |
| with open(os.path.join(args.out_dir, f"{args.name}.config.json"), "w", encoding="utf-8") as f: | |
| json.dump(cfg, f, indent=1, ensure_ascii=False) | |
| # root-level copies used by the documented commands (skip if already at the root) | |
| root = os.getcwd() | |
| def _copy(src, name): | |
| dstp = os.path.join(root, name) | |
| if os.path.abspath(src) == os.path.abspath(dstp): | |
| return dstp | |
| shutil.copyfile(src, dstp) | |
| return dstp | |
| _copy(args.table, "word_table_fa.npy") | |
| _copy(args.vocab, "vocab_fa.txt") | |
| print(f"exported {dst} ({cfg['params']} params); table+vocab at {root}") | |
| if __name__ == "__main__": | |
| main() | |