Download scripts/select_checkpoint.py from hkblockchaintechnology/VLM-all-inclusive: direct link, hf CLI and curl.
- Browser
- Download file 3.52 kB
-
https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/select_checkpoint.py
- Command line
-
hf download hf://hkblockchaintechnology/VLM-all-inclusive/scripts/select_checkpoint.py
-
curl -L -o select_checkpoint.py https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/select_checkpoint.py
3.52 kB
| #!/usr/bin/env python | |
| """Model selection: evaluate the saved milestones and serve the best one. | |
| python scripts/select_checkpoint.py --run vlm-xl-main-df8703 --n 150 | |
| Writes `runs/<run>/active.json`; `omni.infer` then serves that milestone while | |
| `latest` keeps model+optimizer in sync for training resume. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| import numpy as np | |
| import torch | |
| import omni.data as od | |
| import omni.hub as hub | |
| import omni.scenes as scenes | |
| from omni.vlm import OmniVLM, VLMConfig | |
| def read_meta(path): | |
| try: | |
| return json.loads(Path(path).read_text()) | |
| except Exception: | |
| return torch.load(path, map_location="cpu", weights_only=False) | |
| def candidates(run_dir: Path): | |
| out = [("latest", run_dir / "latest" / "model.pt", run_dir / "latest" / "meta.pt")] | |
| for d in sorted(run_dir.glob("mile-*")): | |
| if (d / "model.pt").exists(): | |
| out.append((d.name, d / "model.pt", d / "meta.pt")) | |
| return [c for c in out if c[1].exists()] | |
| def evaluate(model, vocab, items, device) -> float: | |
| ok = 0 | |
| for img, q, a in items: | |
| ids = torch.tensor([[vocab.bos] + vocab.words_to_ids(q.split()) + [vocab.sep]], | |
| device=device) | |
| x = torch.from_numpy(img.astype(np.float32) / 127.5 - 1).permute(2, 0, 1)[None].to(device) | |
| pred = model.generate_text(ids, images=x, max_new=24) | |
| ok += int(" ".join(vocab.ids_to_words(pred[0])).strip() == a.strip()) | |
| return 100.0 * ok / len(items) | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--run", required=True, help="run id, e.g. vlm-xl-main-df8703") | |
| ap.add_argument("--root", default="/marimo/runs") | |
| ap.add_argument("--data-seed", type=int, default=0) | |
| ap.add_argument("--n", type=int, default=150) | |
| ap.add_argument("--eval-seed", type=int, default=777) | |
| args = ap.parse_args() | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| vocab = od.make_vocab(args.data_seed) | |
| run_dir = Path(args.root) / args.run | |
| rng = np.random.default_rng(args.eval_seed) | |
| items = [] | |
| for _ in range(args.n): | |
| sc = scenes.make_scene(rng) | |
| q, a = scenes.question_answer(sc, rng) | |
| items.append((scenes.render_scene(sc, px=128), q, a)) | |
| results = [] | |
| for name, model_path, meta_path in candidates(run_dir): | |
| meta = read_meta(meta_path) | |
| model = OmniVLM(VLMConfig.from_dict(meta["model"]), n_words=meta["n_words"], | |
| n_phones=meta["n_phones"]).to(device).eval() | |
| model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=False)) | |
| pct = evaluate(model, vocab, items, device) | |
| results.append({"ckpt": name, "exact_pct": round(pct, 1)}) | |
| print(json.dumps(results[-1]), flush=True) | |
| del model | |
| if device == "cuda": | |
| torch.cuda.empty_cache() | |
| best = max(results, key=lambda r: r["exact_pct"]) | |
| if best["ckpt"] != "latest": | |
| step = int(best["ckpt"].split("-")[1]) | |
| hub.RunStore(args.run, root=args.root).set_active(step) | |
| print(f"serving {best['ckpt']} ({best['exact_pct']}%), written to active.json") | |
| else: | |
| print(f"latest is already the best ({best['exact_pct']}%)") | |
| (Path("/marimo/out") / "ckpt_selection.json").write_text(json.dumps(results, indent=1)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |