#!/usr/bin/env python """Model selection: evaluate the saved milestones and serve the best one. python scripts/select_checkpoint.py --run vlm-xl-main-df8703 --n 150 Writes `runs//active.json`; `omni.infer` then serves that milestone while `latest` keeps model+optimizer in sync for training resume. """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import numpy as np import torch import omni.data as od import omni.hub as hub import omni.scenes as scenes from omni.vlm import OmniVLM, VLMConfig def read_meta(path): try: return json.loads(Path(path).read_text()) except Exception: return torch.load(path, map_location="cpu", weights_only=False) def candidates(run_dir: Path): out = [("latest", run_dir / "latest" / "model.pt", run_dir / "latest" / "meta.pt")] for d in sorted(run_dir.glob("mile-*")): if (d / "model.pt").exists(): out.append((d.name, d / "model.pt", d / "meta.pt")) return [c for c in out if c[1].exists()] def evaluate(model, vocab, items, device) -> float: ok = 0 for img, q, a in items: ids = torch.tensor([[vocab.bos] + vocab.words_to_ids(q.split()) + [vocab.sep]], device=device) x = torch.from_numpy(img.astype(np.float32) / 127.5 - 1).permute(2, 0, 1)[None].to(device) pred = model.generate_text(ids, images=x, max_new=24) ok += int(" ".join(vocab.ids_to_words(pred[0])).strip() == a.strip()) return 100.0 * ok / len(items) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--run", required=True, help="run id, e.g. vlm-xl-main-df8703") ap.add_argument("--root", default="/marimo/runs") ap.add_argument("--data-seed", type=int, default=0) ap.add_argument("--n", type=int, default=150) ap.add_argument("--eval-seed", type=int, default=777) args = ap.parse_args() device = "cuda" if torch.cuda.is_available() else "cpu" vocab = od.make_vocab(args.data_seed) run_dir = Path(args.root) / args.run rng = np.random.default_rng(args.eval_seed) items = [] for _ in range(args.n): sc = scenes.make_scene(rng) q, a = scenes.question_answer(sc, rng) items.append((scenes.render_scene(sc, px=128), q, a)) results = [] for name, model_path, meta_path in candidates(run_dir): meta = read_meta(meta_path) model = OmniVLM(VLMConfig.from_dict(meta["model"]), n_words=meta["n_words"], n_phones=meta["n_phones"]).to(device).eval() model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=False)) pct = evaluate(model, vocab, items, device) results.append({"ckpt": name, "exact_pct": round(pct, 1)}) print(json.dumps(results[-1]), flush=True) del model if device == "cuda": torch.cuda.empty_cache() best = max(results, key=lambda r: r["exact_pct"]) if best["ckpt"] != "latest": step = int(best["ckpt"].split("-")[1]) hub.RunStore(args.run, root=args.root).set_active(step) print(f"serving {best['ckpt']} ({best['exact_pct']}%), written to active.json") else: print(f"latest is already the best ({best['exact_pct']}%)") (Path("/marimo/out") / "ckpt_selection.json").write_text(json.dumps(results, indent=1)) return 0 if __name__ == "__main__": raise SystemExit(main())