File size: 3,520 Bytes
080b493 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | #!/usr/bin/env python
"""Model selection: evaluate the saved milestones and serve the best one.
python scripts/select_checkpoint.py --run vlm-xl-main-df8703 --n 150
Writes `runs/<run>/active.json`; `omni.infer` then serves that milestone while
`latest` keeps model+optimizer in sync for training resume.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
import torch
import omni.data as od
import omni.hub as hub
import omni.scenes as scenes
from omni.vlm import OmniVLM, VLMConfig
def read_meta(path):
try:
return json.loads(Path(path).read_text())
except Exception:
return torch.load(path, map_location="cpu", weights_only=False)
def candidates(run_dir: Path):
out = [("latest", run_dir / "latest" / "model.pt", run_dir / "latest" / "meta.pt")]
for d in sorted(run_dir.glob("mile-*")):
if (d / "model.pt").exists():
out.append((d.name, d / "model.pt", d / "meta.pt"))
return [c for c in out if c[1].exists()]
def evaluate(model, vocab, items, device) -> float:
ok = 0
for img, q, a in items:
ids = torch.tensor([[vocab.bos] + vocab.words_to_ids(q.split()) + [vocab.sep]],
device=device)
x = torch.from_numpy(img.astype(np.float32) / 127.5 - 1).permute(2, 0, 1)[None].to(device)
pred = model.generate_text(ids, images=x, max_new=24)
ok += int(" ".join(vocab.ids_to_words(pred[0])).strip() == a.strip())
return 100.0 * ok / len(items)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--run", required=True, help="run id, e.g. vlm-xl-main-df8703")
ap.add_argument("--root", default="/marimo/runs")
ap.add_argument("--data-seed", type=int, default=0)
ap.add_argument("--n", type=int, default=150)
ap.add_argument("--eval-seed", type=int, default=777)
args = ap.parse_args()
device = "cuda" if torch.cuda.is_available() else "cpu"
vocab = od.make_vocab(args.data_seed)
run_dir = Path(args.root) / args.run
rng = np.random.default_rng(args.eval_seed)
items = []
for _ in range(args.n):
sc = scenes.make_scene(rng)
q, a = scenes.question_answer(sc, rng)
items.append((scenes.render_scene(sc, px=128), q, a))
results = []
for name, model_path, meta_path in candidates(run_dir):
meta = read_meta(meta_path)
model = OmniVLM(VLMConfig.from_dict(meta["model"]), n_words=meta["n_words"],
n_phones=meta["n_phones"]).to(device).eval()
model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=False))
pct = evaluate(model, vocab, items, device)
results.append({"ckpt": name, "exact_pct": round(pct, 1)})
print(json.dumps(results[-1]), flush=True)
del model
if device == "cuda":
torch.cuda.empty_cache()
best = max(results, key=lambda r: r["exact_pct"])
if best["ckpt"] != "latest":
step = int(best["ckpt"].split("-")[1])
hub.RunStore(args.run, root=args.root).set_active(step)
print(f"serving {best['ckpt']} ({best['exact_pct']}%), written to active.json")
else:
print(f"latest is already the best ({best['exact_pct']}%)")
(Path("/marimo/out") / "ckpt_selection.json").write_text(json.dumps(results, indent=1))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|