File size: 3,520 Bytes
080b493
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
#!/usr/bin/env python
"""Model selection: evaluate the saved milestones and serve the best one.

    python scripts/select_checkpoint.py --run vlm-xl-main-df8703 --n 150

Writes `runs/<run>/active.json`; `omni.infer` then serves that milestone while
`latest` keeps model+optimizer in sync for training resume.
"""
from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import numpy as np
import torch

import omni.data as od
import omni.hub as hub
import omni.scenes as scenes
from omni.vlm import OmniVLM, VLMConfig


def read_meta(path):
    try:
        return json.loads(Path(path).read_text())
    except Exception:
        return torch.load(path, map_location="cpu", weights_only=False)


def candidates(run_dir: Path):
    out = [("latest", run_dir / "latest" / "model.pt", run_dir / "latest" / "meta.pt")]
    for d in sorted(run_dir.glob("mile-*")):
        if (d / "model.pt").exists():
            out.append((d.name, d / "model.pt", d / "meta.pt"))
    return [c for c in out if c[1].exists()]


def evaluate(model, vocab, items, device) -> float:
    ok = 0
    for img, q, a in items:
        ids = torch.tensor([[vocab.bos] + vocab.words_to_ids(q.split()) + [vocab.sep]],
                           device=device)
        x = torch.from_numpy(img.astype(np.float32) / 127.5 - 1).permute(2, 0, 1)[None].to(device)
        pred = model.generate_text(ids, images=x, max_new=24)
        ok += int(" ".join(vocab.ids_to_words(pred[0])).strip() == a.strip())
    return 100.0 * ok / len(items)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--run", required=True, help="run id, e.g. vlm-xl-main-df8703")
    ap.add_argument("--root", default="/marimo/runs")
    ap.add_argument("--data-seed", type=int, default=0)
    ap.add_argument("--n", type=int, default=150)
    ap.add_argument("--eval-seed", type=int, default=777)
    args = ap.parse_args()

    device = "cuda" if torch.cuda.is_available() else "cpu"
    vocab = od.make_vocab(args.data_seed)
    run_dir = Path(args.root) / args.run
    rng = np.random.default_rng(args.eval_seed)
    items = []
    for _ in range(args.n):
        sc = scenes.make_scene(rng)
        q, a = scenes.question_answer(sc, rng)
        items.append((scenes.render_scene(sc, px=128), q, a))

    results = []
    for name, model_path, meta_path in candidates(run_dir):
        meta = read_meta(meta_path)
        model = OmniVLM(VLMConfig.from_dict(meta["model"]), n_words=meta["n_words"],
                        n_phones=meta["n_phones"]).to(device).eval()
        model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=False))
        pct = evaluate(model, vocab, items, device)
        results.append({"ckpt": name, "exact_pct": round(pct, 1)})
        print(json.dumps(results[-1]), flush=True)
        del model
        if device == "cuda":
            torch.cuda.empty_cache()

    best = max(results, key=lambda r: r["exact_pct"])
    if best["ckpt"] != "latest":
        step = int(best["ckpt"].split("-")[1])
        hub.RunStore(args.run, root=args.root).set_active(step)
        print(f"serving {best['ckpt']} ({best['exact_pct']}%), written to active.json")
    else:
        print(f"latest is already the best ({best['exact_pct']}%)")
    (Path("/marimo/out") / "ckpt_selection.json").write_text(json.dumps(results, indent=1))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())