#!/usr/bin/env python """The public challenge: score any published checkpoint on the held-out split. python scripts/eval_checkpoint.py # served weights python scripts/eval_checkpoint.py --repo --n 500 Metric: BLEU-4 against the 5 human captions of flickr30k *val* (1,014 images the training run never saw). The number is printed as JSON so it can be compared across checkpoints, machines and people. """ from __future__ import annotations import argparse import json import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import numpy as np import omni.captions as caps import omni.infer as infer def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--repo", default="hkblockchaintechnology/VLM-all-inclusive") ap.add_argument("--weights", default="weights/omni_weights.pt") ap.add_argument("--split", default="val", choices=["val", "test"]) ap.add_argument("--n", type=int, default=200) ap.add_argument("--token", default=None) args = ap.parse_args() system = infer.OmniSystem.from_bundle(args.repo, filename=args.weights, token=args.token) if not (caps.REAL_DIR / "thumbs_128.npy").exists(): print("preparing the real-data corpus (scripts/fetch_real.sh, then " "python -c 'import omni.captions as c; c.prepare()')", file=sys.stderr) return 2 ds = caps.FlickrCaptions(size=128, split=args.split) t0 = time.time() model = system.model.eval() result = caps.evaluate_captions(model, system.vocab, ds, device=system.device, n=args.n) result.update({"split": args.split, "repo": args.repo, "weights": args.weights, "seconds": round(time.time() - t0, 1)}) print(json.dumps(result, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())