File size: 1,887 Bytes
2abd19a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | #!/usr/bin/env python
"""The public challenge: score any published checkpoint on the held-out split.
python scripts/eval_checkpoint.py # served weights
python scripts/eval_checkpoint.py --repo <owner/repo> --n 500
Metric: BLEU-4 against the 5 human captions of flickr30k *val* (1,014 images the
training run never saw). The number is printed as JSON so it can be compared
across checkpoints, machines and people.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np
import omni.captions as caps
import omni.infer as infer
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--repo", default="hkblockchaintechnology/VLM-all-inclusive")
ap.add_argument("--weights", default="weights/omni_weights.pt")
ap.add_argument("--split", default="val", choices=["val", "test"])
ap.add_argument("--n", type=int, default=200)
ap.add_argument("--token", default=None)
args = ap.parse_args()
system = infer.OmniSystem.from_bundle(args.repo, filename=args.weights, token=args.token)
if not (caps.REAL_DIR / "thumbs_128.npy").exists():
print("preparing the real-data corpus (scripts/fetch_real.sh, then "
"python -c 'import omni.captions as c; c.prepare()')", file=sys.stderr)
return 2
ds = caps.FlickrCaptions(size=128, split=args.split)
t0 = time.time()
model = system.model.eval()
result = caps.evaluate_captions(model, system.vocab, ds, device=system.device, n=args.n)
result.update({"split": args.split, "repo": args.repo, "weights": args.weights,
"seconds": round(time.time() - t0, 1)})
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|