Download scripts/eval_checkpoint.py from hkblockchaintechnology/VLM-all-inclusive: direct link, hf CLI and curl.
- Browser
- Download file 1.89 kB
-
https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/eval_checkpoint.py
- Command line
-
hf download hf://hkblockchaintechnology/VLM-all-inclusive/scripts/eval_checkpoint.py
-
curl -L -o eval_checkpoint.py https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/eval_checkpoint.py
1.89 kB
| #!/usr/bin/env python | |
| """The public challenge: score any published checkpoint on the held-out split. | |
| python scripts/eval_checkpoint.py # served weights | |
| python scripts/eval_checkpoint.py --repo <owner/repo> --n 500 | |
| Metric: BLEU-4 against the 5 human captions of flickr30k *val* (1,014 images the | |
| training run never saw). The number is printed as JSON so it can be compared | |
| across checkpoints, machines and people. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| import numpy as np | |
| import omni.captions as caps | |
| import omni.infer as infer | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--repo", default="hkblockchaintechnology/VLM-all-inclusive") | |
| ap.add_argument("--weights", default="weights/omni_weights.pt") | |
| ap.add_argument("--split", default="val", choices=["val", "test"]) | |
| ap.add_argument("--n", type=int, default=200) | |
| ap.add_argument("--token", default=None) | |
| args = ap.parse_args() | |
| system = infer.OmniSystem.from_bundle(args.repo, filename=args.weights, token=args.token) | |
| if not (caps.REAL_DIR / "thumbs_128.npy").exists(): | |
| print("preparing the real-data corpus (scripts/fetch_real.sh, then " | |
| "python -c 'import omni.captions as c; c.prepare()')", file=sys.stderr) | |
| return 2 | |
| ds = caps.FlickrCaptions(size=128, split=args.split) | |
| t0 = time.time() | |
| model = system.model.eval() | |
| result = caps.evaluate_captions(model, system.vocab, ds, device=system.device, n=args.n) | |
| result.update({"split": args.split, "repo": args.repo, "weights": args.weights, | |
| "seconds": round(time.time() - t0, 1)}) | |
| print(json.dumps(result, indent=2)) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |