Download scripts/save_all.py from hkblockchaintechnology/VLM-all-inclusive: direct link, hf CLI and curl.
- Browser
- Download file 2.49 kB
-
https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/save_all.py
- Command line
-
hf download hf://hkblockchaintechnology/VLM-all-inclusive/scripts/save_all.py
-
curl -L -o save_all.py https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/save_all.py
2.49 kB
| #!/usr/bin/env python | |
| """Push every local run artefact to the Hub. The sandbox ekstarts soon, so this is | |
| the difference between 'finished' and 'lost'. | |
| python scripts/save_all.py # models + state + logs + shared | |
| python scripts/save_all.py --with-optim # + optimizer payloads (exact resume) | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| import omni.hub as hub | |
| CKPT = "hkblockchaintechnology/omni-vlm-ckpt" | |
| def token() -> str: | |
| import os | |
| t = os.environ.get("HF_TOKEN") | |
| if not t and Path("/marimo/.hf_token").exists(): | |
| t = Path("/marimo/.hf_token").read_text().strip() | |
| if not t: | |
| raise SystemExit("no HF token: set HF_TOKEN or write /marimo/.hf_token") | |
| return t | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--repo", default=CKPT) | |
| ap.add_argument("--root", default="/marimo/runs") | |
| ap.add_argument("--with-optim", action="store_true") | |
| ap.add_argument("--force", action="store_true") | |
| args = ap.parse_args() | |
| tok = token() | |
| t0 = time.time() | |
| repo = hub.RemoteRepo(args.repo, token=tok, private=True) | |
| pushed = hub.resync_local_runs(args.repo, tok, root=args.root, retries=3, | |
| include_optim=False, force=args.force) | |
| print(f"runs pushed: {pushed}") | |
| shared = [] | |
| for pat, dest in ((f"{args.root}/shared/*.json*", "shared/"),): | |
| for f in Path(args.root).glob(pat.split("/")[-1]): | |
| shared.append((f, dest + f.name)) | |
| for f in Path("/marimo/out").glob("*.json*"): | |
| shared.append((f, "reports/" + f.name)) | |
| if shared: | |
| repo.commit(shared, "shared artefacts: vocab, alignments, reports") | |
| print(f"shared artefacts: {[d for _, d in shared]}") | |
| if args.with_optim: | |
| deep = hub.resync_local_runs(args.repo, tok, root=args.root, retries=2, | |
| include_optim=True, force=True) | |
| print(f"optimizer payloads pushed: {deep}") | |
| print("verification:") | |
| for stage in ("vlm", "speech", "ear", "vocoder", "align"): | |
| found = hub.find_latest_stage_run(args.repo, stage, tok) | |
| print(f" {stage:8s} {found[0] if found else 'NONE':34s} step={found[1].get('step') if found else '-'}") | |
| print(f"done in {time.time()-t0:.0f}s") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |