VLM-all-inclusive / scripts /save_all.py
hkblockchaintechnology's picture
repo assets: tests, scripts, Makefile, packaging, refreshed card
080b493 verified
Raw History Blame Contribute Delete
2.49 kB
#!/usr/bin/env python
"""Push every local run artefact to the Hub. The sandbox ekstarts soon, so this is
the difference between 'finished' and 'lost'.
python scripts/save_all.py # models + state + logs + shared
python scripts/save_all.py --with-optim # + optimizer payloads (exact resume)
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import omni.hub as hub
CKPT = "hkblockchaintechnology/omni-vlm-ckpt"
def token() -> str:
import os
t = os.environ.get("HF_TOKEN")
if not t and Path("/marimo/.hf_token").exists():
t = Path("/marimo/.hf_token").read_text().strip()
if not t:
raise SystemExit("no HF token: set HF_TOKEN or write /marimo/.hf_token")
return t
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--repo", default=CKPT)
ap.add_argument("--root", default="/marimo/runs")
ap.add_argument("--with-optim", action="store_true")
ap.add_argument("--force", action="store_true")
args = ap.parse_args()
tok = token()
t0 = time.time()
repo = hub.RemoteRepo(args.repo, token=tok, private=True)
pushed = hub.resync_local_runs(args.repo, tok, root=args.root, retries=3,
include_optim=False, force=args.force)
print(f"runs pushed: {pushed}")
shared = []
for pat, dest in ((f"{args.root}/shared/*.json*", "shared/"),):
for f in Path(args.root).glob(pat.split("/")[-1]):
shared.append((f, dest + f.name))
for f in Path("/marimo/out").glob("*.json*"):
shared.append((f, "reports/" + f.name))
if shared:
repo.commit(shared, "shared artefacts: vocab, alignments, reports")
print(f"shared artefacts: {[d for _, d in shared]}")
if args.with_optim:
deep = hub.resync_local_runs(args.repo, tok, root=args.root, retries=2,
include_optim=True, force=True)
print(f"optimizer payloads pushed: {deep}")
print("verification:")
for stage in ("vlm", "speech", "ear", "vocoder", "align"):
found = hub.find_latest_stage_run(args.repo, stage, tok)
print(f" {stage:8s} {found[0] if found else 'NONE':34s} step={found[1].get('step') if found else '-'}")
print(f"done in {time.time()-t0:.0f}s")
return 0
if __name__ == "__main__":
raise SystemExit(main())