VLM-all-inclusive / scripts /select_served.py
hkblockchaintechnology's picture
code: resume-by-discovery, real-data stage v2, GAN vocoder, restore/backup tooling
fcef5b0 verified
Raw History Blame Contribute Delete
4.78 kB
"""Choose which checkpoint each stage serves, and write it down.
"Newest run" is a bad rule for inference: a run that has just been started has a
fresher `updated_at` and a *worse* model, so a fresh run would silently take over
serving the moment it appeared. This scores every run of a stage from its own
`log.jsonl` -- the metric the stage actually optimises -- and records the winner in
`shared/active.json` (locally and on the Hub), which `omni.infer` reads through
`hub.open_stage_run`.
python select_served.py # score, print, and record
python select_served.py --dry-run # print only
python select_served.py --set speech=speech-xl-v2-f93271 --set staleness=6000
python select_served.py --min-step 200 # ignore barely-started runs
"""
from __future__ import annotations
import argparse, json, statistics, sys, time
from pathlib import Path
sys.path.insert(0, "/marimo")
import omni.hub as hub
CKPT = "hkblockchaintechnology/omni-vlm-ckpt"
#: The metric each stage actually optimises, from its own log.jsonl.
METRIC = {"vlm": "loss", "captions": "loss", "speech": "phone_loss",
"ear": "ctc_loss", "vocoder": "stft_loss"}
#: A run must beat the incumbent by this much to take over serving, so the
#: served artefact does not flap between two runs that are effectively equal.
MARGIN = 0.02
def say(*a):
print(f"[{time.strftime('%T')}]", *a, flush=True)
def read_log(run_dir: Path, limit: int = 4000):
p = run_dir / "log.jsonl"
if not p.exists():
return []
rows = []
for line in p.read_text(errors="replace").splitlines()[-limit:]:
try:
rows.append(json.loads(line))
except Exception:
pass
return rows
def score(run_dir: Path, metric: str, k: int = 20):
"""Mean of the last `k` values of `metric`, plus the last step seen."""
rows = read_log(run_dir)
vals = [r[metric] for r in rows if isinstance(r.get(metric), (int, float))]
step = max((r.get("step") or 0) for r in rows) if rows else 0
if not vals:
return None, step
return statistics.fmean(vals[-k:]), step
def candidates(root: Path, stage: str):
out = []
for d in sorted(root.glob(f"{stage}-*")):
if not (d / "meta" / "state.json").exists():
continue
if not (d / "latest" / "model.pt").exists():
continue # never checkpointed: nothing to serve
out.append(d)
return out
def choose(root: Path, stage: str, min_step: int, log=say):
metric = METRIC.get(stage)
if metric is None:
return None, None
scored = []
for d in candidates(root, stage):
s, step = score(d, metric)
if step >= min_step:
scored.append((d.name, s, step))
if not scored:
return None, None
have = [t for t in scored if t[1] is not None]
if not have:
log(f" {stage:9s} no run logs {metric}; leaving the selection alone")
return None, None
best = min(have, key=lambda t: t[1])
log(f" {stage:9s} {metric}: " + ", ".join(
f"{n}@{st}={('%.4g' % v) if v is not None else 'n/a'}"
for n, v, st in sorted(scored, key=lambda t: (t[1] is None, t[1]))))
return best[0], best[2]
def main(argv=None):
ap = argparse.ArgumentParser("select_served")
ap.add_argument("--root", default="/marimo/runs")
ap.add_argument("--min-step", type=int, default=200)
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--set", action="append", default=[],
metavar="STAGE=RUN_ID", help="force a stage's served run")
args = ap.parse_args(argv)
root = Path(args.root)
tok = (Path("/marimo/.hf_token").read_text().strip()
if Path("/marimo/.hf_token").exists() else None)
forced = dict(kv.split("=", 1) for kv in args.set)
say(f"scoring runs under {root} (min step {args.min_step})")
chosen = {}
for stage in ("vlm", "speech", "captions", "ear", "vocoder"):
if stage in forced:
chosen[stage] = forced[stage]
say(f" {stage:9s} forced -> {forced[stage]}")
continue
run_id, step = choose(root, stage, args.min_step)
if run_id:
chosen[stage] = run_id
if args.dry_run:
say("dry run; would serve: " + json.dumps(chosen, indent=1))
return 0
for stage, run_id in chosen.items():
step = json.loads((root / run_id / "meta" / "state.json").read_text()).get("step")
hub.set_active_run(CKPT, stage, run_id, step=step, token=tok, root=str(root),
log=say)
say("selection written to /marimo/runs/shared/active.json and mirrored")
return 0
if __name__ == "__main__":
raise SystemExit(main())