Download scripts/select_served.py from hkblockchaintechnology/VLM-all-inclusive: direct link, hf CLI and curl.
- Browser
- Download file 4.78 kB
-
https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/select_served.py
- Command line
-
hf download hf://hkblockchaintechnology/VLM-all-inclusive/scripts/select_served.py
-
curl -L -o select_served.py https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/select_served.py
4.78 kB
| """Choose which checkpoint each stage serves, and write it down. | |
| "Newest run" is a bad rule for inference: a run that has just been started has a | |
| fresher `updated_at` and a *worse* model, so a fresh run would silently take over | |
| serving the moment it appeared. This scores every run of a stage from its own | |
| `log.jsonl` -- the metric the stage actually optimises -- and records the winner in | |
| `shared/active.json` (locally and on the Hub), which `omni.infer` reads through | |
| `hub.open_stage_run`. | |
| python select_served.py # score, print, and record | |
| python select_served.py --dry-run # print only | |
| python select_served.py --set speech=speech-xl-v2-f93271 --set staleness=6000 | |
| python select_served.py --min-step 200 # ignore barely-started runs | |
| """ | |
| from __future__ import annotations | |
| import argparse, json, statistics, sys, time | |
| from pathlib import Path | |
| sys.path.insert(0, "/marimo") | |
| import omni.hub as hub | |
| CKPT = "hkblockchaintechnology/omni-vlm-ckpt" | |
| #: The metric each stage actually optimises, from its own log.jsonl. | |
| METRIC = {"vlm": "loss", "captions": "loss", "speech": "phone_loss", | |
| "ear": "ctc_loss", "vocoder": "stft_loss"} | |
| #: A run must beat the incumbent by this much to take over serving, so the | |
| #: served artefact does not flap between two runs that are effectively equal. | |
| MARGIN = 0.02 | |
| def say(*a): | |
| print(f"[{time.strftime('%T')}]", *a, flush=True) | |
| def read_log(run_dir: Path, limit: int = 4000): | |
| p = run_dir / "log.jsonl" | |
| if not p.exists(): | |
| return [] | |
| rows = [] | |
| for line in p.read_text(errors="replace").splitlines()[-limit:]: | |
| try: | |
| rows.append(json.loads(line)) | |
| except Exception: | |
| pass | |
| return rows | |
| def score(run_dir: Path, metric: str, k: int = 20): | |
| """Mean of the last `k` values of `metric`, plus the last step seen.""" | |
| rows = read_log(run_dir) | |
| vals = [r[metric] for r in rows if isinstance(r.get(metric), (int, float))] | |
| step = max((r.get("step") or 0) for r in rows) if rows else 0 | |
| if not vals: | |
| return None, step | |
| return statistics.fmean(vals[-k:]), step | |
| def candidates(root: Path, stage: str): | |
| out = [] | |
| for d in sorted(root.glob(f"{stage}-*")): | |
| if not (d / "meta" / "state.json").exists(): | |
| continue | |
| if not (d / "latest" / "model.pt").exists(): | |
| continue # never checkpointed: nothing to serve | |
| out.append(d) | |
| return out | |
| def choose(root: Path, stage: str, min_step: int, log=say): | |
| metric = METRIC.get(stage) | |
| if metric is None: | |
| return None, None | |
| scored = [] | |
| for d in candidates(root, stage): | |
| s, step = score(d, metric) | |
| if step >= min_step: | |
| scored.append((d.name, s, step)) | |
| if not scored: | |
| return None, None | |
| have = [t for t in scored if t[1] is not None] | |
| if not have: | |
| log(f" {stage:9s} no run logs {metric}; leaving the selection alone") | |
| return None, None | |
| best = min(have, key=lambda t: t[1]) | |
| log(f" {stage:9s} {metric}: " + ", ".join( | |
| f"{n}@{st}={('%.4g' % v) if v is not None else 'n/a'}" | |
| for n, v, st in sorted(scored, key=lambda t: (t[1] is None, t[1])))) | |
| return best[0], best[2] | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser("select_served") | |
| ap.add_argument("--root", default="/marimo/runs") | |
| ap.add_argument("--min-step", type=int, default=200) | |
| ap.add_argument("--dry-run", action="store_true") | |
| ap.add_argument("--set", action="append", default=[], | |
| metavar="STAGE=RUN_ID", help="force a stage's served run") | |
| args = ap.parse_args(argv) | |
| root = Path(args.root) | |
| tok = (Path("/marimo/.hf_token").read_text().strip() | |
| if Path("/marimo/.hf_token").exists() else None) | |
| forced = dict(kv.split("=", 1) for kv in args.set) | |
| say(f"scoring runs under {root} (min step {args.min_step})") | |
| chosen = {} | |
| for stage in ("vlm", "speech", "captions", "ear", "vocoder"): | |
| if stage in forced: | |
| chosen[stage] = forced[stage] | |
| say(f" {stage:9s} forced -> {forced[stage]}") | |
| continue | |
| run_id, step = choose(root, stage, args.min_step) | |
| if run_id: | |
| chosen[stage] = run_id | |
| if args.dry_run: | |
| say("dry run; would serve: " + json.dumps(chosen, indent=1)) | |
| return 0 | |
| for stage, run_id in chosen.items(): | |
| step = json.loads((root / run_id / "meta" / "state.json").read_text()).get("step") | |
| hub.set_active_run(CKPT, stage, run_id, step=step, token=tok, root=str(root), | |
| log=say) | |
| say("selection written to /marimo/runs/shared/active.json and mirrored") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |