File size: 4,779 Bytes
fcef5b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Choose which checkpoint each stage serves, and write it down.
"Newest run" is a bad rule for inference: a run that has just been started has a
fresher `updated_at` and a *worse* model, so a fresh run would silently take over
serving the moment it appeared. This scores every run of a stage from its own
`log.jsonl` -- the metric the stage actually optimises -- and records the winner in
`shared/active.json` (locally and on the Hub), which `omni.infer` reads through
`hub.open_stage_run`.
python select_served.py # score, print, and record
python select_served.py --dry-run # print only
python select_served.py --set speech=speech-xl-v2-f93271 --set staleness=6000
python select_served.py --min-step 200 # ignore barely-started runs
"""
from __future__ import annotations
import argparse, json, statistics, sys, time
from pathlib import Path
sys.path.insert(0, "/marimo")
import omni.hub as hub
CKPT = "hkblockchaintechnology/omni-vlm-ckpt"
#: The metric each stage actually optimises, from its own log.jsonl.
METRIC = {"vlm": "loss", "captions": "loss", "speech": "phone_loss",
"ear": "ctc_loss", "vocoder": "stft_loss"}
#: A run must beat the incumbent by this much to take over serving, so the
#: served artefact does not flap between two runs that are effectively equal.
MARGIN = 0.02
def say(*a):
print(f"[{time.strftime('%T')}]", *a, flush=True)
def read_log(run_dir: Path, limit: int = 4000):
p = run_dir / "log.jsonl"
if not p.exists():
return []
rows = []
for line in p.read_text(errors="replace").splitlines()[-limit:]:
try:
rows.append(json.loads(line))
except Exception:
pass
return rows
def score(run_dir: Path, metric: str, k: int = 20):
"""Mean of the last `k` values of `metric`, plus the last step seen."""
rows = read_log(run_dir)
vals = [r[metric] for r in rows if isinstance(r.get(metric), (int, float))]
step = max((r.get("step") or 0) for r in rows) if rows else 0
if not vals:
return None, step
return statistics.fmean(vals[-k:]), step
def candidates(root: Path, stage: str):
out = []
for d in sorted(root.glob(f"{stage}-*")):
if not (d / "meta" / "state.json").exists():
continue
if not (d / "latest" / "model.pt").exists():
continue # never checkpointed: nothing to serve
out.append(d)
return out
def choose(root: Path, stage: str, min_step: int, log=say):
metric = METRIC.get(stage)
if metric is None:
return None, None
scored = []
for d in candidates(root, stage):
s, step = score(d, metric)
if step >= min_step:
scored.append((d.name, s, step))
if not scored:
return None, None
have = [t for t in scored if t[1] is not None]
if not have:
log(f" {stage:9s} no run logs {metric}; leaving the selection alone")
return None, None
best = min(have, key=lambda t: t[1])
log(f" {stage:9s} {metric}: " + ", ".join(
f"{n}@{st}={('%.4g' % v) if v is not None else 'n/a'}"
for n, v, st in sorted(scored, key=lambda t: (t[1] is None, t[1]))))
return best[0], best[2]
def main(argv=None):
ap = argparse.ArgumentParser("select_served")
ap.add_argument("--root", default="/marimo/runs")
ap.add_argument("--min-step", type=int, default=200)
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--set", action="append", default=[],
metavar="STAGE=RUN_ID", help="force a stage's served run")
args = ap.parse_args(argv)
root = Path(args.root)
tok = (Path("/marimo/.hf_token").read_text().strip()
if Path("/marimo/.hf_token").exists() else None)
forced = dict(kv.split("=", 1) for kv in args.set)
say(f"scoring runs under {root} (min step {args.min_step})")
chosen = {}
for stage in ("vlm", "speech", "captions", "ear", "vocoder"):
if stage in forced:
chosen[stage] = forced[stage]
say(f" {stage:9s} forced -> {forced[stage]}")
continue
run_id, step = choose(root, stage, args.min_step)
if run_id:
chosen[stage] = run_id
if args.dry_run:
say("dry run; would serve: " + json.dumps(chosen, indent=1))
return 0
for stage, run_id in chosen.items():
step = json.loads((root / run_id / "meta" / "state.json").read_text()).get("step")
hub.set_active_run(CKPT, stage, run_id, step=step, token=tok, root=str(root),
log=say)
say("selection written to /marimo/runs/shared/active.json and mirrored")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|