"""Choose which checkpoint each stage serves, and write it down. "Newest run" is a bad rule for inference: a run that has just been started has a fresher `updated_at` and a *worse* model, so a fresh run would silently take over serving the moment it appeared. This scores every run of a stage from its own `log.jsonl` -- the metric the stage actually optimises -- and records the winner in `shared/active.json` (locally and on the Hub), which `omni.infer` reads through `hub.open_stage_run`. python select_served.py # score, print, and record python select_served.py --dry-run # print only python select_served.py --set speech=speech-xl-v2-f93271 --set staleness=6000 python select_served.py --min-step 200 # ignore barely-started runs """ from __future__ import annotations import argparse, json, statistics, sys, time from pathlib import Path sys.path.insert(0, "/marimo") import omni.hub as hub CKPT = "hkblockchaintechnology/omni-vlm-ckpt" #: The metric each stage actually optimises, from its own log.jsonl. METRIC = {"vlm": "loss", "captions": "loss", "speech": "phone_loss", "ear": "ctc_loss", "vocoder": "stft_loss"} #: A run must beat the incumbent by this much to take over serving, so the #: served artefact does not flap between two runs that are effectively equal. MARGIN = 0.02 def say(*a): print(f"[{time.strftime('%T')}]", *a, flush=True) def read_log(run_dir: Path, limit: int = 4000): p = run_dir / "log.jsonl" if not p.exists(): return [] rows = [] for line in p.read_text(errors="replace").splitlines()[-limit:]: try: rows.append(json.loads(line)) except Exception: pass return rows def score(run_dir: Path, metric: str, k: int = 20): """Mean of the last `k` values of `metric`, plus the last step seen.""" rows = read_log(run_dir) vals = [r[metric] for r in rows if isinstance(r.get(metric), (int, float))] step = max((r.get("step") or 0) for r in rows) if rows else 0 if not vals: return None, step return statistics.fmean(vals[-k:]), step def candidates(root: Path, stage: str): out = [] for d in sorted(root.glob(f"{stage}-*")): if not (d / "meta" / "state.json").exists(): continue if not (d / "latest" / "model.pt").exists(): continue # never checkpointed: nothing to serve out.append(d) return out def choose(root: Path, stage: str, min_step: int, log=say): metric = METRIC.get(stage) if metric is None: return None, None scored = [] for d in candidates(root, stage): s, step = score(d, metric) if step >= min_step: scored.append((d.name, s, step)) if not scored: return None, None have = [t for t in scored if t[1] is not None] if not have: log(f" {stage:9s} no run logs {metric}; leaving the selection alone") return None, None best = min(have, key=lambda t: t[1]) log(f" {stage:9s} {metric}: " + ", ".join( f"{n}@{st}={('%.4g' % v) if v is not None else 'n/a'}" for n, v, st in sorted(scored, key=lambda t: (t[1] is None, t[1])))) return best[0], best[2] def main(argv=None): ap = argparse.ArgumentParser("select_served") ap.add_argument("--root", default="/marimo/runs") ap.add_argument("--min-step", type=int, default=200) ap.add_argument("--dry-run", action="store_true") ap.add_argument("--set", action="append", default=[], metavar="STAGE=RUN_ID", help="force a stage's served run") args = ap.parse_args(argv) root = Path(args.root) tok = (Path("/marimo/.hf_token").read_text().strip() if Path("/marimo/.hf_token").exists() else None) forced = dict(kv.split("=", 1) for kv in args.set) say(f"scoring runs under {root} (min step {args.min_step})") chosen = {} for stage in ("vlm", "speech", "captions", "ear", "vocoder"): if stage in forced: chosen[stage] = forced[stage] say(f" {stage:9s} forced -> {forced[stage]}") continue run_id, step = choose(root, stage, args.min_step) if run_id: chosen[stage] = run_id if args.dry_run: say("dry run; would serve: " + json.dumps(chosen, indent=1)) return 0 for stage, run_id in chosen.items(): step = json.loads((root / run_id / "meta" / "state.json").read_text()).get("step") hub.set_active_run(CKPT, stage, run_id, step=step, token=tok, root=str(root), log=say) say("selection written to /marimo/runs/shared/active.json and mirrored") return 0 if __name__ == "__main__": raise SystemExit(main())