File size: 4,779 Bytes
fcef5b0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Choose which checkpoint each stage serves, and write it down.

"Newest run" is a bad rule for inference: a run that has just been started has a
fresher `updated_at` and a *worse* model, so a fresh run would silently take over
serving the moment it appeared. This scores every run of a stage from its own
`log.jsonl` -- the metric the stage actually optimises -- and records the winner in
`shared/active.json` (locally and on the Hub), which `omni.infer` reads through
`hub.open_stage_run`.

    python select_served.py                 # score, print, and record
    python select_served.py --dry-run       # print only
    python select_served.py --set speech=speech-xl-v2-f93271 --set staleness=6000
    python select_served.py --min-step 200  # ignore barely-started runs
"""
from __future__ import annotations
import argparse, json, statistics, sys, time
from pathlib import Path

sys.path.insert(0, "/marimo")
import omni.hub as hub

CKPT = "hkblockchaintechnology/omni-vlm-ckpt"

#: The metric each stage actually optimises, from its own log.jsonl.
METRIC = {"vlm": "loss", "captions": "loss", "speech": "phone_loss",
          "ear": "ctc_loss", "vocoder": "stft_loss"}

#: A run must beat the incumbent by this much to take over serving, so the
#: served artefact does not flap between two runs that are effectively equal.
MARGIN = 0.02


def say(*a):
    print(f"[{time.strftime('%T')}]", *a, flush=True)


def read_log(run_dir: Path, limit: int = 4000):
    p = run_dir / "log.jsonl"
    if not p.exists():
        return []
    rows = []
    for line in p.read_text(errors="replace").splitlines()[-limit:]:
        try:
            rows.append(json.loads(line))
        except Exception:
            pass
    return rows


def score(run_dir: Path, metric: str, k: int = 20):
    """Mean of the last `k` values of `metric`, plus the last step seen."""
    rows = read_log(run_dir)
    vals = [r[metric] for r in rows if isinstance(r.get(metric), (int, float))]
    step = max((r.get("step") or 0) for r in rows) if rows else 0
    if not vals:
        return None, step
    return statistics.fmean(vals[-k:]), step


def candidates(root: Path, stage: str):
    out = []
    for d in sorted(root.glob(f"{stage}-*")):
        if not (d / "meta" / "state.json").exists():
            continue
        if not (d / "latest" / "model.pt").exists():
            continue                       # never checkpointed: nothing to serve
        out.append(d)
    return out


def choose(root: Path, stage: str, min_step: int, log=say):
    metric = METRIC.get(stage)
    if metric is None:
        return None, None
    scored = []
    for d in candidates(root, stage):
        s, step = score(d, metric)
        if step >= min_step:
            scored.append((d.name, s, step))
    if not scored:
        return None, None
    have = [t for t in scored if t[1] is not None]
    if not have:
        log(f"  {stage:9s} no run logs {metric}; leaving the selection alone")
        return None, None
    best = min(have, key=lambda t: t[1])
    log(f"  {stage:9s} {metric}: " + ", ".join(
        f"{n}@{st}={('%.4g' % v) if v is not None else 'n/a'}"
        for n, v, st in sorted(scored, key=lambda t: (t[1] is None, t[1]))))
    return best[0], best[2]


def main(argv=None):
    ap = argparse.ArgumentParser("select_served")
    ap.add_argument("--root", default="/marimo/runs")
    ap.add_argument("--min-step", type=int, default=200)
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--set", action="append", default=[],
                    metavar="STAGE=RUN_ID", help="force a stage's served run")
    args = ap.parse_args(argv)
    root = Path(args.root)
    tok = (Path("/marimo/.hf_token").read_text().strip()
           if Path("/marimo/.hf_token").exists() else None)

    forced = dict(kv.split("=", 1) for kv in args.set)
    say(f"scoring runs under {root} (min step {args.min_step})")
    chosen = {}
    for stage in ("vlm", "speech", "captions", "ear", "vocoder"):
        if stage in forced:
            chosen[stage] = forced[stage]
            say(f"  {stage:9s} forced -> {forced[stage]}")
            continue
        run_id, step = choose(root, stage, args.min_step)
        if run_id:
            chosen[stage] = run_id

    if args.dry_run:
        say("dry run; would serve: " + json.dumps(chosen, indent=1))
        return 0
    for stage, run_id in chosen.items():
        step = json.loads((root / run_id / "meta" / "state.json").read_text()).get("step")
        hub.set_active_run(CKPT, stage, run_id, step=step, token=tok, root=str(root),
                           log=say)
    say("selection written to /marimo/runs/shared/active.json and mirrored")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())