"""A local progress page for a pod run: scripts/pod monitor EXP-14, then open http://localhost:8714. Every `every` seconds it reads nnU-Net's training log from the pod over SSH (the lines with epoch, losses, pseudo Dice and epoch time; never case IDs) and writes runs/EXP-/monitor/data.json. index.html beside it redraws from that file. When the pod is gone it writes a final snapshot (with our own Dice from metrics.json, once collected) and stops. """ import datetime as dt import json import re import shlex import time from pathlib import Path from strokecopilot.compute import jobs, pod _STAMP = r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\.\d+: " _LINES = { "epoch": re.compile(_STAMP + r"Epoch (\d+)\s*$"), "train_loss": re.compile(_STAMP + r"train_loss (-?[\d.]+)"), "val_loss": re.compile(_STAMP + r"val_loss (-?[\d.]+)"), "dice": re.compile(_STAMP + r"Pseudo dice \[(?:np\.float32\()?(-?[\d.]+)"), "seconds": re.compile(_STAMP + r"Epoch time: ([\d.]+) s"), } EMA = 0.9 # nnU-Net's smoothing of the pseudo Dice: ema = 0.9 * ema + 0.1 * dice def parse_log(text): """[{epoch, train_loss, val_loss, dice, ema, seconds, ended}] for every finished epoch in a training log.""" out, cur, ema = [], None, None for line in text.splitlines(): for key, pattern in _LINES.items(): m = pattern.match(line.strip()) if not m: continue if key == "epoch": cur = {"epoch": int(m.group(2))} elif cur is not None: cur[key] = float(m.group(2)) if key == "seconds": cur["ended"] = m.group(1) if "dice" in cur: ema = cur["dice"] if ema is None else EMA * ema + (1 - EMA) * cur["dice"] cur["ema"] = round(ema, 4) out.append(cur) cur = None break return out # Every training log, oldest first: a resumed run (EXP-14, after Vast stopped the host) starts a new one. REMOTE = ("t=$(ls -tr {run}/nnUNet_results/*/*/fold_*/training_log_*.txt 2>/dev/null); " "[ -n \"$t\" ] && cat $t | grep -E 'Epoch [0-9]+ *$|train_loss|val_loss|Pseudo dice|Epoch time'; " "echo '#EXIT' $(cat {run}/job.exit 2>/dev/null); grep -h 'armed for' /root/sc/watchdog.log 2>/dev/null | tail -1") def summarise(exp, state, log_text, now=None, final_metrics=None): """The data.json content from the pod state, the remote output and the clock.""" now = now or dt.datetime.now(dt.timezone.utc) spec = jobs.get(exp) m = re.search(r"--epochs (\d+)", spec["args"]) total = int(m.group(1)) if m else None epochs = parse_log(log_text) exit_code = None em = re.search(r"#EXIT (\S+)", log_text) if em: exit_code = int(em.group(1)) armed = re.search(r"armed for (\d+) s at (\S+)", log_text) deadline = None if armed: deadline = (dt.datetime.fromisoformat(armed.group(2).replace("Z", "+00:00")) + dt.timedelta(seconds=int(armed.group(1)))).isoformat() created = dt.datetime.fromisoformat(state["created_utc"]) end = dt.datetime.fromisoformat(state["destroyed_utc"]) if state.get("destroyed_utc") else now hours = (end - created).total_seconds() / 3600 recent = [e["seconds"] for e in epochs[-20:]] per_epoch = sorted(recent)[len(recent) // 2] if recent else None left = None if total is None or per_epoch is None else max(0, total - len(epochs)) eta = None if left is None else (now + dt.timedelta(seconds=left * per_epoch + 300)).isoformat() projected = None if left is None else round((hours + (left * per_epoch + 600) / 3600) * state["dph"], 2) if state.get("destroyed"): phase = "finished, pod destroyed" elif exit_code is not None: phase = "job ended (exit %d); collecting" % exit_code elif not epochs: phase = "setting up (upload, install, preprocessing)" elif left == 0: phase = "validating and scoring" else: phase = "training" return { "exp": exp, "what": spec["what"], "phase": phase, "updated_utc": now.isoformat(), "instance": state.get("instance_id"), "price_per_hour": state["dph"], "hours_up": round(hours, 2), "cost_so_far": round(hours * state["dph"], 2), "cost_projected": projected, "hour_limit": state.get("hours_limit"), "watchdog_deadline_utc": deadline, "epochs_total": total, "epochs_done": len(epochs), "seconds_per_epoch": per_epoch, "eta_utc": eta, "best_ema": max((e.get("ema", 0) for e in epochs), default=None), "last": epochs[-1] if epochs else None, "epochs": [{k: e.get(k) for k in ("epoch", "dice", "ema", "seconds", "train_loss", "val_loss")} for e in epochs], "exit_code": exit_code, "final": final_metrics, } def _final_metrics(exp): p = pod.RUNS / exp / "metrics.json" if not p.exists(): return None m = json.loads(p.read_text()) fold = str(m.get("fold", 0)) s = (m.get("summary") or {}).get("per_fold", {}).get(fold, {}) pick = lambda k: {"mean": s[k]["mean"], "std": s[k].get("std"), "n": s[k].get("n")} if isinstance(s.get(k), dict) else None return {"dice": pick("dice"), "dice_reliable": pick("dice_reliable"), "abs_error_ml": pick("abs_error_ml")} def run(exp, every=60, log=print): out = pod.RUNS / exp / "monitor" out.mkdir(parents=True, exist_ok=True) (out / "index.html").write_text(PAGE) last_text = "" while True: state = pod.load_state(exp) if not state: raise pod.PodError(f"{exp}: no pod.json.") if not state.get("destroyed"): try: run_dir = f"{pod.remote.ROOT}/runs/{exp}" last_text = pod.remote_sh(state, REMOTE.replace("{run}", run_dir), timeout=60) except Exception as e: # SSH can drop (VPN); keep the last good snapshot and try again log(f"{dt.datetime.now(dt.timezone.utc):%H:%M} could not reach the pod: {str(e)[:120]}") data = summarise(exp, state, last_text, final_metrics=_final_metrics(exp)) tmp = out / "data.json.tmp" tmp.write_text(json.dumps(data)) tmp.replace(out / "data.json") if state.get("destroyed"): log(f"{exp}: pod destroyed; final snapshot written.") return data time.sleep(every) PAGE = r""" Pod run monitor

Pod run

– loading

Validation pseudo Dice per epoch

pseudo Dice (each epoch)EMA (what nnU-Net uses to pick the best checkpoint)

Pseudo Dice is nnU-Net's quick estimate on random validation patches, not the final score. The final Dice comes from our own scoring after training.

"""