Spaces:
Running
Running
File size: 12,831 Bytes
974048c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 | """A local progress page for a pod run: scripts/pod monitor EXP-14, then open http://localhost:8714.
Every `every` seconds it reads nnU-Net's training log from the pod over SSH (the lines with epoch, losses, pseudo Dice
and epoch time; never case IDs) and writes runs/EXP-<id>/monitor/data.json. index.html beside it redraws from that
file. When the pod is gone it writes a final snapshot (with our own Dice from metrics.json, once collected) and stops.
"""
import datetime as dt
import json
import re
import shlex
import time
from pathlib import Path
from strokecopilot.compute import jobs, pod
_STAMP = r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\.\d+: "
_LINES = {
"epoch": re.compile(_STAMP + r"Epoch (\d+)\s*$"),
"train_loss": re.compile(_STAMP + r"train_loss (-?[\d.]+)"),
"val_loss": re.compile(_STAMP + r"val_loss (-?[\d.]+)"),
"dice": re.compile(_STAMP + r"Pseudo dice \[(?:np\.float32\()?(-?[\d.]+)"),
"seconds": re.compile(_STAMP + r"Epoch time: ([\d.]+) s"),
}
EMA = 0.9 # nnU-Net's smoothing of the pseudo Dice: ema = 0.9 * ema + 0.1 * dice
def parse_log(text):
"""[{epoch, train_loss, val_loss, dice, ema, seconds, ended}] for every finished epoch in a training log."""
out, cur, ema = [], None, None
for line in text.splitlines():
for key, pattern in _LINES.items():
m = pattern.match(line.strip())
if not m:
continue
if key == "epoch":
cur = {"epoch": int(m.group(2))}
elif cur is not None:
cur[key] = float(m.group(2))
if key == "seconds":
cur["ended"] = m.group(1)
if "dice" in cur:
ema = cur["dice"] if ema is None else EMA * ema + (1 - EMA) * cur["dice"]
cur["ema"] = round(ema, 4)
out.append(cur)
cur = None
break
return out
# Every training log, oldest first: a resumed run (EXP-14, after Vast stopped the host) starts a new one.
REMOTE = ("t=$(ls -tr {run}/nnUNet_results/*/*/fold_*/training_log_*.txt 2>/dev/null); "
"[ -n \"$t\" ] && cat $t | grep -E 'Epoch [0-9]+ *$|train_loss|val_loss|Pseudo dice|Epoch time'; "
"echo '#EXIT' $(cat {run}/job.exit 2>/dev/null); grep -h 'armed for' /root/sc/watchdog.log 2>/dev/null | tail -1")
def summarise(exp, state, log_text, now=None, final_metrics=None):
"""The data.json content from the pod state, the remote output and the clock."""
now = now or dt.datetime.now(dt.timezone.utc)
spec = jobs.get(exp)
m = re.search(r"--epochs (\d+)", spec["args"])
total = int(m.group(1)) if m else None
epochs = parse_log(log_text)
exit_code = None
em = re.search(r"#EXIT (\S+)", log_text)
if em:
exit_code = int(em.group(1))
armed = re.search(r"armed for (\d+) s at (\S+)", log_text)
deadline = None
if armed:
deadline = (dt.datetime.fromisoformat(armed.group(2).replace("Z", "+00:00")) + dt.timedelta(seconds=int(armed.group(1)))).isoformat()
created = dt.datetime.fromisoformat(state["created_utc"])
end = dt.datetime.fromisoformat(state["destroyed_utc"]) if state.get("destroyed_utc") else now
hours = (end - created).total_seconds() / 3600
recent = [e["seconds"] for e in epochs[-20:]]
per_epoch = sorted(recent)[len(recent) // 2] if recent else None
left = None if total is None or per_epoch is None else max(0, total - len(epochs))
eta = None if left is None else (now + dt.timedelta(seconds=left * per_epoch + 300)).isoformat()
projected = None if left is None else round((hours + (left * per_epoch + 600) / 3600) * state["dph"], 2)
if state.get("destroyed"):
phase = "finished, pod destroyed"
elif exit_code is not None:
phase = "job ended (exit %d); collecting" % exit_code
elif not epochs:
phase = "setting up (upload, install, preprocessing)"
elif left == 0:
phase = "validating and scoring"
else:
phase = "training"
return {
"exp": exp, "what": spec["what"], "phase": phase, "updated_utc": now.isoformat(), "instance": state.get("instance_id"),
"price_per_hour": state["dph"], "hours_up": round(hours, 2), "cost_so_far": round(hours * state["dph"], 2),
"cost_projected": projected, "hour_limit": state.get("hours_limit"), "watchdog_deadline_utc": deadline,
"epochs_total": total, "epochs_done": len(epochs), "seconds_per_epoch": per_epoch, "eta_utc": eta,
"best_ema": max((e.get("ema", 0) for e in epochs), default=None), "last": epochs[-1] if epochs else None,
"epochs": [{k: e.get(k) for k in ("epoch", "dice", "ema", "seconds", "train_loss", "val_loss")} for e in epochs],
"exit_code": exit_code, "final": final_metrics,
}
def _final_metrics(exp):
p = pod.RUNS / exp / "metrics.json"
if not p.exists():
return None
m = json.loads(p.read_text())
fold = str(m.get("fold", 0))
s = (m.get("summary") or {}).get("per_fold", {}).get(fold, {})
pick = lambda k: {"mean": s[k]["mean"], "std": s[k].get("std"), "n": s[k].get("n")} if isinstance(s.get(k), dict) else None
return {"dice": pick("dice"), "dice_reliable": pick("dice_reliable"), "abs_error_ml": pick("abs_error_ml")}
def run(exp, every=60, log=print):
out = pod.RUNS / exp / "monitor"
out.mkdir(parents=True, exist_ok=True)
(out / "index.html").write_text(PAGE)
last_text = ""
while True:
state = pod.load_state(exp)
if not state:
raise pod.PodError(f"{exp}: no pod.json.")
if not state.get("destroyed"):
try:
run_dir = f"{pod.remote.ROOT}/runs/{exp}"
last_text = pod.remote_sh(state, REMOTE.replace("{run}", run_dir), timeout=60)
except Exception as e: # SSH can drop (VPN); keep the last good snapshot and try again
log(f"{dt.datetime.now(dt.timezone.utc):%H:%M} could not reach the pod: {str(e)[:120]}")
data = summarise(exp, state, last_text, final_metrics=_final_metrics(exp))
tmp = out / "data.json.tmp"
tmp.write_text(json.dumps(data))
tmp.replace(out / "data.json")
if state.get("destroyed"):
log(f"{exp}: pod destroyed; final snapshot written.")
return data
time.sleep(every)
PAGE = r"""<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>Pod run monitor</title>
<style>
:root{--bg:#f7f7f5;--card:#fff;--ink:#1d1d1b;--muted:#6b6b66;--line:#e4e4df;--accent:#2f6fde;--accent2:#c9d9f7;--ok:#1f8a4c;--warn:#b5651d;--bad:#c0392b}
@media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#121212;--card:#1c1c1c;--ink:#ececea;--muted:#9a9a95;--line:#2c2c2a;--accent:#7aa7ff;--accent2:#2a3b5c;--ok:#4cc38a;--warn:#e0a45a;--bad:#ff7a6b}}
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.45 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif}
main{max-width:960px;margin:0 auto;padding:24px 16px 48px}
h1{font-size:22px;margin:0 0 4px}.sub{color:var(--muted);margin:0 0 20px}
.phase{display:inline-block;padding:2px 10px;border-radius:999px;background:var(--accent2);color:var(--ink);font-size:13px;margin-left:8px;vertical-align:middle}
.bar{height:14px;background:var(--line);border-radius:7px;overflow:hidden;margin:8px 0 4px}.bar>div{height:100%;background:var(--accent);transition:width .6s}
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:12px;margin:18px 0}
.tile{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
.tile .k{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.04em}.tile .v{font-size:22px;font-weight:600;margin-top:2px;font-variant-numeric:tabular-nums}
.tile .s{color:var(--muted);font-size:12px}
.card{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:14px;margin-top:12px}
.card h2{font-size:15px;margin:0 0 8px}svg{width:100%;height:auto;display:block}
.legend{display:flex;gap:16px;color:var(--muted);font-size:12px;margin-top:6px}.sw{display:inline-block;width:14px;height:3px;vertical-align:middle;margin-right:6px}
.final{border-color:var(--ok)}.final .v{color:var(--ok)}.foot{color:var(--muted);font-size:12px;margin-top:18px}
.stale{color:var(--warn)}
</style></head><body><main>
<h1 id="title">Pod run</h1><p class="sub" id="what"></p>
<div><strong id="pct">–</strong> <span id="phase" class="phase">loading</span></div>
<div class="bar"><div id="fill" style="width:0"></div></div><div class="sub" id="eta"></div>
<div class="grid" id="tiles"></div>
<div class="card" id="finalcard" hidden><h2>Result (our scoring, fold validation)</h2><div class="grid" id="final" style="margin:0"></div></div>
<div class="card"><h2>Validation pseudo Dice per epoch</h2><svg id="chart" viewBox="0 0 900 260" role="img" aria-label="Pseudo Dice per epoch"></svg>
<div class="legend"><span><i class="sw" style="background:var(--accent2)"></i>pseudo Dice (each epoch)</span><span><i class="sw" style="background:var(--accent)"></i>EMA (what nnU-Net uses to pick the best checkpoint)</span></div>
<p class="sub" style="margin:8px 0 0;font-size:12px">Pseudo Dice is nnU-Net's quick estimate on random validation patches, not the final score. The final Dice comes from our own scoring after training.</p></div>
<p class="foot" id="foot"></p>
</main><script>
const $=id=>document.getElementById(id);
const fmtT=s=>s?new Date(s).toLocaleTimeString([], {hour:'2-digit',minute:'2-digit'})+' local ('+new Date(s).toISOString().slice(11,16)+' UTC)':'–';
const usd=x=>x==null?'–':'$'+x.toFixed(2);
function tile(k,v,s){return `<div class="tile"><div class="k">${k}</div><div class="v">${v}</div><div class="s">${s||''}</div></div>`}
function chart(ep){const W=900,H=260,P=36,svg=$('chart');if(!ep.length){svg.innerHTML='';return}
const n=Math.max(ep.length, 2), xs=i=>P+(W-P-10)*i/(n-1), vals=ep.map(e=>e.dice).filter(v=>v!=null), hi=Math.max(0.8,...vals), ys=v=>H-P-(H-P-10)*v/hi;
let g='';for(let t=0;t<=hi+1e-9;t+=0.2){g+=`<line x1="${P}" x2="${W-10}" y1="${ys(t)}" y2="${ys(t)}" stroke="var(--line)"/><text x="${P-6}" y="${ys(t)+4}" font-size="11" text-anchor="end" fill="var(--muted)">${t.toFixed(1)}</text>`}
const line=(k,c,w)=>`<polyline fill="none" stroke="${c}" stroke-width="${w}" points="${ep.map((e,i)=>e[k]==null?'':xs(i)+','+ys(e[k])).join(' ')}"/>`;
g+=line('dice','var(--accent2)',1.5)+line('ema','var(--accent)',2.5);
g+=`<text x="${W-10}" y="${H-10}" font-size="11" text-anchor="end" fill="var(--muted)">epoch ${ep[ep.length-1].epoch}</text><text x="${P}" y="${H-10}" font-size="11" fill="var(--muted)">epoch 0</text>`;
svg.innerHTML=g}
async function tick(){let d;try{d=await (await fetch('data.json?'+Date.now())).json()}catch(e){$('foot').textContent='Waiting for data.json…';return}
document.title=`${d.exp} · ${d.epochs_done}/${d.epochs_total||'?'}`;$('title').textContent=d.exp+' monitor';$('what').textContent=d.what+' · instance '+d.instance;
const pct=d.epochs_total?Math.min(100,100*d.epochs_done/d.epochs_total):0;$('pct').textContent=`Epoch ${d.epochs_done} of ${d.epochs_total} (${pct.toFixed(0)}%)`;$('fill').style.width=pct+'%';$('phase').textContent=d.phase;
$('eta').textContent=d.eta_utc&&!d.final?'Expected to finish around '+fmtT(d.eta_utc):'';
const L=d.last||{};$('tiles').innerHTML=tile('Per epoch',d.seconds_per_epoch?d.seconds_per_epoch.toFixed(1)+' s':'–','median of the last 20')+tile('Latest pseudo Dice',L.dice!=null?L.dice.toFixed(3):'–','EMA '+(L.ema!=null?L.ema.toFixed(3):'–')+' · best EMA '+(d.best_ema?d.best_ema.toFixed(3):'–'))
+tile('Cost so far',usd(d.cost_so_far),`${d.hours_up} h at $${d.price_per_hour.toFixed(3)}/h`)+tile('Projected total',usd(d.cost_projected),'if the pace holds')+tile('Watchdog',d.watchdog_deadline_utc?new Date(d.watchdog_deadline_utc).toISOString().slice(11,16)+' UTC':'–',`destroys the pod at the ${d.hour_limit} h limit`);
chart(d.epochs||[]);
if(d.final&&d.final.dice){$('finalcard').hidden=false;const f=d.final;$('final').innerHTML=tile('Dice',f.dice.mean.toFixed(3),'± '+(f.dice.std??0).toFixed(3)+' over '+f.dice.n+' cases')+(f.dice_reliable?tile('Dice, reliable Tmax',f.dice_reliable.mean.toFixed(3),''):'')+(f.abs_error_ml?tile('Abs volume error',f.abs_error_ml.mean.toFixed(1)+' mL',''):'');$('finalcard').classList.add('final')}
const age=(Date.now()-new Date(d.updated_utc))/1000;$('foot').innerHTML=`Updated ${Math.round(age)} s ago`+(age>300?' <span class="stale">(stale: is the monitor running and the VPN on?)</span>':'')+' · refreshes every 30 s';}
tick();setInterval(tick,30000);
</script></body></html>
"""
|