"""Live, loopback-only viewer for a small_run.py experiment; no extra GPU required. python dashboard.py --run-dir ../runs/proof-20261019/results --remote-dir /runs/proof-20261019-7b \ --container ta-... --log /tmp/kernel-proof-train.log --billing /tmp/kernel-proof-billing-progress.json """ import argparse import json import re import subprocess import threading import time from collections import defaultdict from html import escape from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from baseline_eval import summarize from grpo_kernel import episode_return from report import CSS, episode_html # Read artifacts only: never import or execute generated kernels in the viewer. READ = r''' import json,sys from pathlib import Path p=Path(sys.argv[1]) def read(name, lines=False): f=p/name if not f.exists(): return [] if lines else {} if not lines: return json.loads(f.read_text()) out=[] for line in f.read_text().splitlines(): try: out.append(json.loads(line)) except ValueError: pass return out states=sorted((p/'adapter').glob('checkpoint-*/trainer_state.json'),key=lambda x:int(x.parent.name.split('-')[-1])) print(json.dumps({'manifest':read('manifest.json'),'provenance':read('provenance.json'), 'before':read('before.jsonl',True),'train':read('adapter/rollouts-rank0.jsonl',True), 'after':read('after.jsonl',True),'comparison':read('comparison.json'), 'trainer':json.loads(states[-1].read_text()) if states else {}})) ''' def view(snapshot, log_text="", cost=None): manifest = snapshot.get("manifest", {}) events = [] for line in log_text.splitlines(): try: event = json.loads(line) if isinstance(event, dict) and event.get("event") == "step": events.append(event) except ValueError: pass goal = manifest.get("steps", 16) progress = max([int(n) for n in re.findall(r"(\d+)/" + str(goal) + r"\s*\[", log_text)] + [snapshot.get("trainer", {}).get("global_step", 0)]) train = snapshot.get("train", []) by_step = defaultdict(list) for ep in train: by_step[ep["step"] + 1].append(ep) metrics = {int(m["step"]): m for m in snapshot.get("trainer", {}).get("log_history", []) if "step" in m} steps = [] for step, eps in sorted(by_step.items()): s = summarize(eps) steps.append({"step": step, "episodes": len(eps), "correct": s["correct_pct"], "task": escape(", ".join(sorted({e["task_id"].split("/")[-1] for e in eps}))), "first": s["correct_turn1_pct"], "mean_return": sum( episode_return([t["reward"] for t in e["turns"]], "discounted", 0.4) for e in eps) / len(eps), "updated": step <= progress, "grad_norm": metrics.get(step, {}).get("grad_norm"), **{k: metrics.get(step, {}).get(k) for k in ("loss", "epoch", "learning_rate", "entropy")}}) before, after = snapshot.get("before", []), snapshot.get("after", []) phase = "Complete" if snapshot.get("comparison") else "Final evaluation" if after else ( "Training" if train else "Baseline") if not snapshot.get("comparison") and "PHASE after:" in log_text: phase = "Final evaluation" groups = {} for name, ids in manifest.get("groups", {}).items(): groups[name] = {label: summarize([e for e in eps if e["task_id"] in ids]) for label, eps in [("before", before), ("after", after)]} sections = {} for key, eps in [("before", before), ("train", train), ("after", after)]: parts = [] for i, ep in enumerate(eps): label = f'Step {ep["step"] + 1} · episode {i + 1}' if key == "train" else f'Episode {i + 1}' body = episode_html(ep).replace('
', f'
', 1) for j in range(body.count('
')): body = body.replace('
', f'
', 1) parts.append(f'

{escape(label)}

{body}
') sections[key] = "".join(parts) or '

No episodes yet. This updates automatically.

' return {"phase": phase, "updated": time.time(), "steps": steps, "completed": progress, "goal": goal, "kernel_evals": len(events) if events else sum(len(e["turns"]) for k in ("before", "train", "after") for e in snapshot.get(k, [])), "activity": (f'{events[-1]["task_id"]} · attempt {events[-1]["turn"]} · {events[-1]["stage"]}' if events else "Collecting the first submissions…"), "cost": cost, "before": summarize(before), "after": summarize(after), "sections": sections, "comparison": snapshot.get("comparison"), "model": manifest.get("model", "Loading…"), "groups": groups, "metrics": metrics, "warning": None} PAGE = r''' Kernel Lab · Live experiment
Kernel Lab / OpenEnv × GRPO

A small run. Every step visible.

Connecting to experiment…

● Connecting
–Training updates
–Kernel submissions
–Baseline correct
PendingAfter RL, correct
–Reported spend / $50 cap

Optimizer traces

Recorded metrics for each update. Hover over a point for its exact value and epoch.

Download all metrics ↓

Reward / discounted episode return

GRPO policy loss

More traces

X axis: optimizer step. Groups use different training tasks. A lower policy loss or higher training reward alone does not establish better held-out kernels.

How training is progressing

Each step samples four episodes of two attempts. Green = all four solved; amber = some or none. A filled cell means the optimizer update completed.

Bar height: correct episodes in that training group. This is training performance, not held-out progress.

The comparison that matters

Same tasks, model family, GPU and greedy decoding. Each task gets two attempts.

Task groupBaseAfter RL

Four training tasks, three reserved transfer tasks, three official held-out tasks. A ten-task pilot cannot establish broad generalization.

Training steps

StepEpochTaskStateCorrect episodesTurn 1 correctMean returnLossLearning rateGradient norm

Groups can contain different tasks, so bar heights are not a learning curve on a fixed task. Trainer traces sync from saved checkpoint history. Final evaluation measures improvement.

Inspect every attempt

Open an episode for code, feedback, runtime and a diff between turns.

''' def main(): p = argparse.ArgumentParser() p.add_argument("--run-dir", type=Path, required=True) p.add_argument("--remote-dir", required=True) p.add_argument("--container", default="") p.add_argument("--log", type=Path, required=True) p.add_argument("--billing", type=Path, required=True) p.add_argument("--app-id", action="append", default=[]) p.add_argument("--port", type=int, default=8099) args = p.parse_args() args.run_dir.mkdir(parents=True, exist_ok=True) cache = args.run_dir / "live-snapshot.json" snapshot = json.loads(cache.read_text()) if cache.exists() else {} state = view(snapshot) synced_at = cache.stat().st_mtime if cache.exists() else time.time() def collect(): nonlocal snapshot, state, synced_at while True: warning = None try: if (args.run_dir / "comparison.json").exists(): r = subprocess.run(["python3", "-c", READ, str(args.run_dir)], capture_output=True, text=True, check=True) else: r = subprocess.run(["modal", "container", "exec", args.container, "--", "python", "-c", READ, args.remote_dir], capture_output=True, text=True, check=True, timeout=25) snapshot = json.loads(r.stdout) cache.write_text(json.dumps(snapshot)) synced_at = time.time() except (OSError, ValueError, subprocess.SubprocessError): warning = "Cloud sync unavailable; showing the last snapshot. Final artifacts will replace it when downloaded." cost = None if args.billing.exists(): try: rows = json.loads(args.billing.read_text()) cost = sum(float(r["Cost"]) for r in rows if r["Object ID"] in args.app_id) except ValueError: cost = state.get("cost") state = view(snapshot, args.log.read_text() if args.log.exists() else "", cost) state["updated"] = synced_at state["warning"] = warning time.sleep(20) class Handler(BaseHTTPRequestHandler): def do_GET(self): if self.path not in ("/", "/state", "/metrics.json"): self.send_error(404) return body = (json.dumps(state["metrics"] if self.path == "/metrics.json" else state).encode() if self.path != "/" else PAGE.replace("__CSS__", CSS).encode()) self.send_response(200) self.send_header("Content-Type", "application/json" if self.path != "/" else "text/html; charset=utf-8") self.send_header("Cache-Control", "no-store") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *_): pass threading.Thread(target=collect, daemon=True).start() print(f"Kernel Lab: http://127.0.0.1:{args.port}", flush=True) ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() if __name__ == "__main__": main()