# nbdrive.py -- run notebook cells one at a time in a long-lived Jupyter kernel and keep an executed copy of the notebook. # # python -m ipykernel_launcher -f kernel.json (in its own tmux session; holds all state between runs) # python nbdrive.py kernel.json wan_quant.ipynb executed.ipynb 1-9 run cells "1)" .. "9)" # python nbdrive.py kernel.json wan_quant.ipynb executed.ipynb 'code:FT_VARIANT = ("hqq", 3, 128)' # # Text output streams to stdout (tee it to a log). Prints CELL_DONE n / CELL_FAILED n; exit code 1 on failure. import re import sys import time from pathlib import Path import nbformat from jupyter_client import BlockingKernelClient conn, nb_in, nb_out, what = sys.argv[1:5] kc = BlockingKernelClient() kc.load_connection_file(conn) kc.start_channels() kc.wait_for_ready(timeout=120) nb = nbformat.read(nb_out if Path(nb_out).exists() else nb_in, as_version=4) by_num = {} for c in nb.cells: if c.cell_type == "code": m = re.match(r"#@title (\d+)\)", c.source) if m: by_num[int(m.group(1))] = c MAX_HTML = 1_000_000 # embedded videos: keep the file small, they are saved as mp4 anyway def run(source, cell=None, label="code"): msg_id = kc.execute(source) outputs, failed, t0 = [], False, time.time() while True: msg = kc.get_iopub_msg(timeout=None) if msg["parent_header"].get("msg_id") != msg_id: continue t, c = msg["msg_type"], msg["content"] if t == "stream": print(c["text"], end="", flush=True) if outputs and outputs[-1].get("output_type") == "stream" and outputs[-1]["name"] == c["name"]: outputs[-1]["text"] += c["text"] else: outputs.append(nbformat.v4.output_from_msg(msg)) elif t in ("display_data", "execute_result"): out = nbformat.v4.output_from_msg(msg) html = out.get("data", {}).get("text/html", "") if len(html) > MAX_HTML: out["data"] = {"text/plain": f"[video omitted from the notebook file ({len(html) // 1024} KB), see the mp4 in the videos folder]"} outputs.append(out) txt = out.get("data", {}).get("text/plain", "") print(("[display]\n" + txt[:4000]) if txt else "[display]", flush=True) elif t == "error": outputs.append(nbformat.v4.output_from_msg(msg)) print("\n".join(re.sub(r"\x1b\[[0-9;]*m", "", l) for l in c["traceback"]), flush=True) failed = True elif t == "status" and c["execution_state"] == "idle": break if cell is not None: cell.outputs = outputs nbformat.write(nb, nb_out) print(f"\n{'CELL_FAILED' if failed else 'CELL_DONE'} {label} ({time.time() - t0:.0f}s)", flush=True) return not failed if what.startswith("code:"): ok = run(what[5:], label="snippet") else: a, _, b = what.partition("-") nums = range(int(a), int(b or a) + 1) ok = True for n in nums: print(f"\n===== cell {n} =====", flush=True) if not run(by_num[n].source, by_num[n], label=str(n)): ok = False break kc.stop_channels() sys.exit(0 if ok else 1)