wan1.3b-bora-quant / code /nbdrive.py
Ami1ne's picture
executed notebook + run log
7d8e631 verified
Raw History Blame Contribute Delete
3.25 kB
# nbdrive.py -- run notebook cells one at a time in a long-lived Jupyter kernel and keep an executed copy of the notebook.
#
# python -m ipykernel_launcher -f kernel.json (in its own tmux session; holds all state between runs)
# python nbdrive.py kernel.json wan_quant.ipynb executed.ipynb 1-9 run cells "1)" .. "9)"
# python nbdrive.py kernel.json wan_quant.ipynb executed.ipynb 'code:FT_VARIANT = ("hqq", 3, 128)'
#
# Text output streams to stdout (tee it to a log). Prints CELL_DONE n / CELL_FAILED n; exit code 1 on failure.
import re
import sys
import time
from pathlib import Path
import nbformat
from jupyter_client import BlockingKernelClient
conn, nb_in, nb_out, what = sys.argv[1:5]
kc = BlockingKernelClient()
kc.load_connection_file(conn)
kc.start_channels()
kc.wait_for_ready(timeout=120)
nb = nbformat.read(nb_out if Path(nb_out).exists() else nb_in, as_version=4)
by_num = {}
for c in nb.cells:
if c.cell_type == "code":
m = re.match(r"#@title (\d+)\)", c.source)
if m:
by_num[int(m.group(1))] = c
MAX_HTML = 1_000_000 # embedded videos: keep the file small, they are saved as mp4 anyway
def run(source, cell=None, label="code"):
msg_id = kc.execute(source)
outputs, failed, t0 = [], False, time.time()
while True:
msg = kc.get_iopub_msg(timeout=None)
if msg["parent_header"].get("msg_id") != msg_id:
continue
t, c = msg["msg_type"], msg["content"]
if t == "stream":
print(c["text"], end="", flush=True)
if outputs and outputs[-1].get("output_type") == "stream" and outputs[-1]["name"] == c["name"]:
outputs[-1]["text"] += c["text"]
else:
outputs.append(nbformat.v4.output_from_msg(msg))
elif t in ("display_data", "execute_result"):
out = nbformat.v4.output_from_msg(msg)
html = out.get("data", {}).get("text/html", "")
if len(html) > MAX_HTML:
out["data"] = {"text/plain": f"[video omitted from the notebook file ({len(html) // 1024} KB), see the mp4 in the videos folder]"}
outputs.append(out)
txt = out.get("data", {}).get("text/plain", "")
print(("[display]\n" + txt[:4000]) if txt else "[display]", flush=True)
elif t == "error":
outputs.append(nbformat.v4.output_from_msg(msg))
print("\n".join(re.sub(r"\x1b\[[0-9;]*m", "", l) for l in c["traceback"]), flush=True)
failed = True
elif t == "status" and c["execution_state"] == "idle":
break
if cell is not None:
cell.outputs = outputs
nbformat.write(nb, nb_out)
print(f"\n{'CELL_FAILED' if failed else 'CELL_DONE'} {label} ({time.time() - t0:.0f}s)", flush=True)
return not failed
if what.startswith("code:"):
ok = run(what[5:], label="snippet")
else:
a, _, b = what.partition("-")
nums = range(int(a), int(b or a) + 1)
ok = True
for n in nums:
print(f"\n===== cell {n} =====", flush=True)
if not run(by_num[n].source, by_num[n], label=str(n)):
ok = False
break
kc.stop_channels()
sys.exit(0 if ok else 1)