File size: 4,365 Bytes
8e36acf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | """Lightweight structural tests for ``notebooks/train_kinchat.ipynb``.
We cannot execute the training notebook in CI — it requires a GPU, network
access to the deployed env, and ~45 min of wall clock. Instead we assert
that the notebook is a *valid* nbformat v4 file, that it contains the
marker strings the hackathon judges look for (GRPO, Unsloth, Qwen2.5, the
env URL, the four rubric names, the three plot filenames), that it has
enough cells to hang the required flow off, and that every code cell
parses as Python.
Keep these cheap; we just need a tripwire if a future edit breaks
structure.
"""
from __future__ import annotations
import ast
from pathlib import Path
import nbformat
import pytest
NOTEBOOK_PATH = (
Path(__file__).resolve().parents[1] / "notebooks" / "train_kinchat.ipynb"
)
@pytest.fixture(scope="module")
def notebook():
assert NOTEBOOK_PATH.exists(), f"missing notebook: {NOTEBOOK_PATH}"
nb = nbformat.read(NOTEBOOK_PATH, as_version=4)
return nb
def test_notebook_is_valid_nbformat_v4(notebook):
assert notebook.nbformat == 4
# nbformat.validate raises ValidationError on anything malformed.
nbformat.validate(notebook)
def test_notebook_has_at_least_twelve_cells(notebook):
assert len(notebook.cells) >= 12, (
f"expected >=12 cells, got {len(notebook.cells)}"
)
def test_notebook_has_both_markdown_and_code_cells(notebook):
code_cells = [c for c in notebook.cells if c.cell_type == "code"]
md_cells = [c for c in notebook.cells if c.cell_type == "markdown"]
assert code_cells, "no code cells in notebook"
assert md_cells, "no markdown cells in notebook"
def _all_source(notebook) -> str:
"""Concatenate every cell source into one blob for marker checks."""
return "\n\n".join(c.source for c in notebook.cells)
@pytest.mark.parametrize(
"marker",
[
"GRPO",
"unsloth",
"Qwen2.5",
"reward_curve",
"per_rubric_curves",
"session_trust",
"vex-0-kinchat.hf.space",
# four rubric names:
"leak",
"audience_fit",
"restraint",
"trust_delta",
],
)
def test_notebook_contains_marker(notebook, marker):
blob = _all_source(notebook)
assert marker in blob, f"marker {marker!r} missing from notebook source"
def test_notebook_references_docs_plots_dir(notebook):
blob = _all_source(notebook)
# The plot cell must write into docs/plots/ relative to repo root.
assert "docs/plots" in blob, "plot cell should target docs/plots/"
def test_notebook_mentions_huggingface_hub_push(notebook):
blob = _all_source(notebook)
# A Hub upload is required by the hackathon checklist.
assert "huggingface_hub" in blob or "HfApi" in blob, (
"notebook should push the LoRA to the HF Hub"
)
def test_every_code_cell_parses_as_python(notebook):
"""Every code cell must be syntactically valid Python, ignoring
notebook-specific magics (``%pip``, ``!cmd``, ``?ident``).
"""
for i, cell in enumerate(notebook.cells):
if cell.cell_type != "code":
continue
src = cell.source
lines = src.splitlines()
# Strip magics/shell/inspect lines; keep a raw blank so line numbers
# in any future SyntaxError match the original cell.
cleaned = []
for line in lines:
stripped = line.lstrip()
if (
stripped.startswith("%")
or stripped.startswith("!")
or stripped.startswith("?")
):
cleaned.append("")
else:
cleaned.append(line)
cleaned_src = "\n".join(cleaned)
if not cleaned_src.strip():
continue
try:
ast.parse(cleaned_src)
except SyntaxError as exc: # pragma: no cover — test failure path
pytest.fail(f"cell {i} failed to parse: {exc}\n---\n{src}\n---")
def test_notebook_uses_grpo_trainer_or_documents_fallback(notebook):
blob = _all_source(notebook)
assert "GRPOTrainer" in blob, "notebook must reference GRPOTrainer (TRL)"
# The fallback path must also be documented so graders see both rails.
assert "manual GRPO" in blob.lower() or "fallback" in blob.lower(), (
"notebook should document a fallback path if GRPOTrainer fails"
)
|