scripts/verify_case.py

#!/usr/bin/env python3
"""
Deterministic verifier for narrative.json files — checks code citations,
the Overview, the Model diagram's block/edge structure, and the results
tables. No LLM involved on purpose: this is the local, cheap, non-negotiable
check that catches fabricated citations, or a diagram that references nodes
that don't exist, before a case is called done.

Pure stdlib — no dependencies, no venv needed.

Usage:
    python3 scripts/verify_case.py cases/<slug>/narrative.json
    python3 scripts/verify_case.py cases/<slug>/narrative.json --repo cases/<slug>/repo

Exit code 0  -> no hard failures (warnings may still be printed)
Exit code 1  -> at least one hard failure
"""
import argparse
import ast
import base64
import json
import re
import sys
from pathlib import Path

STOPWORDS = {
    "the", "a", "an", "of", "for", "and", "or", "to", "in", "on", "is",
    "are", "with", "this", "that", "as", "by", "at", "we", "our", "it",
    "be", "into", "from", "which", "layer", "module", "function", "class",
}


def split_identifier(token: str):
    """camelCase / snake_case / PascalCase -> lowercase word parts."""
    token = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", token)
    parts = re.split(r"[_\-\s]+", token)
    return [p.lower() for p in parts if p]


def keywords_from(*texts):
    words = set()
    for text in texts:
        if not text:
            continue
        for raw in re.findall(r"[A-Za-z0-9]+", text):
            for w in split_identifier(raw):
                if len(w) > 2 and w not in STOPWORDS:
                    words.add(w)
    return words


class Result:
    def __init__(self):
        self.failures = []
        self.warnings = []

    def fail(self, node_id, msg):
        self.failures.append(f"[FAIL] {node_id}: {msg}")

    def warn(self, node_id, msg):
        self.warnings.append(f"[warn] {node_id}: {msg}")


def check_code_ref(result: Result, node_id: str, node: dict, ref: dict, repo_root: Path):
    file_rel = ref.get("file", "")
    start = ref.get("start_line")
    end = ref.get("end_line")

    if not file_rel:
        result.fail(node_id, "code_ref missing 'file'")
        return
    if not isinstance(start, int) or not isinstance(end, int) or start < 1 or end < start:
        result.fail(node_id, f"invalid line range {start}-{end} for {file_rel}")
        return

    target = repo_root / file_rel
    if not target.is_file():
        # Tolerate the common mistake of including the repo dir's own name
        # in `file` (e.g. "repo/foo.py" when repo_path is already "repo") --
        # `file` is meant to be repo_path-relative, but every other tool a
        # grounding session uses (read_file, bash) is case-dir-relative, so
        # this ambiguity is an easy slip, not necessarily a bad citation.
        fallback = repo_root.parent / file_rel
        if fallback.is_file():
            target = fallback
        else:
            result.fail(node_id, f"file not found on disk: {file_rel} (looked under {repo_root})")
            return

    lines = target.read_text(errors="replace").splitlines()
    if end > len(lines):
        result.fail(node_id, f"end_line {end} exceeds file length {len(lines)} in {file_rel}")
        return

    actual = "\n".join(lines[start - 1:end])
    claimed = (ref.get("snippet") or "").rstrip("\n")

    def normalize(s):
        return "\n".join(line.rstrip() for line in s.splitlines())

    if normalize(actual) != normalize(claimed):
        result.fail(
            node_id,
            f"embedded snippet does not match {file_rel}:{start}-{end} on disk "
            f"(citation has drifted or 'snippet' was not copied verbatim)",
        )
        return

    if target.suffix == ".py":
        try:
            tree = ast.parse(target.read_text(errors="replace"))
        except SyntaxError:
            tree = None
        if tree is not None:
            covers = False
            for n in ast.walk(tree):
                if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
                    n_end = getattr(n, "end_lineno", n.lineno)
                    if n.lineno <= end and n_end >= start:
                        covers = True
                        break
            if not covers:
                result.warn(
                    node_id,
                    f"{file_rel}:{start}-{end} doesn't overlap any def/class — double-check the span",
                )

    node_words = keywords_from(node.get("title"), ref.get("symbol"),
                                node.get("paper_ref", {}).get("quote"))
    code_words = keywords_from(actual, ref.get("symbol"))
    if node_words and not (node_words & code_words):
        result.warn(
            node_id,
            f"no shared terminology between node title/quote and {file_rel}:{start}-{end} "
            f"— confidence {ref.get('confidence')} may be optimistic",
        )


IMAGE_URI_RE = re.compile(r"^data:image/(png|jpe?g|webp);base64,([A-Za-z0-9+/=]+)$")


def check_figure_ref(result: Result, where: str, fig: dict):
    if not (fig.get("label") or "").strip():
        result.fail(where, "a figure entry is missing 'label'")
    if not (fig.get("caption") or "").strip():
        result.fail(where, f"figure '{fig.get('label', '?')}' is missing 'caption'")
    image = fig.get("image")
    if image is not None:
        m = IMAGE_URI_RE.match(image)
        if not m:
            result.fail(where, f"figure '{fig.get('label', '?')}' has an 'image' that isn't a well-formed data:image/...;base64,... URI")
        else:
            try:
                decoded = base64.b64decode(m.group(2), validate=True)
            except Exception:
                result.fail(where, f"figure '{fig.get('label', '?')}' has an 'image' whose base64 payload doesn't decode")
            else:
                if len(decoded) < 512:
                    result.warn(where, f"figure '{fig.get('label', '?')}' has a suspiciously tiny embedded image ({len(decoded)} bytes)")


def check_bullets(result: Result, where: str, field: str, value):
    if not isinstance(value, list) or not value:
        result.fail(where, f"{field} must be a non-empty list of bullet points")
        return
    for i, item in enumerate(value):
        if not isinstance(item, str) or not item.strip():
            result.fail(where, f"{field}[{i}] is empty")


def verify(data: dict, repo_root: Path) -> Result:
    result = Result()
    nodes = data.get("nodes", [])
    node_ids = [n.get("id") for n in nodes]

    dupes = {i for i in node_ids if node_ids.count(i) > 1}
    if dupes:
        result.fail("(file)", f"duplicate node ids: {sorted(dupes)}")
    node_id_set = set(node_ids)

    # --- case.venue ---
    if not (data.get("case", {}).get("venue") or "").strip():
        result.fail("(file)", "case.venue is missing — every case needs a conference/journal + year")

    # --- case.overview ---
    case_overview = data.get("case", {}).get("overview")
    if not case_overview:
        result.fail("(file)", "case.overview is missing — every case needs a task diagram + motivation + framework")
    else:
        task = case_overview.get("task")
        if not task:
            result.fail("(case.overview)", "missing 'task' — every case needs the Input -> Task -> Output schematic")
        else:
            for field in ("input", "task", "output"):
                if not (task.get(field) or "").strip():
                    result.fail("(case.overview.task)", f"'{field}' is empty")
        check_bullets(result, "(case.overview)", "motivation", case_overview.get("motivation"))
        check_bullets(result, "(case.overview)", "framework", case_overview.get("framework"))

        overview_figure = case_overview.get("figure")
        if overview_figure:
            check_figure_ref(result, "(case.overview.figure)", overview_figure)

        framework_text = " ".join(case_overview.get("framework") or [])
        for node in nodes:
            title = node.get("title") or ""
            if title and f"**{title}**" not in framework_text:
                result.warn(
                    "(case.overview)",
                    f"no framework bullet names block '{title}' in **bold** — a reader should see where "
                    f"each contribution sits before reaching the Model section",
                )

    # --- case.model_diagram ---
    diagram = data.get("case", {}).get("model_diagram")
    if not diagram:
        result.fail("(file)", "case.model_diagram is missing")
    else:
        blocks = diagram.get("blocks", [])
        block_ids = [b.get("id") for b in blocks]
        dupe_blocks = {i for i in block_ids if block_ids.count(i) > 1}
        if dupe_blocks:
            result.fail("(model_diagram)", f"duplicate block ids: {sorted(dupe_blocks)}")
        block_id_set = set(block_ids)

        referenced_node_ids = []
        for b in blocks:
            bid = b.get("id", "(missing id)")
            ref = b.get("node_ref")
            if not ref:
                result.fail(f"block:{bid}", "block missing 'node_ref'")
            elif ref not in node_id_set:
                result.fail(f"block:{bid}", f"node_ref '{ref}' does not match any id in the top-level 'nodes' array")
            else:
                referenced_node_ids.append(ref)

        unreferenced = node_id_set - set(referenced_node_ids)
        if unreferenced:
            result.fail("(model_diagram)", f"nodes not referenced by any block: {sorted(unreferenced)}")
        dupe_refs = {i for i in referenced_node_ids if referenced_node_ids.count(i) > 1}
        if dupe_refs:
            result.fail("(model_diagram)", f"nodes referenced by more than one block: {sorted(dupe_refs)}")

        for edge in diagram.get("edges", []):
            if len(edge) != 2:
                result.fail("(model_diagram)", f"edge {edge} must be [from_id, to_id]")
                continue
            a, b_ = edge
            if a not in block_id_set:
                result.fail("(model_diagram)", f"edge references unknown block id '{a}'")
            if b_ not in block_id_set:
                result.fail("(model_diagram)", f"edge references unknown block id '{b_}'")

        if len(blocks) > 6:
            result.warn(
                "(model_diagram)",
                f"{len(blocks)} blocks — most papers should land at 2-5 genuine contributions; "
                f"double-check each is really the paper's own claim, not a background component",
            )

    # --- case.results ---
    results = data.get("case", {}).get("results")
    if not results:
        result.fail("(file)", "case.results is missing or empty — every case needs at least one results table")
    else:
        for i, table in enumerate(results):
            where = f"(results[{i}] {table.get('table_label', '?')})"
            if not (table.get("table_label") or "").strip():
                result.fail(where, "missing 'table_label'")
            if not (table.get("caption") or "").strip():
                result.fail(where, "missing 'caption'")
            check_bullets(result, where, "takeaways", table.get("takeaways"))
            cols = table.get("columns") or []
            if not cols:
                result.fail(where, "'columns' is empty")
            rows = table.get("rows") or []
            if not rows:
                result.fail(where, "'rows' is empty")
            for r_i, row in enumerate(rows):
                if len(row) != len(cols):
                    result.fail(where, f"row {r_i} has {len(row)} cells but there are {len(cols)} columns")
            for hr in table.get("highlight_rows", []):
                if not (0 <= hr < len(rows)):
                    result.fail(where, f"highlight_rows index {hr} out of range (0-{len(rows) - 1})")
        if len(results) > 2:
            result.warn("(file)", f"{len(results)} results tables — keep it to the headline table(s), typically 1")

    # --- case.related_papers ---
    related = data.get("case", {}).get("related_papers")
    if not related:
        result.fail("(file)", "case.related_papers is missing or empty — every case needs the small set of papers it builds on/compares against")
    else:
        for i, rp in enumerate(related):
            where = f"(related_papers[{i}] {rp.get('title', '?')})"
            if not (rp.get("title") or "").strip():
                result.fail(where, "missing 'title'")
            if rp.get("role") not in ("foundational", "baseline"):
                result.fail(where, f"role must be 'foundational' or 'baseline', got {rp.get('role')!r}")
            if not (rp.get("note") or "").strip():
                result.fail(where, "missing 'note' — one short phrase on why this paper is relevant")
        if len(related) > 12:
            result.warn("(file)", f"{len(related)} related_papers — this should be a small, deliberate set, not the full bibliography")

    # --- nodes ---
    for node in nodes:
        node_id = node.get("id", "(missing id)")
        status = node.get("status")
        refs = node.get("code_refs", [])

        if not (node.get("section") or "").strip():
            result.fail(node_id, "missing/empty 'section' — every node must carry the paper's own subsection heading")

        why = (node.get("why") or "").strip()
        if not why:
            result.fail(node_id, "missing/empty 'why' — every node must say why this block belongs in the story")
        elif why.count(".") > 1 or len(why.split()) > 40:
            result.warn(node_id, "'why' reads like more than one sentence — keep it to a single sentence")

        check_bullets(result, node_id, "summary", node.get("summary"))

        for eq in node.get("equations", []):
            if not (eq.get("label") or "").strip():
                result.fail(node_id, "an equation entry is missing 'label'")
            if not (eq.get("expression") or "").strip():
                result.fail(node_id, f"equation '{eq.get('label', '?')}' is missing 'expression'")

        for fig in node.get("figure_refs", []):
            check_figure_ref(result, node_id, fig)

        if status == "resolved" and not refs:
            result.fail(node_id, "status is 'resolved' but code_refs is empty")
        if status == "unresolved" and refs:
            result.fail(node_id, "status is 'unresolved' but code_refs is non-empty")
        if status == "ambiguous" and len(refs) < 2:
            result.warn(node_id, "status is 'ambiguous' but fewer than 2 code_refs given")

        for ref in refs:
            check_code_ref(result, node_id, node, ref, repo_root)

    return result


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("narrative_json", type=Path)
    parser.add_argument("--repo", type=Path, default=None,
                         help="path to the cloned repo (default: <case_dir>/repo)")
    args = parser.parse_args()

    if not args.narrative_json.is_file():
        print(f"error: {args.narrative_json} not found", file=sys.stderr)
        sys.exit(1)

    data = json.loads(args.narrative_json.read_text())
    case_dir = args.narrative_json.parent
    repo_root = args.repo or (case_dir / (data.get("case", {}).get("repo_path") or "repo"))

    if not repo_root.is_dir():
        print(f"error: repo directory not found at {repo_root}", file=sys.stderr)
        sys.exit(1)

    result = verify(data, repo_root)

    n_nodes = len(data.get("nodes", []))
    n_resolved = sum(1 for n in data.get("nodes", []) if n.get("status") == "resolved")
    n_blocks = len((data.get("case", {}).get("model_diagram") or {}).get("blocks", []))
    print(f"{args.narrative_json}: {n_blocks} innovation blocks, "
          f"{n_nodes} nodes, {n_resolved} code-resolved\n")

    for w in result.warnings:
        print(w)
    if result.warnings:
        print()
    for f in result.failures:
        print(f)

    if result.failures:
        print(f"\n{len(result.failures)} hard failure(s). Fix these before calling the case done.")
        sys.exit(1)
    else:
        print(f"\nno hard failures ({len(result.warnings)} warning(s)).")
        sys.exit(0)


if __name__ == "__main__":
    main()