eval/reference_tutor.py

"""FALLBACK ONLY: minimal SPEC §8.1/§8.3/§8.7 tutor pieces so the faithfulness harness runs end to end before
backend.app.tutor lands. eval.adapters prefers the production modules and labels any use of this file in
ENGINE_STATUS; live faithfulness runs refuse to use it (condition G must be the production pipeline).
"""

from __future__ import annotations

import json
import re
from pathlib import Path
from typing import Any

import yaml

from eval import checks, render
from eval.adapters import display, repo_for, zone_human
from eval.common import REPO_ROOT
from shared.contracts import (
    AttemptSubmit,
    Case,
    DebriefFacts,
    DebriefOutput,
    FactsCase,
    FactsFinding,
    FactsLearner,
    FactsMark,
    FactsPattern,
    FactsSearch,
    Outcome,
)

MISSED = {"missed_search", "missed_recognition", "missed_decision"}


def spec_system_prompt() -> str:
    """The v1 debrief system prompt quoted in docs/SPEC.md §8.3 (first fenced block after the heading)."""
    spec = (REPO_ROOT / "docs" / "SPEC.md").read_text()
    sec = spec.split("### 8.3", 1)[1]
    m = re.search(r"```\n(.*?)```", sec, re.S)
    return m.group(1).strip() if m else "You write the debrief for Blindspot."


def load_cards() -> dict[str, dict[str, Any]]:
    d = REPO_ROOT / "content" / "teaching_cards"
    out: dict[str, dict[str, Any]] = {}
    for p in sorted(d.glob("*.yaml")) if d.exists() else []:
        try:
            c = yaml.safe_load(p.read_text()) or {}
        except yaml.YAMLError:
            continue
        if isinstance(c, dict) and c.get("label"):
            out[c["label"]] = c
    return out


def _difficulty(v: float | None) -> str | None:
    if v is None:
        return None
    return "hard" if v > 0.3 else ("easy" if v < -0.3 else "moderate")


def _size(area_frac: float) -> str:
    word = "small" if area_frac < 0.005 else ("medium" if area_frac < 0.03 else "large")
    return f"{word} (about {100 * area_frac:.2g}% of the image)"


def build_facts(
    *,
    case: Case,
    submit: AttemptSubmit,
    outcomes: list[Outcome],
    spatial_relations: list[Any],
    search: FactsSearch,
    mark_zones: dict[str, str | None],
    level: str,
    history: dict[str, Any],
) -> DebriefFacts:
    findings = [
        FactsFinding(
            id=f.short_id,
            label=f.label,
            display=display(f.label),
            kind=f.kind,
            side=f.side,
            primary_zone=f.primary_zone,
            zones=list(f.zones),
            relative_location=f.relative_location,
            size=_size(f.area_frac) if f.kind == "focal" else None,
            difficulty=_difficulty(f.difficulty),
            zones_approximate=case.zones_approximate,
            ctr=case.cardiothoracic_ratio if f.label == "cardiomegaly" else None,
        )
        for f in case.findings
    ]
    t_end = submit.telemetry[-1].t / 1000.0 if submit.telemetry else 0.0
    learner = FactsLearner(
        level=level,
        declared_normal=submit.declared_normal,
        normal_confidence=submit.normal_confidence,
        hints_used=submit.hints_used,
        time_to_submit_s=round(t_end, 1),
        marks=[
            FactsMark(id=m.mark_id, label=m.label, confidence=m.confidence, zone=mark_zones.get(m.mark_id))
            for m in submit.marks
        ],
        pattern_selections=[FactsPattern(label=p.label, confidence=p.confidence) for p in submit.patterns],
    )
    labels = {f.label for f in case.findings} | {m.label for m in submit.marks if m.label != "not_sure"}
    return DebriefFacts(
        case=FactsCase(
            case_id=case.case_id,
            is_normal=case.is_normal,
            projection="frontal; PA vs AP not recorded",
            pixel_spacing_mm=case.pixel_spacing_mm,
            findings=findings,
        ),
        learner=learner,
        outcomes=list(outcomes),
        spatial_relations=list(spatial_relations),
        search=search,
        history=history,
        teaching_cards=sorted(labels),
    )


def verdict_for(facts: DebriefFacts) -> str:
    res = {o.target: o.result for o in facts.outcomes}
    fps = [o for o in facts.outcomes if o.result == "false_positive"]
    if facts.case.is_normal:
        return "correct_normal" if not fps and not facts.learner.marks else "overcall"
    if facts.learner.declared_normal:
        return "missed_normal_call"
    fres = [res.get(f.id) for f in facts.case.findings]
    good = sum(r in ("found", "pattern_found") for r in fres)
    if good == len(fres):
        return "all_found"
    if good == 0 and not any(r == "mislabeled" for r in fres):
        return "missed"
    return "partly_found"


WHY = {
    "found": "You found it and named it correctly.",
    "mislabeled": "You found the right spot but chose a different label.",
    "missed_search": "Your search never paused in this region.",
    "missed_recognition": "Your cursor passed over this region only briefly.",
    "missed_decision": "You looked here at length but judged it normal.",
    "pattern_found": "You reported this global finding.",
    "pattern_missed": "This global finding was not selected.",
}


def template_debrief(facts: DebriefFacts, cards: dict[str, Any] | None = None) -> DebriefOutput:
    cards = cards or {}
    res = {o.target: o for o in facts.outcomes}
    findings = []
    for f in facts.case.findings:
        o = res.get(f.id)
        result = o.result if o else ("pattern_missed" if f.kind == "pattern" else "missed_search")
        card = cards.get(f.label) or {}
        signs = list((card.get("key_signs") if isinstance(card, dict) else getattr(card, "key_signs", [])) or [])
        where = f.relative_location or zone_human(f.primary_zone) if f.kind == "focal" else "the whole chest"
        findings.append(
            {
                "finding_id": f.id,
                "result": result,
                "where_to_look": where,
                "what_it_looks_like": signs[:2] or [f"See the outlined {f.display.lower()}."],
                "why": WHY.get(result, ""),
            }
        )
    overcalls = [
        {
            "mark_id": o.target,
            "explanation": f"Radiologists marked nothing at your mark in the {zone_human(o.zone)}.",
            "possible_mimics": [],
        }
        for o in facts.outcomes
        if o.result == "false_positive"
    ]
    unv = [zone_human(z) for z in facts.search.unvisited_review_areas[:3]]
    out = {
        "headline": {
            "all_found": "Everything found.",
            "correct_normal": "Correct: this film is normal.",
            "overcall": "Nothing was there to mark.",
            "missed": "This one was missed.",
            "partly_found": "Partly found.",
            "missed_normal_call": "This film was not normal.",
        }[verdict_for(facts)],
        "verdict": verdict_for(facts),
        "findings": findings,
        "overcalls": overcalls,
        "search_coaching": ("Areas you did not visit: " + ", ".join(unv) + ".")
        if unv
        else "You visited every review area.",
        "calibration_note": "",
        "next_step": "Try another case.",
        "fact_ids": [f.id for f in facts.case.findings] + [o["mark_id"] for o in overcalls],
    }
    return DebriefOutput.model_validate(out)


def cards_text(cards: dict[str, Any]) -> str:
    if not cards:
        return "TEACHING CARDS: (none available)"
    return "TEACHING CARDS:\n" + json.dumps(
        {k: (v if isinstance(v, dict) else v.model_dump()) for k, v in sorted(cards.items())},
        sort_keys=True,
        ensure_ascii=False,
    )


def g_images(case: Case, data_root: Path, submit: AttemptSubmit, facts: DebriefFacts) -> list[tuple[dict, str]]:
    """SPEC §8.2: full image with outlines + marks; clean crop and outlined crop around the primary miss."""
    repo = repo_for(data_root)
    gray = render.load_gray(data_root, case)
    full = render.draw_marks(render.draw_outlines(render.to_rgb(gray), case, repo), list(submit.marks))
    res = {o.target: o.result for o in facts.outcomes}
    focal = [f for f in case.findings if f.kind == "focal"]
    target = next((f for f in focal if res.get(f.short_id) in MISSED | {"mislabeled"}), focal[0] if focal else None)
    blocks = [render.image_block(full)]
    if target is not None:
        size = max(64, case.width // 2)
        cx, cy = target.centroid
        blocks.append(render.image_block(render.crop(render.to_rgb(gray), cx, cy, size)))
        outl = render.draw_outlines(render.to_rgb(gray), case, repo, [target])
        blocks.append(render.image_block(render.crop(outl, cx, cy, size)))
    return blocks


def generate_debrief(
    facts: DebriefFacts,
    case: Case,
    submit: AttemptSubmit,
    *,
    client: Any,
    data_root: Path,
    model: str,
    cards: dict[str, Any],
) -> dict[str, Any]:
    schema = json.loads(checks.load_schema_path().read_text())
    schema = {k: v for k, v in schema.items() if not k.startswith("$") and k not in ("title", "description")}
    system = [
        {"type": "text", "text": spec_system_prompt()},
        {"type": "text", "text": cards_text(cards), "cache_control": {"type": "ephemeral"}},
    ]
    content: list[dict[str, Any]] = [b for b, _ in g_images(case, data_root, submit, facts)]
    content.append({"type": "text", "text": "FACTS:\n" + facts.model_dump_json(by_alias=True)})
    kwargs = {
        "model": model,
        "max_tokens": 1200,
        "system": system,
        "output_config": {"format": {"type": "json_schema", "schema": schema}},
    }
    first_v: dict[str, Any] | None = None
    for attempt in range(2):
        resp = client.messages.create(messages=[{"role": "user", "content": content}], **kwargs)
        text = next((b.text for b in resp.content if b.type == "text"), None)
        try:
            out = DebriefOutput.model_validate(json.loads(text or ""))
        except Exception:  # noqa: BLE001 — a malformed response counts as a validator failure
            out = None
        v = checks.validate(out.model_dump(), facts, cards) if out else {"ok": False, "errors": ["unparseable"]}
        first_v = first_v or v
        if out is not None and v["ok"]:
            return {
                "debrief": out,
                "source": "live",
                "validator": {"first": first_v, "final": v, "regenerated": attempt == 1},
            }
        content = content + [{"type": "text", "text": "Fix these problems: " + "; ".join(v["errors"][:8])}]
    tmpl = template_debrief(facts, cards)
    return {"debrief": tmpl, "source": "template", "validator": {"first": first_v, "final": None, "regenerated": True}}