File size: 1,418 Bytes
dfb775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | """Inspect-AI adapter — agentic evaluations.
Subprocess `inspect eval`; emits per-task JSON under `out_dir`.
"""
from __future__ import annotations
import json
import shutil
import subprocess
from pathlib import Path
def _inspect_available() -> bool:
return shutil.which("inspect") is not None
def run_inspect(
checkpoint: Path,
tasks: list[str],
*,
out_dir: Path | None = None,
) -> dict[str, float]:
"""Run an inspect-ai task suite; return summary metrics."""
if not _inspect_available():
msg = "inspect-ai not installed; run `uv sync --extra eval`."
raise RuntimeError(msg)
out_dir = Path(out_dir or checkpoint / "inspect")
out_dir.mkdir(parents=True, exist_ok=True)
cmd = [
"inspect",
"eval",
*tasks,
"--model",
f"hf/{checkpoint}",
"--log-dir",
str(out_dir),
]
subprocess.run(cmd, check=True)
out: dict[str, float] = {}
for p in sorted(out_dir.glob("**/*.json")):
try:
raw = json.loads(p.read_text())
except (OSError, json.JSONDecodeError):
continue
scores = raw.get("results", {}).get("scores", {})
for k, v in scores.items():
if isinstance(v, dict) and "value" in v and isinstance(v["value"], int | float):
out[f"{p.stem}/{k}"] = float(v["value"])
return out
__all__ = ["run_inspect"]
|