File size: 1,358 Bytes
dfb775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """τ-Bench adapter — multi-turn agentic evaluation (Sierra)."""
from __future__ import annotations
import json
import shutil
import subprocess
from pathlib import Path
def _tau_bench_available() -> bool:
return shutil.which("tau-bench") is not None or shutil.which("tau_bench") is not None
def run_tau_bench(
checkpoint: Path,
*,
scenario: str = "airline",
out_dir: Path | None = None,
) -> dict[str, float]:
"""Run τ-Bench against `checkpoint`; return scenario scores."""
if not _tau_bench_available():
msg = "tau_bench not installed; install per https://github.com/sierra-research/tau-bench"
raise RuntimeError(msg)
out_dir = Path(out_dir or checkpoint / "tau_bench")
out_dir.mkdir(parents=True, exist_ok=True)
bin_name = shutil.which("tau-bench") or shutil.which("tau_bench") or "tau-bench"
cmd = [
bin_name,
"--task",
scenario,
"--model",
str(checkpoint),
"--output",
str(out_dir / "results.json"),
]
subprocess.run(cmd, check=True)
rp = out_dir / "results.json"
if not rp.exists():
return {}
raw = json.loads(rp.read_text())
out: dict[str, float] = {}
for k, v in raw.items():
if isinstance(v, int | float):
out[k] = float(v)
return out
__all__ = ["run_tau_bench"]
|