File size: 1,453 Bytes
dfb775d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """BFCL — Berkeley Function-Calling Leaderboard adapter.
Subprocess `bfcl evaluate`. Lazy availability check.
"""
from __future__ import annotations
import json
import shutil
import subprocess
from pathlib import Path
def _bfcl_available() -> bool:
return shutil.which("bfcl") is not None
def run_bfcl(
checkpoint: Path,
*,
version: str = "v4",
out_dir: Path | None = None,
categories: list[str] | None = None,
) -> dict[str, float]:
"""Run the BFCL evaluation suite; return per-category scores."""
if not _bfcl_available():
msg = "bfcl not installed; run `uv sync --extra eval`."
raise RuntimeError(msg)
out_dir = Path(out_dir or checkpoint / "bfcl")
out_dir.mkdir(parents=True, exist_ok=True)
cmd = [
"bfcl",
"evaluate",
"--model",
str(checkpoint),
"--test-category",
",".join(categories or ["simple", "parallel", "multiple", "multi_turn"]),
"--version",
version,
"--output-dir",
str(out_dir),
]
subprocess.run(cmd, check=True)
out: dict[str, float] = {}
for p in sorted(out_dir.glob("*.json")):
try:
raw = json.loads(p.read_text())
except (OSError, json.JSONDecodeError):
continue
for k, v in raw.items():
if isinstance(v, int | float):
out[f"{p.stem}/{k}"] = float(v)
return out
__all__ = ["run_bfcl"]
|