File size: 2,223 Bytes
c34ff1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | from __future__ import annotations
import sys
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts" / "evaluate"))
from evaluate import ( # type: ignore[import-not-found]
save_eval_provenance,
)
class TestSaveEvalProvenance:
def test_creates_file(self, tmp_path: Path):
save_eval_provenance(tmp_path)
assert (tmp_path / "eval_command.txt").exists()
def test_contains_timestamp(self, tmp_path: Path):
save_eval_provenance(tmp_path)
content = (tmp_path / "eval_command.txt").read_text()
assert "# Timestamp:" in content
def test_contains_git_sha(self, tmp_path: Path):
save_eval_provenance(tmp_path)
content = (tmp_path / "eval_command.txt").read_text()
assert "# Git SHA:" in content
def test_contains_package_versions(self, tmp_path: Path):
save_eval_provenance(tmp_path)
content = (tmp_path / "eval_command.txt").read_text()
pkgs = ("speculators", "vllm", "transformers", "torch", "compressed-tensors")
for pkg in pkgs:
assert f"# {pkg}:" in content
def test_contains_sys_argv(self, tmp_path: Path):
argv = ["evaluate.py", "--target", "http://localhost:8000/v1"]
with patch.object(sys, "argv", argv):
save_eval_provenance(tmp_path)
content = (tmp_path / "eval_command.txt").read_text()
assert "evaluate.py --target" in content
def test_no_leftover_tmp_files(self, tmp_path: Path):
save_eval_provenance(tmp_path)
tmp_files = [
f for f in tmp_path.iterdir() if f.name.startswith(".eval_command")
]
assert tmp_files == []
def test_best_effort_never_raises(self, tmp_path: Path):
with patch("evaluate.atomic_write", side_effect=OSError("disk full")):
save_eval_provenance(tmp_path)
def test_overwrites_existing(self, tmp_path: Path):
(tmp_path / "eval_command.txt").write_text("old content")
save_eval_provenance(tmp_path)
content = (tmp_path / "eval_command.txt").read_text()
assert "old content" not in content
assert "# Timestamp:" in content
|