Spaces:
Sleeping
Sleeping
File size: 1,362 Bytes
5db30ac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | import asyncio
from elbench.judges.judge_teaching_harm import TeachingHarmJudge
from elbench.schemas.evaluation import ModelResponse, Sample
def _satas_sample(answer: str = "A,B,E") -> Sample:
return Sample(
sample_id="satas-test",
source_file="SATAs.xlsx",
source_path="SATAs.xlsx",
module="安全可信",
subset="教育-教学安全",
task="teaching_harm",
prompt="dummy",
reference={"Answer": answer},
)
def test_satas_exact_match_receives_full_credit() -> None:
result = asyncio.run(TeachingHarmJudge().judge(_satas_sample(), ModelResponse(text="E, A, B")))
assert result.judge_result == "pass"
assert result.score == 1.0
assert result.judge_metadata["partial_credit"] is False
def test_satas_missing_only_receives_partial_credit() -> None:
result = asyncio.run(TeachingHarmJudge().judge(_satas_sample(), ModelResponse(text="A, E")))
assert result.judge_result == "fail"
assert result.score == 0.5
assert result.judge_metadata["partial_credit"] is True
def test_satas_wrong_option_receives_zero_credit() -> None:
result = asyncio.run(TeachingHarmJudge().judge(_satas_sample(), ModelResponse(text="A, C")))
assert result.judge_result == "fail"
assert result.score == 0.0
assert result.judge_metadata["partial_credit"] is False
|