Spaces:
Running on Zero
Running on Zero
File size: 7,003 Bytes
a964978 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 | import sys
from pathlib import Path
import pytest
# Ensure eval and root directory are in path
EVAL_DIR = Path(__file__).resolve().parent.parent.parent / "eval"
ROOT_DIR = Path(__file__).resolve().parent.parent.parent
if str(EVAL_DIR) not in sys.path:
sys.path.insert(0, str(EVAL_DIR))
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
from scorer import ( # noqa: E402
AggregateSummary,
compute_latencies,
compute_token_f1,
extract_numbers_and_codes,
format_markdown_table,
is_chunk_match,
score_answer_correctness,
score_retrieval,
)
def test_chunk_matching():
# Direct match
assert is_chunk_match("doc_chunk_0001", None, ["doc_chunk_0001"], []) is True
# Suffix match on session prefix
assert is_chunk_match("sess_abc_chunk_0002", None, ["doc_chunk_0002"], []) is True
# Index match
assert is_chunk_match("sess_abc_chunk_0003", 2, [], [2]) is True
# Parsed suffix index match (chunk_0003 -> index 2)
assert is_chunk_match("sess_abc_chunk_0003", None, [], [2]) is True
# Mismatch
assert is_chunk_match("sess_abc_chunk_0001", None, ["doc_chunk_0003"], [2]) is False
def test_retrieval_scoring():
# All hits
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0002", "sess_x_chunk_0003"],
expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
expected_chunk_indices=[1, 2],
)
assert hit_any is True
assert hit_all is True
assert recall == 1.0
# Partial hit
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0002"],
expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
expected_chunk_indices=[1, 2],
)
assert hit_any is True
assert hit_all is False
assert recall == 0.5
# Total miss
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0001"],
expected_chunk_ids=["doc_chunk_0002"],
expected_chunk_indices=[1],
)
assert hit_any is False
assert hit_all is False
assert recall == 0.0
# Absent question
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0001"],
expected_chunk_ids=[],
expected_chunk_indices=[],
)
assert hit_any is True
assert hit_all is True
assert recall == 1.0
def test_answer_correctness_and_refusal():
# Absent question correctly refused
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="absent",
actual_answer="Not covered in this document.",
expected_answer="Not covered in this document.",
refused=True,
)
assert is_corr is True
assert ref_corr is True
assert reason == "correct_refusal"
# Absent question missed refusal
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="absent",
actual_answer="Here is the pet policy for dogs.",
expected_answer="Not covered in this document.",
refused=False,
)
assert is_corr is False
assert ref_corr is False
assert reason == "missed_refusal"
# Non-absent question false refusal
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="normal",
actual_answer="I answer only from your uploaded document — ask me about it.",
expected_answer="60 calendar days advance notice.",
refused=True,
)
assert is_corr is False
assert ref_corr is False
assert reason == "false_refusal"
# Non-absent question correct factual match
is_corr, ref_corr, f1, reason = score_answer_correctness(
question_type="normal",
actual_answer="Permanent employees must tender formal notice at least 60 calendar days in advance [S1].",
expected_answer="60 calendar days advance written notice to the designated reporting officer.",
refused=False,
)
assert is_corr is True
assert ref_corr is True
assert f1 > 0.3
assert reason == "factual_match"
def test_compute_latencies():
lats = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
p50, p95 = compute_latencies(lats)
assert p50 == 60.0
assert p95 == 100.0
def test_extract_numbers_and_codes():
text = "Under Rule 63.7.4 and standard ISO-27001, procedure EC-101 requires 60 days and 24 days."
codes = extract_numbers_and_codes(text)
assert "63.7.4" in codes
assert "iso-27001" in codes
assert "ec-101" in codes
assert "60" in codes
assert "24" in codes
def test_token_f1():
s1 = "60 calendar days advance notice"
s2 = "notice of 60 calendar days in advance"
f1 = compute_token_f1(s1, s2)
assert f1 > 0.8
def test_summary_and_table_format():
s = AggregateSummary("hybrid")
s.record_question("normal", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=25.0)
s.record_question("absent", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=10.0)
d = s.to_dict()
assert d["retrieval_hit_rate"] == 100.0
assert d["refusal_accuracy"] == 100.0
assert d["answer_correctness"] == 100.0
table = format_markdown_table([d])
assert "| **hybrid** |" in table
assert "| Configuration |" in table
@pytest.mark.anyio
async def test_run_eval_stub_integration():
from eval.run import evaluate_configuration # noqa: I001
from app.sessions import SessionStore
fixture_path = ROOT_DIR / "eval" / "fixtures" / "handbook.pdf"
assert fixture_path.exists()
with open(fixture_path, "rb") as f:
doc_bytes = f.read()
store = SessionStore()
session = store.create_session_from_bytes("handbook.pdf", doc_bytes)
sample_questions = [
{
"id": "q_01",
"type": "normal",
"question": "What is the mandatory advance notice period for voluntary resignation?",
"expected_answer": "60 calendar days advance written notice to the designated reporting officer.",
"expected_chunk_ids": ["doc_chunk_0002"],
"expected_chunk_indices": [1],
},
{
"id": "q_27",
"type": "absent",
"question": "What is the company policy on bringing pets or dogs to the office?",
"expected_answer": "Not covered in this document.",
"expected_chunk_ids": [],
"expected_chunk_indices": [],
},
]
summary, q_logs, failures = await evaluate_configuration(
session=session,
questions=sample_questions,
config_name="hybrid",
provider_type="stub",
)
summary_dict = summary.to_dict()
assert summary_dict["total_questions"] == 2
assert summary_dict["retrieval_hit_rate"] == 100.0
assert summary_dict["refusal_accuracy"] == 100.0
assert summary_dict["answer_correctness"] == 100.0
assert len(failures) == 0
assert len(q_logs) == 2
|