allcited-api / tests /test_eval_runner.py
allcited-deploy
deploy 2026-10-25-1939
a964978
Raw History Blame Contribute Delete
7 kB
import sys
from pathlib import Path
import pytest
# Ensure eval and root directory are in path
EVAL_DIR = Path(__file__).resolve().parent.parent.parent / "eval"
ROOT_DIR = Path(__file__).resolve().parent.parent.parent
if str(EVAL_DIR) not in sys.path:
sys.path.insert(0, str(EVAL_DIR))
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
from scorer import ( # noqa: E402
AggregateSummary,
compute_latencies,
compute_token_f1,
extract_numbers_and_codes,
format_markdown_table,
is_chunk_match,
score_answer_correctness,
score_retrieval,
)
def test_chunk_matching():
# Direct match
assert is_chunk_match("doc_chunk_0001", None, ["doc_chunk_0001"], []) is True
# Suffix match on session prefix
assert is_chunk_match("sess_abc_chunk_0002", None, ["doc_chunk_0002"], []) is True
# Index match
assert is_chunk_match("sess_abc_chunk_0003", 2, [], [2]) is True
# Parsed suffix index match (chunk_0003 -> index 2)
assert is_chunk_match("sess_abc_chunk_0003", None, [], [2]) is True
# Mismatch
assert is_chunk_match("sess_abc_chunk_0001", None, ["doc_chunk_0003"], [2]) is False
def test_retrieval_scoring():
# All hits
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0002", "sess_x_chunk_0003"],
expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
expected_chunk_indices=[1, 2],
)
assert hit_any is True
assert hit_all is True
assert recall == 1.0
# Partial hit
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0002"],
expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
expected_chunk_indices=[1, 2],
)
assert hit_any is True
assert hit_all is False
assert recall == 0.5
# Total miss
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0001"],
expected_chunk_ids=["doc_chunk_0002"],
expected_chunk_indices=[1],
)
assert hit_any is False
assert hit_all is False
assert recall == 0.0
# Absent question
hit_any, hit_all, recall = score_retrieval(
retrieved_chunk_ids=["sess_x_chunk_0001"],
expected_chunk_ids=[],
expected_chunk_indices=[],
)
assert hit_any is True
assert hit_all is True
assert recall == 1.0
def test_answer_correctness_and_refusal():
# Absent question correctly refused
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="absent",
actual_answer="Not covered in this document.",
expected_answer="Not covered in this document.",
refused=True,
)
assert is_corr is True
assert ref_corr is True
assert reason == "correct_refusal"
# Absent question missed refusal
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="absent",
actual_answer="Here is the pet policy for dogs.",
expected_answer="Not covered in this document.",
refused=False,
)
assert is_corr is False
assert ref_corr is False
assert reason == "missed_refusal"
# Non-absent question false refusal
is_corr, ref_corr, _, reason = score_answer_correctness(
question_type="normal",
actual_answer="I answer only from your uploaded document — ask me about it.",
expected_answer="60 calendar days advance notice.",
refused=True,
)
assert is_corr is False
assert ref_corr is False
assert reason == "false_refusal"
# Non-absent question correct factual match
is_corr, ref_corr, f1, reason = score_answer_correctness(
question_type="normal",
actual_answer="Permanent employees must tender formal notice at least 60 calendar days in advance [S1].",
expected_answer="60 calendar days advance written notice to the designated reporting officer.",
refused=False,
)
assert is_corr is True
assert ref_corr is True
assert f1 > 0.3
assert reason == "factual_match"
def test_compute_latencies():
lats = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
p50, p95 = compute_latencies(lats)
assert p50 == 60.0
assert p95 == 100.0
def test_extract_numbers_and_codes():
text = "Under Rule 63.7.4 and standard ISO-27001, procedure EC-101 requires 60 days and 24 days."
codes = extract_numbers_and_codes(text)
assert "63.7.4" in codes
assert "iso-27001" in codes
assert "ec-101" in codes
assert "60" in codes
assert "24" in codes
def test_token_f1():
s1 = "60 calendar days advance notice"
s2 = "notice of 60 calendar days in advance"
f1 = compute_token_f1(s1, s2)
assert f1 > 0.8
def test_summary_and_table_format():
s = AggregateSummary("hybrid")
s.record_question("normal", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=25.0)
s.record_question("absent", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=10.0)
d = s.to_dict()
assert d["retrieval_hit_rate"] == 100.0
assert d["refusal_accuracy"] == 100.0
assert d["answer_correctness"] == 100.0
table = format_markdown_table([d])
assert "| **hybrid** |" in table
assert "| Configuration |" in table
@pytest.mark.anyio
async def test_run_eval_stub_integration():
from eval.run import evaluate_configuration # noqa: I001
from app.sessions import SessionStore
fixture_path = ROOT_DIR / "eval" / "fixtures" / "handbook.pdf"
assert fixture_path.exists()
with open(fixture_path, "rb") as f:
doc_bytes = f.read()
store = SessionStore()
session = store.create_session_from_bytes("handbook.pdf", doc_bytes)
sample_questions = [
{
"id": "q_01",
"type": "normal",
"question": "What is the mandatory advance notice period for voluntary resignation?",
"expected_answer": "60 calendar days advance written notice to the designated reporting officer.",
"expected_chunk_ids": ["doc_chunk_0002"],
"expected_chunk_indices": [1],
},
{
"id": "q_27",
"type": "absent",
"question": "What is the company policy on bringing pets or dogs to the office?",
"expected_answer": "Not covered in this document.",
"expected_chunk_ids": [],
"expected_chunk_indices": [],
},
]
summary, q_logs, failures = await evaluate_configuration(
session=session,
questions=sample_questions,
config_name="hybrid",
provider_type="stub",
)
summary_dict = summary.to_dict()
assert summary_dict["total_questions"] == 2
assert summary_dict["retrieval_hit_rate"] == 100.0
assert summary_dict["refusal_accuracy"] == 100.0
assert summary_dict["answer_correctness"] == 100.0
assert len(failures) == 0
assert len(q_logs) == 2