File size: 7,003 Bytes
a964978
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
import sys
from pathlib import Path

import pytest

# Ensure eval and root directory are in path
EVAL_DIR = Path(__file__).resolve().parent.parent.parent / "eval"
ROOT_DIR = Path(__file__).resolve().parent.parent.parent
if str(EVAL_DIR) not in sys.path:
    sys.path.insert(0, str(EVAL_DIR))
if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from scorer import (  # noqa: E402
    AggregateSummary,
    compute_latencies,
    compute_token_f1,
    extract_numbers_and_codes,
    format_markdown_table,
    is_chunk_match,
    score_answer_correctness,
    score_retrieval,
)


def test_chunk_matching():
    # Direct match
    assert is_chunk_match("doc_chunk_0001", None, ["doc_chunk_0001"], []) is True
    # Suffix match on session prefix
    assert is_chunk_match("sess_abc_chunk_0002", None, ["doc_chunk_0002"], []) is True
    # Index match
    assert is_chunk_match("sess_abc_chunk_0003", 2, [], [2]) is True
    # Parsed suffix index match (chunk_0003 -> index 2)
    assert is_chunk_match("sess_abc_chunk_0003", None, [], [2]) is True
    # Mismatch
    assert is_chunk_match("sess_abc_chunk_0001", None, ["doc_chunk_0003"], [2]) is False


def test_retrieval_scoring():
    # All hits
    hit_any, hit_all, recall = score_retrieval(
        retrieved_chunk_ids=["sess_x_chunk_0002", "sess_x_chunk_0003"],
        expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
        expected_chunk_indices=[1, 2],
    )
    assert hit_any is True
    assert hit_all is True
    assert recall == 1.0

    # Partial hit
    hit_any, hit_all, recall = score_retrieval(
        retrieved_chunk_ids=["sess_x_chunk_0002"],
        expected_chunk_ids=["doc_chunk_0002", "doc_chunk_0003"],
        expected_chunk_indices=[1, 2],
    )
    assert hit_any is True
    assert hit_all is False
    assert recall == 0.5

    # Total miss
    hit_any, hit_all, recall = score_retrieval(
        retrieved_chunk_ids=["sess_x_chunk_0001"],
        expected_chunk_ids=["doc_chunk_0002"],
        expected_chunk_indices=[1],
    )
    assert hit_any is False
    assert hit_all is False
    assert recall == 0.0

    # Absent question
    hit_any, hit_all, recall = score_retrieval(
        retrieved_chunk_ids=["sess_x_chunk_0001"],
        expected_chunk_ids=[],
        expected_chunk_indices=[],
    )
    assert hit_any is True
    assert hit_all is True
    assert recall == 1.0


def test_answer_correctness_and_refusal():
    # Absent question correctly refused
    is_corr, ref_corr, _, reason = score_answer_correctness(
        question_type="absent",
        actual_answer="Not covered in this document.",
        expected_answer="Not covered in this document.",
        refused=True,
    )
    assert is_corr is True
    assert ref_corr is True
    assert reason == "correct_refusal"

    # Absent question missed refusal
    is_corr, ref_corr, _, reason = score_answer_correctness(
        question_type="absent",
        actual_answer="Here is the pet policy for dogs.",
        expected_answer="Not covered in this document.",
        refused=False,
    )
    assert is_corr is False
    assert ref_corr is False
    assert reason == "missed_refusal"

    # Non-absent question false refusal
    is_corr, ref_corr, _, reason = score_answer_correctness(
        question_type="normal",
        actual_answer="I answer only from your uploaded document — ask me about it.",
        expected_answer="60 calendar days advance notice.",
        refused=True,
    )
    assert is_corr is False
    assert ref_corr is False
    assert reason == "false_refusal"

    # Non-absent question correct factual match
    is_corr, ref_corr, f1, reason = score_answer_correctness(
        question_type="normal",
        actual_answer="Permanent employees must tender formal notice at least 60 calendar days in advance [S1].",
        expected_answer="60 calendar days advance written notice to the designated reporting officer.",
        refused=False,
    )
    assert is_corr is True
    assert ref_corr is True
    assert f1 > 0.3
    assert reason == "factual_match"


def test_compute_latencies():
    lats = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
    p50, p95 = compute_latencies(lats)
    assert p50 == 60.0
    assert p95 == 100.0


def test_extract_numbers_and_codes():
    text = "Under Rule 63.7.4 and standard ISO-27001, procedure EC-101 requires 60 days and 24 days."
    codes = extract_numbers_and_codes(text)
    assert "63.7.4" in codes
    assert "iso-27001" in codes
    assert "ec-101" in codes
    assert "60" in codes
    assert "24" in codes


def test_token_f1():
    s1 = "60 calendar days advance notice"
    s2 = "notice of 60 calendar days in advance"
    f1 = compute_token_f1(s1, s2)
    assert f1 > 0.8


def test_summary_and_table_format():
    s = AggregateSummary("hybrid")
    s.record_question("normal", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=25.0)
    s.record_question("absent", retrieval_hit=True, answer_correct=True, refusal_correct=True, latency_ms=10.0)
    d = s.to_dict()
    assert d["retrieval_hit_rate"] == 100.0
    assert d["refusal_accuracy"] == 100.0
    assert d["answer_correctness"] == 100.0

    table = format_markdown_table([d])
    assert "| **hybrid** |" in table
    assert "| Configuration |" in table


@pytest.mark.anyio
async def test_run_eval_stub_integration():
    from eval.run import evaluate_configuration  # noqa: I001

    from app.sessions import SessionStore

    fixture_path = ROOT_DIR / "eval" / "fixtures" / "handbook.pdf"
    assert fixture_path.exists()

    with open(fixture_path, "rb") as f:
        doc_bytes = f.read()

    store = SessionStore()
    session = store.create_session_from_bytes("handbook.pdf", doc_bytes)

    sample_questions = [
        {
            "id": "q_01",
            "type": "normal",
            "question": "What is the mandatory advance notice period for voluntary resignation?",
            "expected_answer": "60 calendar days advance written notice to the designated reporting officer.",
            "expected_chunk_ids": ["doc_chunk_0002"],
            "expected_chunk_indices": [1],
        },
        {
            "id": "q_27",
            "type": "absent",
            "question": "What is the company policy on bringing pets or dogs to the office?",
            "expected_answer": "Not covered in this document.",
            "expected_chunk_ids": [],
            "expected_chunk_indices": [],
        },
    ]

    summary, q_logs, failures = await evaluate_configuration(
        session=session,
        questions=sample_questions,
        config_name="hybrid",
        provider_type="stub",
    )

    summary_dict = summary.to_dict()
    assert summary_dict["total_questions"] == 2
    assert summary_dict["retrieval_hit_rate"] == 100.0
    assert summary_dict["refusal_accuracy"] == 100.0
    assert summary_dict["answer_correctness"] == 100.0
    assert len(failures) == 0
    assert len(q_logs) == 2