File size: 8,741 Bytes
d3d09d6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
"""Claim-Level Answer Verification Module.

Performs a second-pass verification check ensuring every claim in the generated answer
is strictly supported by the retrieved document chunks. Flags or removes unsupported claims.
"""

import re
from dataclasses import dataclass, field, asdict
from typing import List, Dict, Any, Optional

from ingest import DocumentChunk


@dataclass
class ClaimVerification:
    claim: str
    status: str  # "SUPPORTED", "UNSUPPORTED", "CONTRADICTED"
    confidence: float
    supporting_chunk_id: Optional[str] = None
    supporting_quote: Optional[str] = None


@dataclass
class VerificationResult:
    original_answer: str
    verified_answer: str
    total_claims: int
    supported_claims: List[str]
    unsupported_claims: List[str]
    faithfulness_score: float
    claim_details: List[ClaimVerification] = field(default_factory=list)
    has_hallucination: bool = False

    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)


class AnswerVerifier:
    """
    Second-pass verifier for Document Question Answering.
    1. Extracts atomic claims/propositions from the answer.
    2. Verifies each claim against the retrieved context chunks using semantic + lexical entailment.
    3. Removes or flags unsupported claims and computes answer faithfulness.
    """

    def __init__(
        self,
        cross_encoder=None,
        threshold: float = 0.55,
        flag_unsupported: bool = True,
        llm_client=None
    ):
        self.cross_encoder = cross_encoder
        self.threshold = threshold
        self.flag_unsupported = flag_unsupported
        self.llm_client = llm_client

    def extract_claims(self, text: str) -> List[str]:
        """Decomposes an answer into discrete factual propositions/sentences."""
        text = text.strip()
        if not text or "not found in the document" in text.lower():
            return []

        # Remove citation brackets like [Page 2, Section "Intro"] for claim extraction
        clean_text = re.sub(r"\[Page\s+\d+[^\]]*\]", "", text, flags=re.IGNORECASE)
        
        # Split on sentence boundaries and bullet points
        raw_sentences = re.split(r"(?<=[.!?])\s+|\n+[\s*-]*", clean_text)
        claims = []
        
        filler_patterns = [
            r"^(in summary|in conclusion|to conclude|based on the document|according to the document)\b",
            r"^(as mentioned above|therefore|hence|overall)\b",
        ]

        for s in raw_sentences:
            s_clean = s.strip()
            # Strip markdown bullets
            s_clean = re.sub(r"^[\d+.\-*#]+\s*", "", s_clean).strip()
            if len(s_clean.split()) < 3:
                continue

            # Skip pure filler
            if any(re.match(pat, s_clean, re.IGNORECASE) for pat in filler_patterns) and len(s_clean.split()) < 6:
                continue

            claims.append(s_clean)

        return claims

    def verify_claim(self, claim: str, retrieved_chunks: List[DocumentChunk]) -> ClaimVerification:
        """Verifies an individual claim against all retrieved chunks."""
        if not retrieved_chunks:
            return ClaimVerification(
                claim=claim,
                status="UNSUPPORTED",
                confidence=0.0
            )

        best_score = 0.0
        best_chunk = None
        best_quote = None

        # Clean claim tokens for lexical checking
        claim_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", claim.lower()))
        # Check for numbers/dates/entities in claim (key facts that must match)
        claim_numbers = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", claim))

        # 1. First pass: High-speed lexical + entity matching across chunks
        for chunk in retrieved_chunks:
            chunk_text = chunk.text.lower()
            chunk_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", chunk_text))
            
            # Entity/number check: if claim has numbers, chunk must contain them
            if claim_numbers:
                numbers_in_chunk = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", chunk.text))
                if not claim_numbers.issubset(numbers_in_chunk) and not (claim_numbers & numbers_in_chunk):
                    # Penalize if numbers conflict
                    number_match_penalty = 0.5
                else:
                    number_match_penalty = 1.0
            else:
                number_match_penalty = 1.0

            # Jaccard overlap on content words
            if claim_words:
                overlap = len(claim_words & chunk_words) / len(claim_words)
            else:
                overlap = 0.0

            combined_lexical = overlap * number_match_penalty
            if combined_lexical > best_score:
                best_score = combined_lexical
                best_chunk = chunk
                best_quote = chunk.text[:150]

        # 2. Second pass: Cross-encoder semantic entailment if available
        if self.cross_encoder is not None and best_chunk is not None:
            try:
                pairs = [[best_chunk.text, claim]]
                ce_score = float(self.cross_encoder.predict(pairs)[0])
                # Sigmoid normalization of cross-encoder logit for ms-marco
                prob = 1.0 / (1.0 + float(re.sub(r"-", "", f"{math.exp(-ce_score):.8f}"))) if ce_score < 700 else 1.0
                # Blend cross-encoder confidence with entity overlap
                confidence = (prob * 0.6) + (best_score * 0.4)
            except Exception:
                confidence = best_score
        else:
            confidence = best_score

        is_supported = confidence >= self.threshold

        return ClaimVerification(
            claim=claim,
            status="SUPPORTED" if is_supported else "UNSUPPORTED",
            confidence=round(confidence, 3),
            supporting_chunk_id=best_chunk.chunk_id if (best_chunk and is_supported) else None,
            supporting_quote=best_quote if is_supported else None
        )

    def verify_answer(
        self,
        answer: str,
        retrieved_chunks: List[DocumentChunk]
    ) -> VerificationResult:
        """Verifies the complete answer and produces an audit report."""
        if "not found in the document" in answer.lower():
            return VerificationResult(
                original_answer=answer,
                verified_answer=answer,
                total_claims=0,
                supported_claims=[],
                unsupported_claims=[],
                faithfulness_score=1.0,
                has_hallucination=False
            )

        claims = self.extract_claims(answer)
        if not claims:
            return VerificationResult(
                original_answer=answer,
                verified_answer=answer,
                total_claims=0,
                supported_claims=[],
                unsupported_claims=[],
                faithfulness_score=1.0,
                has_hallucination=False
            )

        verifications: List[ClaimVerification] = []
        supported: List[str] = []
        unsupported: List[str] = []

        for c in claims:
            res = self.verify_claim(c, retrieved_chunks)
            verifications.append(res)
            if res.status == "SUPPORTED":
                supported.append(c)
            else:
                unsupported.append(c)

        total_claims = len(claims)
        faithfulness_score = len(supported) / total_claims if total_claims > 0 else 1.0
        has_hallucination = len(unsupported) > 0

        # Construct verified answer
        verified_lines = []
        for line in answer.splitlines():
            line_clean = line.strip()
            if not line_clean:
                verified_lines.append(line)
                continue

            # Check if this line corresponds to any unsupported claim
            matching_unsupported = [
                u for u in unsupported if u.lower() in line_clean.lower() or line_clean.lower() in u.lower()
            ]

            if matching_unsupported:
                if self.flag_unsupported:
                    verified_lines.append(f"{line} ⚠️ [Claim unsupported by retrieved context]")
                # If not flagging, it is excluded (filtered out)
            else:
                verified_lines.append(line)

        verified_answer = "\n".join(verified_lines)

        return VerificationResult(
            original_answer=answer,
            verified_answer=verified_answer,
            total_claims=total_claims,
            supported_claims=supported,
            unsupported_claims=unsupported,
            faithfulness_score=round(faithfulness_score, 3),
            claim_details=verifications,
            has_hallucination=has_hallucination
        )