"""Claim-Level Answer Verification Module. Performs a second-pass verification check ensuring every claim in the generated answer is strictly supported by the retrieved document chunks. Flags or removes unsupported claims. """ import re from dataclasses import dataclass, field, asdict from typing import List, Dict, Any, Optional from ingest import DocumentChunk @dataclass class ClaimVerification: claim: str status: str # "SUPPORTED", "UNSUPPORTED", "CONTRADICTED" confidence: float supporting_chunk_id: Optional[str] = None supporting_quote: Optional[str] = None @dataclass class VerificationResult: original_answer: str verified_answer: str total_claims: int supported_claims: List[str] unsupported_claims: List[str] faithfulness_score: float claim_details: List[ClaimVerification] = field(default_factory=list) has_hallucination: bool = False def to_dict(self) -> Dict[str, Any]: return asdict(self) class AnswerVerifier: """ Second-pass verifier for Document Question Answering. 1. Extracts atomic claims/propositions from the answer. 2. Verifies each claim against the retrieved context chunks using semantic + lexical entailment. 3. Removes or flags unsupported claims and computes answer faithfulness. """ def __init__( self, cross_encoder=None, threshold: float = 0.55, flag_unsupported: bool = True, llm_client=None ): self.cross_encoder = cross_encoder self.threshold = threshold self.flag_unsupported = flag_unsupported self.llm_client = llm_client def extract_claims(self, text: str) -> List[str]: """Decomposes an answer into discrete factual propositions/sentences.""" text = text.strip() if not text or "not found in the document" in text.lower(): return [] # Remove citation brackets like [Page 2, Section "Intro"] for claim extraction clean_text = re.sub(r"\[Page\s+\d+[^\]]*\]", "", text, flags=re.IGNORECASE) # Split on sentence boundaries and bullet points raw_sentences = re.split(r"(?<=[.!?])\s+|\n+[\s*-]*", clean_text) claims = [] filler_patterns = [ r"^(in summary|in conclusion|to conclude|based on the document|according to the document)\b", r"^(as mentioned above|therefore|hence|overall)\b", ] for s in raw_sentences: s_clean = s.strip() # Strip markdown bullets s_clean = re.sub(r"^[\d+.\-*#]+\s*", "", s_clean).strip() if len(s_clean.split()) < 3: continue # Skip pure filler if any(re.match(pat, s_clean, re.IGNORECASE) for pat in filler_patterns) and len(s_clean.split()) < 6: continue claims.append(s_clean) return claims def verify_claim(self, claim: str, retrieved_chunks: List[DocumentChunk]) -> ClaimVerification: """Verifies an individual claim against all retrieved chunks.""" if not retrieved_chunks: return ClaimVerification( claim=claim, status="UNSUPPORTED", confidence=0.0 ) best_score = 0.0 best_chunk = None best_quote = None # Clean claim tokens for lexical checking claim_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", claim.lower())) # Check for numbers/dates/entities in claim (key facts that must match) claim_numbers = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", claim)) # 1. First pass: High-speed lexical + entity matching across chunks for chunk in retrieved_chunks: chunk_text = chunk.text.lower() chunk_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", chunk_text)) # Entity/number check: if claim has numbers, chunk must contain them if claim_numbers: numbers_in_chunk = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", chunk.text)) if not claim_numbers.issubset(numbers_in_chunk) and not (claim_numbers & numbers_in_chunk): # Penalize if numbers conflict number_match_penalty = 0.5 else: number_match_penalty = 1.0 else: number_match_penalty = 1.0 # Jaccard overlap on content words if claim_words: overlap = len(claim_words & chunk_words) / len(claim_words) else: overlap = 0.0 combined_lexical = overlap * number_match_penalty if combined_lexical > best_score: best_score = combined_lexical best_chunk = chunk best_quote = chunk.text[:150] # 2. Second pass: Cross-encoder semantic entailment if available if self.cross_encoder is not None and best_chunk is not None: try: pairs = [[best_chunk.text, claim]] ce_score = float(self.cross_encoder.predict(pairs)[0]) # Sigmoid normalization of cross-encoder logit for ms-marco prob = 1.0 / (1.0 + float(re.sub(r"-", "", f"{math.exp(-ce_score):.8f}"))) if ce_score < 700 else 1.0 # Blend cross-encoder confidence with entity overlap confidence = (prob * 0.6) + (best_score * 0.4) except Exception: confidence = best_score else: confidence = best_score is_supported = confidence >= self.threshold return ClaimVerification( claim=claim, status="SUPPORTED" if is_supported else "UNSUPPORTED", confidence=round(confidence, 3), supporting_chunk_id=best_chunk.chunk_id if (best_chunk and is_supported) else None, supporting_quote=best_quote if is_supported else None ) def verify_answer( self, answer: str, retrieved_chunks: List[DocumentChunk] ) -> VerificationResult: """Verifies the complete answer and produces an audit report.""" if "not found in the document" in answer.lower(): return VerificationResult( original_answer=answer, verified_answer=answer, total_claims=0, supported_claims=[], unsupported_claims=[], faithfulness_score=1.0, has_hallucination=False ) claims = self.extract_claims(answer) if not claims: return VerificationResult( original_answer=answer, verified_answer=answer, total_claims=0, supported_claims=[], unsupported_claims=[], faithfulness_score=1.0, has_hallucination=False ) verifications: List[ClaimVerification] = [] supported: List[str] = [] unsupported: List[str] = [] for c in claims: res = self.verify_claim(c, retrieved_chunks) verifications.append(res) if res.status == "SUPPORTED": supported.append(c) else: unsupported.append(c) total_claims = len(claims) faithfulness_score = len(supported) / total_claims if total_claims > 0 else 1.0 has_hallucination = len(unsupported) > 0 # Construct verified answer verified_lines = [] for line in answer.splitlines(): line_clean = line.strip() if not line_clean: verified_lines.append(line) continue # Check if this line corresponds to any unsupported claim matching_unsupported = [ u for u in unsupported if u.lower() in line_clean.lower() or line_clean.lower() in u.lower() ] if matching_unsupported: if self.flag_unsupported: verified_lines.append(f"{line} ⚠️ [Claim unsupported by retrieved context]") # If not flagging, it is excluded (filtered out) else: verified_lines.append(line) verified_answer = "\n".join(verified_lines) return VerificationResult( original_answer=answer, verified_answer=verified_answer, total_claims=total_claims, supported_claims=supported, unsupported_claims=unsupported, faithfulness_score=round(faithfulness_score, 3), claim_details=verifications, has_hallucination=has_hallucination )