Spaces:
Running on Zero
Running on Zero
Download verify.py from Varun2007/document-qa-system: direct link, hf CLI and curl.
- Browser
- Download file 8.74 kB
-
https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/verify.py
- Command line
-
hf download hf://spaces/Varun2007/document-qa-system/verify.py
-
curl -L -o verify.py https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/verify.py
8.74 kB
| """Claim-Level Answer Verification Module. | |
| Performs a second-pass verification check ensuring every claim in the generated answer | |
| is strictly supported by the retrieved document chunks. Flags or removes unsupported claims. | |
| """ | |
| import re | |
| from dataclasses import dataclass, field, asdict | |
| from typing import List, Dict, Any, Optional | |
| from ingest import DocumentChunk | |
| class ClaimVerification: | |
| claim: str | |
| status: str # "SUPPORTED", "UNSUPPORTED", "CONTRADICTED" | |
| confidence: float | |
| supporting_chunk_id: Optional[str] = None | |
| supporting_quote: Optional[str] = None | |
| class VerificationResult: | |
| original_answer: str | |
| verified_answer: str | |
| total_claims: int | |
| supported_claims: List[str] | |
| unsupported_claims: List[str] | |
| faithfulness_score: float | |
| claim_details: List[ClaimVerification] = field(default_factory=list) | |
| has_hallucination: bool = False | |
| def to_dict(self) -> Dict[str, Any]: | |
| return asdict(self) | |
| class AnswerVerifier: | |
| """ | |
| Second-pass verifier for Document Question Answering. | |
| 1. Extracts atomic claims/propositions from the answer. | |
| 2. Verifies each claim against the retrieved context chunks using semantic + lexical entailment. | |
| 3. Removes or flags unsupported claims and computes answer faithfulness. | |
| """ | |
| def __init__( | |
| self, | |
| cross_encoder=None, | |
| threshold: float = 0.55, | |
| flag_unsupported: bool = True, | |
| llm_client=None | |
| ): | |
| self.cross_encoder = cross_encoder | |
| self.threshold = threshold | |
| self.flag_unsupported = flag_unsupported | |
| self.llm_client = llm_client | |
| def extract_claims(self, text: str) -> List[str]: | |
| """Decomposes an answer into discrete factual propositions/sentences.""" | |
| text = text.strip() | |
| if not text or "not found in the document" in text.lower(): | |
| return [] | |
| # Remove citation brackets like [Page 2, Section "Intro"] for claim extraction | |
| clean_text = re.sub(r"\[Page\s+\d+[^\]]*\]", "", text, flags=re.IGNORECASE) | |
| # Split on sentence boundaries and bullet points | |
| raw_sentences = re.split(r"(?<=[.!?])\s+|\n+[\s*-]*", clean_text) | |
| claims = [] | |
| filler_patterns = [ | |
| r"^(in summary|in conclusion|to conclude|based on the document|according to the document)\b", | |
| r"^(as mentioned above|therefore|hence|overall)\b", | |
| ] | |
| for s in raw_sentences: | |
| s_clean = s.strip() | |
| # Strip markdown bullets | |
| s_clean = re.sub(r"^[\d+.\-*#]+\s*", "", s_clean).strip() | |
| if len(s_clean.split()) < 3: | |
| continue | |
| # Skip pure filler | |
| if any(re.match(pat, s_clean, re.IGNORECASE) for pat in filler_patterns) and len(s_clean.split()) < 6: | |
| continue | |
| claims.append(s_clean) | |
| return claims | |
| def verify_claim(self, claim: str, retrieved_chunks: List[DocumentChunk]) -> ClaimVerification: | |
| """Verifies an individual claim against all retrieved chunks.""" | |
| if not retrieved_chunks: | |
| return ClaimVerification( | |
| claim=claim, | |
| status="UNSUPPORTED", | |
| confidence=0.0 | |
| ) | |
| best_score = 0.0 | |
| best_chunk = None | |
| best_quote = None | |
| # Clean claim tokens for lexical checking | |
| claim_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", claim.lower())) | |
| # Check for numbers/dates/entities in claim (key facts that must match) | |
| claim_numbers = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", claim)) | |
| # 1. First pass: High-speed lexical + entity matching across chunks | |
| for chunk in retrieved_chunks: | |
| chunk_text = chunk.text.lower() | |
| chunk_words = set(re.findall(r"\b[a-zA-Z0-9_]{3,}\b", chunk_text)) | |
| # Entity/number check: if claim has numbers, chunk must contain them | |
| if claim_numbers: | |
| numbers_in_chunk = set(re.findall(r"\b\d+(?:\.\d+)?%?\b", chunk.text)) | |
| if not claim_numbers.issubset(numbers_in_chunk) and not (claim_numbers & numbers_in_chunk): | |
| # Penalize if numbers conflict | |
| number_match_penalty = 0.5 | |
| else: | |
| number_match_penalty = 1.0 | |
| else: | |
| number_match_penalty = 1.0 | |
| # Jaccard overlap on content words | |
| if claim_words: | |
| overlap = len(claim_words & chunk_words) / len(claim_words) | |
| else: | |
| overlap = 0.0 | |
| combined_lexical = overlap * number_match_penalty | |
| if combined_lexical > best_score: | |
| best_score = combined_lexical | |
| best_chunk = chunk | |
| best_quote = chunk.text[:150] | |
| # 2. Second pass: Cross-encoder semantic entailment if available | |
| if self.cross_encoder is not None and best_chunk is not None: | |
| try: | |
| pairs = [[best_chunk.text, claim]] | |
| ce_score = float(self.cross_encoder.predict(pairs)[0]) | |
| # Sigmoid normalization of cross-encoder logit for ms-marco | |
| prob = 1.0 / (1.0 + float(re.sub(r"-", "", f"{math.exp(-ce_score):.8f}"))) if ce_score < 700 else 1.0 | |
| # Blend cross-encoder confidence with entity overlap | |
| confidence = (prob * 0.6) + (best_score * 0.4) | |
| except Exception: | |
| confidence = best_score | |
| else: | |
| confidence = best_score | |
| is_supported = confidence >= self.threshold | |
| return ClaimVerification( | |
| claim=claim, | |
| status="SUPPORTED" if is_supported else "UNSUPPORTED", | |
| confidence=round(confidence, 3), | |
| supporting_chunk_id=best_chunk.chunk_id if (best_chunk and is_supported) else None, | |
| supporting_quote=best_quote if is_supported else None | |
| ) | |
| def verify_answer( | |
| self, | |
| answer: str, | |
| retrieved_chunks: List[DocumentChunk] | |
| ) -> VerificationResult: | |
| """Verifies the complete answer and produces an audit report.""" | |
| if "not found in the document" in answer.lower(): | |
| return VerificationResult( | |
| original_answer=answer, | |
| verified_answer=answer, | |
| total_claims=0, | |
| supported_claims=[], | |
| unsupported_claims=[], | |
| faithfulness_score=1.0, | |
| has_hallucination=False | |
| ) | |
| claims = self.extract_claims(answer) | |
| if not claims: | |
| return VerificationResult( | |
| original_answer=answer, | |
| verified_answer=answer, | |
| total_claims=0, | |
| supported_claims=[], | |
| unsupported_claims=[], | |
| faithfulness_score=1.0, | |
| has_hallucination=False | |
| ) | |
| verifications: List[ClaimVerification] = [] | |
| supported: List[str] = [] | |
| unsupported: List[str] = [] | |
| for c in claims: | |
| res = self.verify_claim(c, retrieved_chunks) | |
| verifications.append(res) | |
| if res.status == "SUPPORTED": | |
| supported.append(c) | |
| else: | |
| unsupported.append(c) | |
| total_claims = len(claims) | |
| faithfulness_score = len(supported) / total_claims if total_claims > 0 else 1.0 | |
| has_hallucination = len(unsupported) > 0 | |
| # Construct verified answer | |
| verified_lines = [] | |
| for line in answer.splitlines(): | |
| line_clean = line.strip() | |
| if not line_clean: | |
| verified_lines.append(line) | |
| continue | |
| # Check if this line corresponds to any unsupported claim | |
| matching_unsupported = [ | |
| u for u in unsupported if u.lower() in line_clean.lower() or line_clean.lower() in u.lower() | |
| ] | |
| if matching_unsupported: | |
| if self.flag_unsupported: | |
| verified_lines.append(f"{line} ⚠️ [Claim unsupported by retrieved context]") | |
| # If not flagging, it is excluded (filtered out) | |
| else: | |
| verified_lines.append(line) | |
| verified_answer = "\n".join(verified_lines) | |
| return VerificationResult( | |
| original_answer=answer, | |
| verified_answer=verified_answer, | |
| total_claims=total_claims, | |
| supported_claims=supported, | |
| unsupported_claims=unsupported, | |
| faithfulness_score=round(faithfulness_score, 3), | |
| claim_details=verifications, | |
| has_hallucination=has_hallucination | |
| ) | |