"""Comprehensive Evaluation Module for Document Question Answering. Evaluates 50 curated Q&A pairs (35 answerable across PDF, DOCX, TXT + 15 unanswerable/negative controls). Measures: - Retrieval Recall@5 and Recall@20 - Answer Faithfulness (supported claims / total claims) - Answer Correctness (semantic overlap with ground-truth) - Hallucination Rate (% of responses containing unsupported claims) Compares 4 configurations: 1. Baseline 1: Plain Dense Vector Search (FAISS only, top 5, no reranker) 2. Baseline 2: Plain BM25 Lexical Search (BM25 only, top 5, no reranker) 3. Baseline 3: Hybrid Search without Reranker (Dense + BM25 RRF, top 5) 4. Proposed: Full Hybrid Search (top 20) + Cross-Encoder Reranker (top 5) + Verification """ import json import os import re import time from dataclasses import dataclass, asdict from typing import List, Dict, Any, Tuple from ingest import DocumentIngestionPipeline, DocumentChunk from retrieve import HybridRetriever from generate import Generator from verify import AnswerVerifier @dataclass class QAPair: id: int question: str expected_answer: str document: str expected_section: str expected_page: int is_answerable: bool keywords: List[str] # 50 Curated Q&A pairs covering PDF, DOCX, and TXT (35 answerable + 15 negative controls) EVALUATION_DATASET: List[Dict[str, Any]] = [ # --- PDF: Financial Report (Questions 1 to 15) --- { "id": 1, "question": "What was the total consolidated revenue in Q3 fiscal year 2023?", "expected_answer": "Total consolidated revenue was $4.2 billion, representing a 14.8% increase year-over-year.", "document": "sample_financial_report.pdf", "expected_section": "1. Executive Summary", "expected_page": 1, "is_answerable": True, "keywords": ["$4.2 billion", "14.8%", "consolidated revenue"] }, { "id": 2, "question": "What was the net profit margin and net income in Q3?", "expected_answer": "Net income was $620 million, yielding a net profit margin of 18.4%.", "document": "sample_financial_report.pdf", "expected_section": "1. Executive Summary", "expected_page": 1, "is_answerable": True, "keywords": ["18.4%", "$620 million", "net profit margin"] }, { "id": 3, "question": "How much was spent on capital expenditures and what was the free cash flow in Q3?", "expected_answer": "Capital expenditures were $210 million, resulting in free cash flow of $710 million.", "document": "sample_financial_report.pdf", "expected_section": "1. Executive Summary", "expected_page": 1, "is_answerable": True, "keywords": ["$210 million", "$710 million", "free cash flow"] }, { "id": 4, "question": "What is the quarterly dividend amount declared per common share?", "expected_answer": "The Board declared a quarterly cash dividend of $0.45 per common share.", "document": "sample_financial_report.pdf", "expected_section": "1. Executive Summary", "expected_page": 1, "is_answerable": True, "keywords": ["$0.45", "quarterly", "dividend"] }, { "id": 5, "question": "What percentage of total revenue was contributed by North America?", "expected_answer": "North America contributed $2.18 billion or 52% of total revenue.", "document": "sample_financial_report.pdf", "expected_section": "2. Regional Performance Breakdown", "expected_page": 1, "is_answerable": True, "keywords": ["52%", "$2.18 billion", "North America"] }, { "id": 6, "question": "Which region recorded the highest year-over-year growth rate and what was that rate?", "expected_answer": "Asia-Pacific (APAC) recorded the highest regional growth rate at 22.4% year-over-year.", "document": "sample_financial_report.pdf", "expected_section": "2. Regional Performance Breakdown", "expected_page": 1, "is_answerable": True, "keywords": ["APAC", "Asia-Pacific", "22.4%"] }, { "id": 7, "question": "What was the gross margin and Q3 revenue for Cloud & AI Solutions?", "expected_answer": "Cloud & AI Solutions generated $2,150 million in revenue with a gross margin of 68.2%.", "document": "sample_financial_report.pdf", "expected_section": "3. Operating Segment Financials", "expected_page": 2, "is_answerable": True, "keywords": ["68.2%", "2,150", "Cloud & AI Solutions"] }, { "id": 8, "question": "What was the operating margin for Enterprise Hardware in Q3?", "expected_answer": "The operating margin for Enterprise Hardware was 14.1%.", "document": "sample_financial_report.pdf", "expected_section": "3. Operating Segment Financials", "expected_page": 2, "is_answerable": True, "keywords": ["14.1%", "Enterprise Hardware", "operating margin"] }, { "id": 9, "question": "How much revenue did Consulting & Managed Services generate in Q3?", "expected_answer": "Consulting & Managed Services generated $800 million in Q3 revenue.", "document": "sample_financial_report.pdf", "expected_section": "3. Operating Segment Financials", "expected_page": 2, "is_answerable": True, "keywords": ["800", "Consulting & Managed Services"] }, { "id": 10, "question": "How much was invested in Research and Development during Q3?", "expected_answer": "Acme invested $410 million in R&D, representing 9.8% of net revenue.", "document": "sample_financial_report.pdf", "expected_section": "4. Research & Development Investment", "expected_page": 2, "is_answerable": True, "keywords": ["$410 million", "9.8%", "R&D"] }, { "id": 11, "question": "How many patents were granted to Acme by the USPTO in Q3?", "expected_answer": "Acme was granted 47 new patents by the USPTO, bringing its global patent portfolio to over 1,280.", "document": "sample_financial_report.pdf", "expected_section": "4. Research & Development Investment", "expected_page": 2, "is_answerable": True, "keywords": ["47", "USPTO", "patents"] }, { "id": 12, "question": "How much are compliance preparations for the EU AI Act projected to cost across FY2024?", "expected_answer": "Compliance investments for the EU AI Act are projected at $25 million across FY2024.", "document": "sample_financial_report.pdf", "expected_section": "5. Risk Factors and Regulatory Environment", "expected_page": 3, "is_answerable": True, "keywords": ["$25 million", "EU AI Act", "compliance"] }, { "id": 13, "question": "What is the full year 2024 revenue guidance range?", "expected_answer": "Full year 2024 revenue guidance is raised to $17.5 billion, with a range of $17.4B to $17.6B.", "document": "sample_financial_report.pdf", "expected_section": "6. Full Year Guidance and Capital Allocation", "expected_page": 3, "is_answerable": True, "keywords": ["$17.5 billion", "17.4B", "17.6B", "guidance"] }, { "id": 14, "question": "Where are the new data center expansions planned according to the FY2024 capex budget?", "expected_answer": "Data center expansions are focused in Ireland and Singapore with an $850 million budget.", "document": "sample_financial_report.pdf", "expected_section": "6. Full Year Guidance and Capital Allocation", "expected_page": 3, "is_answerable": True, "keywords": ["Ireland", "Singapore", "data center"] }, { "id": 15, "question": "How much is targeted for total share repurchases in FY2024?", "expected_answer": "Total share repurchases for FY2024 are targeted at $1.2 billion.", "document": "sample_financial_report.pdf", "expected_section": "6. Full Year Guidance and Capital Allocation", "expected_page": 3, "is_answerable": True, "keywords": ["$1.2 billion", "share repurchases"] }, # --- DOCX: Cloud Architecture (Questions 16 to 26) --- { "id": 16, "question": "Across which cloud provider regions is the NexaScale Cloud Platform deployed?", "expected_answer": "It is deployed active-active across AWS (us-east-1 and us-west-2) and GCP (us-central1).", "document": "sample_cloud_architecture.docx", "expected_section": "1. Architecture Overview", "expected_page": 1, "is_answerable": True, "keywords": ["us-east-1", "us-west-2", "us-central1", "GCP", "AWS"] }, { "id": 17, "question": "What technology is used for container orchestration?", "expected_answer": "The platform leverages Kubernetes (EKS and GKE) for stateless workload container orchestration.", "document": "sample_cloud_architecture.docx", "expected_section": "1. Architecture Overview", "expected_page": 1, "is_answerable": True, "keywords": ["Kubernetes", "EKS", "GKE"] }, { "id": 18, "question": "What is the CIDR block used for VPCs in us-east-1 and us-west-2?", "expected_answer": "us-east-1 uses 10.100.0.0/16 and us-west-2 uses 10.101.0.0/16.", "document": "sample_cloud_architecture.docx", "expected_section": "2. Network Topography and Interconnects", "expected_page": 1, "is_answerable": True, "keywords": ["10.100.0.0/16", "10.101.0.0/16", "CIDR"] }, { "id": 19, "question": "What bandwidth is provisioned for AWS Direct Connect dedicated circuits?", "expected_answer": "Dual redundant AWS Direct Connect 10 Gbps dedicated circuits are provisioned.", "document": "sample_cloud_architecture.docx", "expected_section": "2. Network Topography and Interconnects", "expected_page": 1, "is_answerable": True, "keywords": ["10 Gbps", "Direct Connect", "redundant"] }, { "id": 20, "question": "What database technology powers the transactional persistence tier?", "expected_answer": "Amazon Aurora PostgreSQL (Serverless v2) configured in Multi-Region Global Database mode.", "document": "sample_cloud_architecture.docx", "expected_section": "3. Database Tier and Caching Layer", "expected_page": 1, "is_answerable": True, "keywords": ["Aurora PostgreSQL", "Serverless v2", "Multi-Region"] }, { "id": 21, "question": "What is the average replication lag of the Aurora read replicas?", "expected_answer": "Average replication lag is less than 12 milliseconds.", "document": "sample_cloud_architecture.docx", "expected_section": "3. Database Tier and Caching Layer", "expected_page": 1, "is_answerable": True, "keywords": ["12 milliseconds", "replication lag"] }, { "id": 22, "question": "How many shards are configured in the Redis Cluster?", "expected_answer": "Redis Cluster (v7.2) is deployed across 6 shards (3 primary, 3 replica).", "document": "sample_cloud_architecture.docx", "expected_section": "3. Database Tier and Caching Layer", "expected_page": 1, "is_answerable": True, "keywords": ["6 shards", "3 primary", "Redis Cluster"] }, { "id": 23, "question": "What is the target p99 API latency specified in the SLO?", "expected_answer": "The target API latency (p99) is less than 120 ms over a rolling 30 days window.", "document": "sample_cloud_architecture.docx", "expected_section": "4. Service Level Objectives (SLO) Specification", "expected_page": 1, "is_answerable": True, "keywords": ["120 ms", "p99", "latency"] }, { "id": 24, "question": "What is the target platform availability SLA and quarterly error budget?", "expected_answer": "Availability SLA is 99.99% with an error budget of 13.14 minutes downtime per calendar quarter.", "document": "sample_cloud_architecture.docx", "expected_section": "4. Service Level Objectives (SLO) Specification", "expected_page": 1, "is_answerable": True, "keywords": ["99.99%", "13.14 minutes", "availability"] }, { "id": 25, "question": "What is the Recovery Time Objective (RTO) and Recovery Point Objective (RPO)?", "expected_answer": "RTO is less than 30 minutes and RPO is less than 1 minute.", "document": "sample_cloud_architecture.docx", "expected_section": "4. Service Level Objectives (SLO) Specification", "expected_page": 1, "is_answerable": True, "keywords": ["30 minutes", "1 minute", "RTO", "RPO"] }, { "id": 26, "question": "How many consecutive synthetic canary probe failures trigger an automated DNS failover?", "expected_answer": "If 3 consecutive probes fail (evaluated every 10 seconds), ARC shifts DNS routing.", "document": "sample_cloud_architecture.docx", "expected_section": "5. Disaster Recovery and Failover Automation", "expected_page": 1, "is_answerable": True, "keywords": ["3 consecutive", "10 seconds", "probe"] }, # --- TXT: Security Policy (Questions 27 to 35) --- { "id": 27, "question": "What hardware authentication is required for corporate and production systems?", "expected_answer": "Mandatory MFA using FIDO2-compliant WebAuthn hardware security keys is required.", "document": "sample_security_policy.txt", "expected_section": "1. Access Control and Identity Management (IAM)", "expected_page": 1, "is_answerable": True, "keywords": ["FIDO2", "WebAuthn", "hardware security keys"] }, { "id": 28, "question": "What is the session timeout duration for inactive user console sessions?", "expected_answer": "Inactive user console sessions automatically disconnect after 15 minutes of inactivity.", "document": "sample_security_policy.txt", "expected_section": "1. Access Control and Identity Management (IAM)", "expected_page": 1, "is_answerable": True, "keywords": ["15 minutes", "inactivity", "session"] }, { "id": 29, "question": "How long do temporary privileged just-in-time (JIT) access credentials remain valid?", "expected_answer": "Just-in-time access approvals expire after 4 hours.", "document": "sample_security_policy.txt", "expected_section": "1. Access Control and Identity Management (IAM)", "expected_page": 1, "is_answerable": True, "keywords": ["4 hours", "JIT", "credentials"] }, { "id": 30, "question": "What TLS version and encryption algorithm are mandated for data in transit and data at rest?", "expected_answer": "Mandatory TLS version 1.3 in transit and AES-256 (AES-256-GCM) for data at rest.", "document": "sample_security_policy.txt", "expected_section": "2. Cryptographic and Encryption Standards", "expected_page": 1, "is_answerable": True, "keywords": ["TLS 1.3", "AES-256", "encryption"] }, { "id": 31, "question": "How frequently must master cryptographic encryption keys be rotated?", "expected_answer": "Master encryption keys must undergo automated cryptographic rotation every 90 days.", "document": "sample_security_policy.txt", "expected_section": "2. Cryptographic and Encryption Standards", "expected_page": 1, "is_answerable": True, "keywords": ["90 days", "rotation", "keys"] }, { "id": 32, "question": "What is the declaration SLA for a Severity 0 (P0) security incident?", "expected_answer": "Maximum declaration SLA is 5 minutes from initial detection for a P0 incident.", "document": "sample_security_policy.txt", "expected_section": "3. Incident Response and Declaration SLA", "expected_page": 1, "is_answerable": True, "keywords": ["5 minutes", "Severity 0", "P0"] }, { "id": 33, "question": "Within what timeframe must a comprehensive post-mortem analysis be published after a P0 incident?", "expected_answer": "A comprehensive post-mortem analysis must be published within 72 hours.", "document": "sample_security_policy.txt", "expected_section": "3. Incident Response and Declaration SLA", "expected_page": 1, "is_answerable": True, "keywords": ["72 hours", "post-mortem", "P0"] }, { "id": 34, "question": "What is the remediation SLA for Critical vulnerabilities (CVSS 9.0 to 10.0)?", "expected_answer": "Remediation SLA is 48 hours from discovery for Critical vulnerabilities.", "document": "sample_security_policy.txt", "expected_section": "4. Vulnerability Management and Remediation Timelines", "expected_page": 2, "is_answerable": True, "keywords": ["48 hours", "Critical", "CVSS"] }, { "id": 35, "question": "Within what timeframe must a lost or stolen employee device be reported to security?", "expected_answer": "Any lost or stolen device must be reported to security@acme.corp within 2 hours.", "document": "sample_security_policy.txt", "expected_section": "5. Employee Endpoint and Device Security", "expected_page": 2, "is_answerable": True, "keywords": ["2 hours", "security@acme.corp", "device"] }, # --- Negative / Out-of-Domain Controls (Questions 36 to 50) --- { "id": 36, "question": "What was the total CEO annual salary and executive bonus in fiscal year 2021?", "expected_answer": "Not found in the document.", "document": "sample_financial_report.pdf", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 37, "question": "What is the company policy regarding reimbursement for home office internet bills?", "expected_answer": "Not found in the document.", "document": "sample_security_policy.txt", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 38, "question": "Does the NexaScale Cloud Platform use Azure Cosmos DB for multi-master global writes?", "expected_answer": "Not found in the document.", "document": "sample_cloud_architecture.docx", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 39, "question": "What was Acme's total revenue in Latin America during Q2 2022?", "expected_answer": "Not found in the document.", "document": "sample_financial_report.pdf", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 40, "question": "How many days of paid parental leave do full-time employees receive?", "expected_answer": "Not found in the document.", "document": "sample_security_policy.txt", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 41, "question": "What is the specific brand and model of the physical top-of-rack data center switches?", "expected_answer": "Not found in the document.", "document": "sample_cloud_architecture.docx", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 42, "question": "What was the stock price of Acme Global Enterprises on December 31, 2023?", "expected_answer": "Not found in the document.", "document": "sample_financial_report.pdf", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 43, "question": "Which catering vendor provides cafeteria lunch services at the company headquarters?", "expected_answer": "Not found in the document.", "document": "sample_security_policy.txt", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 44, "question": "Does the system support Apache Cassandra for cold telemetry archiving?", "expected_answer": "Not found in the document.", "document": "sample_cloud_architecture.docx", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 45, "question": "What is the budget for annual employee gym memberships and wellness stipends?", "expected_answer": "Not found in the document.", "document": "sample_financial_report.pdf", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 46, "question": "What are the rules for reserving visitor parking spots in the subterranean garage?", "expected_answer": "Not found in the document.", "document": "sample_security_policy.txt", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 47, "question": "Is Docker Swarm used as a backup container orchestrator in us-west-2?", "expected_answer": "Not found in the document.", "document": "sample_cloud_architecture.docx", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 48, "question": "What was the total marketing spend on Super Bowl television commercials in 2023?", "expected_answer": "Not found in the document.", "document": "sample_financial_report.pdf", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 49, "question": "What is the mandatory dress code policy for employees working in customer-facing roles?", "expected_answer": "Not found in the document.", "document": "sample_security_policy.txt", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, { "id": 50, "question": "Does the cloud architecture run on Oracle Cloud Infrastructure (OCI) in Frankfurt?", "expected_answer": "Not found in the document.", "document": "sample_cloud_architecture.docx", "expected_section": "None", "expected_page": 0, "is_answerable": False, "keywords": ["Not found in the document."] }, ] def compute_token_f1(predicted: str, ground_truth: str) -> float: """Computes token-level precision, recall, and F1 score.""" pred_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", predicted.lower()) gt_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", ground_truth.lower()) if not pred_tokens or not gt_tokens: return 0.0 common = set(pred_tokens) & set(gt_tokens) if not common: return 0.0 precision = len(common) / len(pred_tokens) recall = len(common) / len(gt_tokens) f1 = (2 * precision * recall) / (precision + recall) return f1 def evaluate_pipeline( pipeline_mode: str, retriever: HybridRetriever, generator: Generator, dataset: List[Dict[str, Any]], use_rerank: bool = True, use_verification: bool = True ) -> Dict[str, Any]: """Evaluates a specific pipeline configuration on the 50 Q&A benchmark.""" print(f"\nEvaluating pipeline: {pipeline_mode.upper()} (rerank={use_rerank}, verifier={use_verification})...") start_time = time.time() total_questions = len(dataset) answerable_count = sum(1 for q in dataset if q["is_answerable"]) unanswerable_count = total_questions - answerable_count recall_at_5_hits = 0 recall_at_20_hits = 0 correctness_scores = [] faithfulness_scores = [] hallucination_count = 0 detailed_results = [] for item in dataset: qid = item["id"] q_text = item["question"] is_ans = item["is_answerable"] exp_doc = item["document"] exp_sec = item["expected_section"] exp_page = item["expected_page"] keywords = item["keywords"] # 1. Retrieval retrieval_mode = pipeline_mode top_k_fetch = 20 if use_rerank else 5 rerank_k = 5 ret_res = retriever.retrieve( query=q_text, initial_top_k=top_k_fetch, rerank_top_k=rerank_k, mode=retrieval_mode ) retrieved_chunks = ret_res["chunks"] # 2. Check Retrieval Recall (for answerable questions) is_hit_at_5 = False is_hit_at_20 = False if is_ans: # Check if relevant chunk is in top 5 for c in retrieved_chunks[:5]: if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page): is_hit_at_5 = True break if is_hit_at_5: recall_at_5_hits += 1 # Check if relevant chunk was in candidates (top 20) candidate_chunks = ret_res.get("chunks", []) for c in candidate_chunks: if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page): is_hit_at_20 = True break if is_hit_at_20 or is_hit_at_5: recall_at_20_hits += 1 # 3. Generation & Verification answer, verification = generator.generate_answer( query=q_text, context_chunks=retrieved_chunks[:5], verify=use_verification ) # 4. Measure Correctness & Hallucination ans_lower = answer.lower() if not is_ans: # Unanswerable question: model should refuse with "Not found in the document." if "not found in the document" in ans_lower: correctness = 1.0 is_hallucinated = False faithfulness = 1.0 else: correctness = 0.0 is_hallucinated = True faithfulness = 0.0 hallucination_count += 1 else: # Answerable question kw_hits = sum(1 for kw in keywords if kw.lower() in ans_lower) kw_ratio = kw_hits / len(keywords) if keywords else 0.0 f1 = compute_token_f1(answer, item["expected_answer"]) correctness = (kw_ratio * 0.7) + (f1 * 0.3) # Faithfulness if verification: faithfulness = verification.faithfulness_score is_hallucinated = verification.has_hallucination else: # Fallback faithfulness check against retrieved text claims = generator.verifier.extract_claims(answer) supported = sum(1 for c in claims if any(c.lower() in rc.text.lower() for rc in retrieved_chunks[:5])) faithfulness = supported / len(claims) if claims else 1.0 is_hallucinated = faithfulness < 0.7 if is_hallucinated: hallucination_count += 1 correctness_scores.append(correctness) faithfulness_scores.append(faithfulness) detailed_results.append({ "id": qid, "question": q_text, "answer": answer, "is_answerable": is_ans, "retrieval_hit_top5": is_hit_at_5, "correctness": round(correctness, 3), "faithfulness": round(faithfulness, 3), "is_hallucinated": is_hallucinated }) elapsed = time.time() - start_time # Calculate aggregate metrics recall_at_5 = recall_at_5_hits / answerable_count if answerable_count > 0 else 0.0 recall_at_20 = recall_at_20_hits / answerable_count if answerable_count > 0 else 0.0 avg_correctness = sum(correctness_scores) / len(correctness_scores) avg_faithfulness = sum(faithfulness_scores) / len(faithfulness_scores) hallucination_rate = hallucination_count / total_questions metrics = { "pipeline": pipeline_mode, "use_rerank": use_rerank, "use_verification": use_verification, "total_questions": total_questions, "recall_at_5": round(recall_at_5 * 100, 2), "recall_at_20": round(recall_at_20 * 100, 2), "answer_correctness": round(avg_correctness * 100, 2), "answer_faithfulness": round(avg_faithfulness * 100, 2), "hallucination_rate": round(hallucination_rate * 100, 2), "elapsed_sec": round(elapsed, 2), "details": detailed_results } return metrics def run_full_evaluation(): print("=" * 70) print("DOCUMENT QUESTION ANSWERING SYSTEM - 50 Q&A BENCHMARK EVALUATION") print("=" * 70) # 1. Ingest all 3 documents pipeline = DocumentIngestionPipeline(chunk_size=400, chunk_overlap=50) files = [ "data/sample_financial_report.pdf", "data/sample_cloud_architecture.docx", "data/sample_security_policy.txt" ] parsed_docs = [] total_chunks = 0 for f in files: if not os.path.exists(f): print(f"File {f} not found! Run create_sample_docs.py first.") return doc = pipeline.ingest_file(f) parsed_docs.append(doc) total_chunks += len(doc.chunks) print(f"Ingested {doc.file_name}: {doc.total_pages} pages, {len(doc.chunks)} chunks.") print(f"\nTotal indexed chunks: {total_chunks}") # 2. Setup Retriever & Generator retriever = HybridRetriever() retriever.index_documents(parsed_docs) verifier = AnswerVerifier(cross_encoder=retriever.reranker, threshold=0.55) generator = Generator(verifier=verifier, enable_verification=True) # 3. Run Evaluations across Configurations # Config 1: Baseline 1 - Plain Dense Vector Search (no BM25, no reranker) res_dense = evaluate_pipeline( pipeline_mode="dense_only", retriever=retriever, generator=generator, dataset=EVALUATION_DATASET, use_rerank=False, use_verification=False ) # Config 2: Baseline 2 - Plain BM25 Search (no Dense, no reranker) res_bm25 = evaluate_pipeline( pipeline_mode="bm25_only", retriever=retriever, generator=generator, dataset=EVALUATION_DATASET, use_rerank=False, use_verification=False ) # Config 3: Baseline 3 - Hybrid Search without Reranker (Dense + BM25, no cross-encoder) res_hybrid_no_rerank = evaluate_pipeline( pipeline_mode="hybrid_no_rerank", retriever=retriever, generator=generator, dataset=EVALUATION_DATASET, use_rerank=False, use_verification=False ) # Config 4: Proposed - Full Hybrid (top 20) + Cross-Encoder Reranker (top 5) + Verification res_proposed = evaluate_pipeline( pipeline_mode="hybrid_rerank", retriever=retriever, generator=generator, dataset=EVALUATION_DATASET, use_rerank=True, use_verification=True ) all_results = [res_dense, res_bm25, res_hybrid_no_rerank, res_proposed] # Save results to json report_data = { "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), "total_test_pairs": len(EVALUATION_DATASET), "configurations": [ {k: v for k, v in r.items() if k != "details"} for r in all_results ] } with open("evaluation_results.json", "w", encoding="utf-8") as f: json.dump(report_data, f, indent=2) # Print Formatted Markdown Comparison Table print("\n" + "=" * 70) print("BENCHMARK COMPARISON RESULTS") print("=" * 70) header = ( f"| {'Pipeline Configuration':<40} | {'Recall@5':<10} | {'Recall@20':<11} | " f"{'Correctness':<13} | {'Faithfulness':<14} | {'Hallucination Rate':<20} |" ) separator = ( f"|{'-' * 42}|{'-' * 12}|{'-' * 13}|{'-' * 15}|{'-' * 16}|{'-' * 22}|" ) print(header) print(separator) labels = [ "1. Baseline: Plain Dense (FAISS)", "2. Baseline: Plain BM25", "3. Baseline: Hybrid Search (No Rerank)", "4. Proposed: Hybrid + Reranker + Verify" ] for label, res in zip(labels, all_results): row = ( f"| {label:<40} | " f"{res['recall_at_5']:>8.1f}% | " f"{res['recall_at_20']:>9.1f}% | " f"{res['answer_correctness']:>11.1f}% | " f"{res['answer_faithfulness']:>12.1f}% | " f"{res['hallucination_rate']:>18.1f}% |" ) print(row) print(separator) print("\nResults successfully saved to evaluation_results.json") if __name__ == "__main__": run_full_evaluation()