Spaces:
Running on Zero
Running on Zero
Download evaluate.py from Varun2007/document-qa-system: direct link, hf CLI and curl.
- Browser
- Download file 34.3 kB
-
https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/evaluate.py
- Command line
-
hf download hf://spaces/Varun2007/document-qa-system/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/evaluate.py
34.3 kB
| """Comprehensive Evaluation Module for Document Question Answering. | |
| Evaluates 50 curated Q&A pairs (35 answerable across PDF, DOCX, TXT + 15 unanswerable/negative controls). | |
| Measures: | |
| - Retrieval Recall@5 and Recall@20 | |
| - Answer Faithfulness (supported claims / total claims) | |
| - Answer Correctness (semantic overlap with ground-truth) | |
| - Hallucination Rate (% of responses containing unsupported claims) | |
| Compares 4 configurations: | |
| 1. Baseline 1: Plain Dense Vector Search (FAISS only, top 5, no reranker) | |
| 2. Baseline 2: Plain BM25 Lexical Search (BM25 only, top 5, no reranker) | |
| 3. Baseline 3: Hybrid Search without Reranker (Dense + BM25 RRF, top 5) | |
| 4. Proposed: Full Hybrid Search (top 20) + Cross-Encoder Reranker (top 5) + Verification | |
| """ | |
| import json | |
| import os | |
| import re | |
| import time | |
| from dataclasses import dataclass, asdict | |
| from typing import List, Dict, Any, Tuple | |
| from ingest import DocumentIngestionPipeline, DocumentChunk | |
| from retrieve import HybridRetriever | |
| from generate import Generator | |
| from verify import AnswerVerifier | |
| class QAPair: | |
| id: int | |
| question: str | |
| expected_answer: str | |
| document: str | |
| expected_section: str | |
| expected_page: int | |
| is_answerable: bool | |
| keywords: List[str] | |
| # 50 Curated Q&A pairs covering PDF, DOCX, and TXT (35 answerable + 15 negative controls) | |
| EVALUATION_DATASET: List[Dict[str, Any]] = [ | |
| # --- PDF: Financial Report (Questions 1 to 15) --- | |
| { | |
| "id": 1, | |
| "question": "What was the total consolidated revenue in Q3 fiscal year 2023?", | |
| "expected_answer": "Total consolidated revenue was $4.2 billion, representing a 14.8% increase year-over-year.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "1. Executive Summary", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["$4.2 billion", "14.8%", "consolidated revenue"] | |
| }, | |
| { | |
| "id": 2, | |
| "question": "What was the net profit margin and net income in Q3?", | |
| "expected_answer": "Net income was $620 million, yielding a net profit margin of 18.4%.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "1. Executive Summary", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["18.4%", "$620 million", "net profit margin"] | |
| }, | |
| { | |
| "id": 3, | |
| "question": "How much was spent on capital expenditures and what was the free cash flow in Q3?", | |
| "expected_answer": "Capital expenditures were $210 million, resulting in free cash flow of $710 million.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "1. Executive Summary", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["$210 million", "$710 million", "free cash flow"] | |
| }, | |
| { | |
| "id": 4, | |
| "question": "What is the quarterly dividend amount declared per common share?", | |
| "expected_answer": "The Board declared a quarterly cash dividend of $0.45 per common share.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "1. Executive Summary", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["$0.45", "quarterly", "dividend"] | |
| }, | |
| { | |
| "id": 5, | |
| "question": "What percentage of total revenue was contributed by North America?", | |
| "expected_answer": "North America contributed $2.18 billion or 52% of total revenue.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "2. Regional Performance Breakdown", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["52%", "$2.18 billion", "North America"] | |
| }, | |
| { | |
| "id": 6, | |
| "question": "Which region recorded the highest year-over-year growth rate and what was that rate?", | |
| "expected_answer": "Asia-Pacific (APAC) recorded the highest regional growth rate at 22.4% year-over-year.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "2. Regional Performance Breakdown", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["APAC", "Asia-Pacific", "22.4%"] | |
| }, | |
| { | |
| "id": 7, | |
| "question": "What was the gross margin and Q3 revenue for Cloud & AI Solutions?", | |
| "expected_answer": "Cloud & AI Solutions generated $2,150 million in revenue with a gross margin of 68.2%.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "3. Operating Segment Financials", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["68.2%", "2,150", "Cloud & AI Solutions"] | |
| }, | |
| { | |
| "id": 8, | |
| "question": "What was the operating margin for Enterprise Hardware in Q3?", | |
| "expected_answer": "The operating margin for Enterprise Hardware was 14.1%.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "3. Operating Segment Financials", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["14.1%", "Enterprise Hardware", "operating margin"] | |
| }, | |
| { | |
| "id": 9, | |
| "question": "How much revenue did Consulting & Managed Services generate in Q3?", | |
| "expected_answer": "Consulting & Managed Services generated $800 million in Q3 revenue.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "3. Operating Segment Financials", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["800", "Consulting & Managed Services"] | |
| }, | |
| { | |
| "id": 10, | |
| "question": "How much was invested in Research and Development during Q3?", | |
| "expected_answer": "Acme invested $410 million in R&D, representing 9.8% of net revenue.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "4. Research & Development Investment", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["$410 million", "9.8%", "R&D"] | |
| }, | |
| { | |
| "id": 11, | |
| "question": "How many patents were granted to Acme by the USPTO in Q3?", | |
| "expected_answer": "Acme was granted 47 new patents by the USPTO, bringing its global patent portfolio to over 1,280.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "4. Research & Development Investment", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["47", "USPTO", "patents"] | |
| }, | |
| { | |
| "id": 12, | |
| "question": "How much are compliance preparations for the EU AI Act projected to cost across FY2024?", | |
| "expected_answer": "Compliance investments for the EU AI Act are projected at $25 million across FY2024.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "5. Risk Factors and Regulatory Environment", | |
| "expected_page": 3, | |
| "is_answerable": True, | |
| "keywords": ["$25 million", "EU AI Act", "compliance"] | |
| }, | |
| { | |
| "id": 13, | |
| "question": "What is the full year 2024 revenue guidance range?", | |
| "expected_answer": "Full year 2024 revenue guidance is raised to $17.5 billion, with a range of $17.4B to $17.6B.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "6. Full Year Guidance and Capital Allocation", | |
| "expected_page": 3, | |
| "is_answerable": True, | |
| "keywords": ["$17.5 billion", "17.4B", "17.6B", "guidance"] | |
| }, | |
| { | |
| "id": 14, | |
| "question": "Where are the new data center expansions planned according to the FY2024 capex budget?", | |
| "expected_answer": "Data center expansions are focused in Ireland and Singapore with an $850 million budget.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "6. Full Year Guidance and Capital Allocation", | |
| "expected_page": 3, | |
| "is_answerable": True, | |
| "keywords": ["Ireland", "Singapore", "data center"] | |
| }, | |
| { | |
| "id": 15, | |
| "question": "How much is targeted for total share repurchases in FY2024?", | |
| "expected_answer": "Total share repurchases for FY2024 are targeted at $1.2 billion.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "6. Full Year Guidance and Capital Allocation", | |
| "expected_page": 3, | |
| "is_answerable": True, | |
| "keywords": ["$1.2 billion", "share repurchases"] | |
| }, | |
| # --- DOCX: Cloud Architecture (Questions 16 to 26) --- | |
| { | |
| "id": 16, | |
| "question": "Across which cloud provider regions is the NexaScale Cloud Platform deployed?", | |
| "expected_answer": "It is deployed active-active across AWS (us-east-1 and us-west-2) and GCP (us-central1).", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "1. Architecture Overview", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["us-east-1", "us-west-2", "us-central1", "GCP", "AWS"] | |
| }, | |
| { | |
| "id": 17, | |
| "question": "What technology is used for container orchestration?", | |
| "expected_answer": "The platform leverages Kubernetes (EKS and GKE) for stateless workload container orchestration.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "1. Architecture Overview", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["Kubernetes", "EKS", "GKE"] | |
| }, | |
| { | |
| "id": 18, | |
| "question": "What is the CIDR block used for VPCs in us-east-1 and us-west-2?", | |
| "expected_answer": "us-east-1 uses 10.100.0.0/16 and us-west-2 uses 10.101.0.0/16.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "2. Network Topography and Interconnects", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["10.100.0.0/16", "10.101.0.0/16", "CIDR"] | |
| }, | |
| { | |
| "id": 19, | |
| "question": "What bandwidth is provisioned for AWS Direct Connect dedicated circuits?", | |
| "expected_answer": "Dual redundant AWS Direct Connect 10 Gbps dedicated circuits are provisioned.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "2. Network Topography and Interconnects", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["10 Gbps", "Direct Connect", "redundant"] | |
| }, | |
| { | |
| "id": 20, | |
| "question": "What database technology powers the transactional persistence tier?", | |
| "expected_answer": "Amazon Aurora PostgreSQL (Serverless v2) configured in Multi-Region Global Database mode.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "3. Database Tier and Caching Layer", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["Aurora PostgreSQL", "Serverless v2", "Multi-Region"] | |
| }, | |
| { | |
| "id": 21, | |
| "question": "What is the average replication lag of the Aurora read replicas?", | |
| "expected_answer": "Average replication lag is less than 12 milliseconds.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "3. Database Tier and Caching Layer", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["12 milliseconds", "replication lag"] | |
| }, | |
| { | |
| "id": 22, | |
| "question": "How many shards are configured in the Redis Cluster?", | |
| "expected_answer": "Redis Cluster (v7.2) is deployed across 6 shards (3 primary, 3 replica).", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "3. Database Tier and Caching Layer", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["6 shards", "3 primary", "Redis Cluster"] | |
| }, | |
| { | |
| "id": 23, | |
| "question": "What is the target p99 API latency specified in the SLO?", | |
| "expected_answer": "The target API latency (p99) is less than 120 ms over a rolling 30 days window.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "4. Service Level Objectives (SLO) Specification", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["120 ms", "p99", "latency"] | |
| }, | |
| { | |
| "id": 24, | |
| "question": "What is the target platform availability SLA and quarterly error budget?", | |
| "expected_answer": "Availability SLA is 99.99% with an error budget of 13.14 minutes downtime per calendar quarter.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "4. Service Level Objectives (SLO) Specification", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["99.99%", "13.14 minutes", "availability"] | |
| }, | |
| { | |
| "id": 25, | |
| "question": "What is the Recovery Time Objective (RTO) and Recovery Point Objective (RPO)?", | |
| "expected_answer": "RTO is less than 30 minutes and RPO is less than 1 minute.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "4. Service Level Objectives (SLO) Specification", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["30 minutes", "1 minute", "RTO", "RPO"] | |
| }, | |
| { | |
| "id": 26, | |
| "question": "How many consecutive synthetic canary probe failures trigger an automated DNS failover?", | |
| "expected_answer": "If 3 consecutive probes fail (evaluated every 10 seconds), ARC shifts DNS routing.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "5. Disaster Recovery and Failover Automation", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["3 consecutive", "10 seconds", "probe"] | |
| }, | |
| # --- TXT: Security Policy (Questions 27 to 35) --- | |
| { | |
| "id": 27, | |
| "question": "What hardware authentication is required for corporate and production systems?", | |
| "expected_answer": "Mandatory MFA using FIDO2-compliant WebAuthn hardware security keys is required.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "1. Access Control and Identity Management (IAM)", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["FIDO2", "WebAuthn", "hardware security keys"] | |
| }, | |
| { | |
| "id": 28, | |
| "question": "What is the session timeout duration for inactive user console sessions?", | |
| "expected_answer": "Inactive user console sessions automatically disconnect after 15 minutes of inactivity.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "1. Access Control and Identity Management (IAM)", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["15 minutes", "inactivity", "session"] | |
| }, | |
| { | |
| "id": 29, | |
| "question": "How long do temporary privileged just-in-time (JIT) access credentials remain valid?", | |
| "expected_answer": "Just-in-time access approvals expire after 4 hours.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "1. Access Control and Identity Management (IAM)", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["4 hours", "JIT", "credentials"] | |
| }, | |
| { | |
| "id": 30, | |
| "question": "What TLS version and encryption algorithm are mandated for data in transit and data at rest?", | |
| "expected_answer": "Mandatory TLS version 1.3 in transit and AES-256 (AES-256-GCM) for data at rest.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "2. Cryptographic and Encryption Standards", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["TLS 1.3", "AES-256", "encryption"] | |
| }, | |
| { | |
| "id": 31, | |
| "question": "How frequently must master cryptographic encryption keys be rotated?", | |
| "expected_answer": "Master encryption keys must undergo automated cryptographic rotation every 90 days.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "2. Cryptographic and Encryption Standards", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["90 days", "rotation", "keys"] | |
| }, | |
| { | |
| "id": 32, | |
| "question": "What is the declaration SLA for a Severity 0 (P0) security incident?", | |
| "expected_answer": "Maximum declaration SLA is 5 minutes from initial detection for a P0 incident.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "3. Incident Response and Declaration SLA", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["5 minutes", "Severity 0", "P0"] | |
| }, | |
| { | |
| "id": 33, | |
| "question": "Within what timeframe must a comprehensive post-mortem analysis be published after a P0 incident?", | |
| "expected_answer": "A comprehensive post-mortem analysis must be published within 72 hours.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "3. Incident Response and Declaration SLA", | |
| "expected_page": 1, | |
| "is_answerable": True, | |
| "keywords": ["72 hours", "post-mortem", "P0"] | |
| }, | |
| { | |
| "id": 34, | |
| "question": "What is the remediation SLA for Critical vulnerabilities (CVSS 9.0 to 10.0)?", | |
| "expected_answer": "Remediation SLA is 48 hours from discovery for Critical vulnerabilities.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "4. Vulnerability Management and Remediation Timelines", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["48 hours", "Critical", "CVSS"] | |
| }, | |
| { | |
| "id": 35, | |
| "question": "Within what timeframe must a lost or stolen employee device be reported to security?", | |
| "expected_answer": "Any lost or stolen device must be reported to security@acme.corp within 2 hours.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "5. Employee Endpoint and Device Security", | |
| "expected_page": 2, | |
| "is_answerable": True, | |
| "keywords": ["2 hours", "security@acme.corp", "device"] | |
| }, | |
| # --- Negative / Out-of-Domain Controls (Questions 36 to 50) --- | |
| { | |
| "id": 36, | |
| "question": "What was the total CEO annual salary and executive bonus in fiscal year 2021?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 37, | |
| "question": "What is the company policy regarding reimbursement for home office internet bills?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 38, | |
| "question": "Does the NexaScale Cloud Platform use Azure Cosmos DB for multi-master global writes?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 39, | |
| "question": "What was Acme's total revenue in Latin America during Q2 2022?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 40, | |
| "question": "How many days of paid parental leave do full-time employees receive?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 41, | |
| "question": "What is the specific brand and model of the physical top-of-rack data center switches?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 42, | |
| "question": "What was the stock price of Acme Global Enterprises on December 31, 2023?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 43, | |
| "question": "Which catering vendor provides cafeteria lunch services at the company headquarters?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 44, | |
| "question": "Does the system support Apache Cassandra for cold telemetry archiving?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 45, | |
| "question": "What is the budget for annual employee gym memberships and wellness stipends?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 46, | |
| "question": "What are the rules for reserving visitor parking spots in the subterranean garage?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 47, | |
| "question": "Is Docker Swarm used as a backup container orchestrator in us-west-2?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 48, | |
| "question": "What was the total marketing spend on Super Bowl television commercials in 2023?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_financial_report.pdf", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 49, | |
| "question": "What is the mandatory dress code policy for employees working in customer-facing roles?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_security_policy.txt", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| { | |
| "id": 50, | |
| "question": "Does the cloud architecture run on Oracle Cloud Infrastructure (OCI) in Frankfurt?", | |
| "expected_answer": "Not found in the document.", | |
| "document": "sample_cloud_architecture.docx", | |
| "expected_section": "None", | |
| "expected_page": 0, | |
| "is_answerable": False, | |
| "keywords": ["Not found in the document."] | |
| }, | |
| ] | |
| def compute_token_f1(predicted: str, ground_truth: str) -> float: | |
| """Computes token-level precision, recall, and F1 score.""" | |
| pred_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", predicted.lower()) | |
| gt_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", ground_truth.lower()) | |
| if not pred_tokens or not gt_tokens: | |
| return 0.0 | |
| common = set(pred_tokens) & set(gt_tokens) | |
| if not common: | |
| return 0.0 | |
| precision = len(common) / len(pred_tokens) | |
| recall = len(common) / len(gt_tokens) | |
| f1 = (2 * precision * recall) / (precision + recall) | |
| return f1 | |
| def evaluate_pipeline( | |
| pipeline_mode: str, | |
| retriever: HybridRetriever, | |
| generator: Generator, | |
| dataset: List[Dict[str, Any]], | |
| use_rerank: bool = True, | |
| use_verification: bool = True | |
| ) -> Dict[str, Any]: | |
| """Evaluates a specific pipeline configuration on the 50 Q&A benchmark.""" | |
| print(f"\nEvaluating pipeline: {pipeline_mode.upper()} (rerank={use_rerank}, verifier={use_verification})...") | |
| start_time = time.time() | |
| total_questions = len(dataset) | |
| answerable_count = sum(1 for q in dataset if q["is_answerable"]) | |
| unanswerable_count = total_questions - answerable_count | |
| recall_at_5_hits = 0 | |
| recall_at_20_hits = 0 | |
| correctness_scores = [] | |
| faithfulness_scores = [] | |
| hallucination_count = 0 | |
| detailed_results = [] | |
| for item in dataset: | |
| qid = item["id"] | |
| q_text = item["question"] | |
| is_ans = item["is_answerable"] | |
| exp_doc = item["document"] | |
| exp_sec = item["expected_section"] | |
| exp_page = item["expected_page"] | |
| keywords = item["keywords"] | |
| # 1. Retrieval | |
| retrieval_mode = pipeline_mode | |
| top_k_fetch = 20 if use_rerank else 5 | |
| rerank_k = 5 | |
| ret_res = retriever.retrieve( | |
| query=q_text, | |
| initial_top_k=top_k_fetch, | |
| rerank_top_k=rerank_k, | |
| mode=retrieval_mode | |
| ) | |
| retrieved_chunks = ret_res["chunks"] | |
| # 2. Check Retrieval Recall (for answerable questions) | |
| is_hit_at_5 = False | |
| is_hit_at_20 = False | |
| if is_ans: | |
| # Check if relevant chunk is in top 5 | |
| for c in retrieved_chunks[:5]: | |
| if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page): | |
| is_hit_at_5 = True | |
| break | |
| if is_hit_at_5: | |
| recall_at_5_hits += 1 | |
| # Check if relevant chunk was in candidates (top 20) | |
| candidate_chunks = ret_res.get("chunks", []) | |
| for c in candidate_chunks: | |
| if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page): | |
| is_hit_at_20 = True | |
| break | |
| if is_hit_at_20 or is_hit_at_5: | |
| recall_at_20_hits += 1 | |
| # 3. Generation & Verification | |
| answer, verification = generator.generate_answer( | |
| query=q_text, | |
| context_chunks=retrieved_chunks[:5], | |
| verify=use_verification | |
| ) | |
| # 4. Measure Correctness & Hallucination | |
| ans_lower = answer.lower() | |
| if not is_ans: | |
| # Unanswerable question: model should refuse with "Not found in the document." | |
| if "not found in the document" in ans_lower: | |
| correctness = 1.0 | |
| is_hallucinated = False | |
| faithfulness = 1.0 | |
| else: | |
| correctness = 0.0 | |
| is_hallucinated = True | |
| faithfulness = 0.0 | |
| hallucination_count += 1 | |
| else: | |
| # Answerable question | |
| kw_hits = sum(1 for kw in keywords if kw.lower() in ans_lower) | |
| kw_ratio = kw_hits / len(keywords) if keywords else 0.0 | |
| f1 = compute_token_f1(answer, item["expected_answer"]) | |
| correctness = (kw_ratio * 0.7) + (f1 * 0.3) | |
| # Faithfulness | |
| if verification: | |
| faithfulness = verification.faithfulness_score | |
| is_hallucinated = verification.has_hallucination | |
| else: | |
| # Fallback faithfulness check against retrieved text | |
| claims = generator.verifier.extract_claims(answer) | |
| supported = sum(1 for c in claims if any(c.lower() in rc.text.lower() for rc in retrieved_chunks[:5])) | |
| faithfulness = supported / len(claims) if claims else 1.0 | |
| is_hallucinated = faithfulness < 0.7 | |
| if is_hallucinated: | |
| hallucination_count += 1 | |
| correctness_scores.append(correctness) | |
| faithfulness_scores.append(faithfulness) | |
| detailed_results.append({ | |
| "id": qid, | |
| "question": q_text, | |
| "answer": answer, | |
| "is_answerable": is_ans, | |
| "retrieval_hit_top5": is_hit_at_5, | |
| "correctness": round(correctness, 3), | |
| "faithfulness": round(faithfulness, 3), | |
| "is_hallucinated": is_hallucinated | |
| }) | |
| elapsed = time.time() - start_time | |
| # Calculate aggregate metrics | |
| recall_at_5 = recall_at_5_hits / answerable_count if answerable_count > 0 else 0.0 | |
| recall_at_20 = recall_at_20_hits / answerable_count if answerable_count > 0 else 0.0 | |
| avg_correctness = sum(correctness_scores) / len(correctness_scores) | |
| avg_faithfulness = sum(faithfulness_scores) / len(faithfulness_scores) | |
| hallucination_rate = hallucination_count / total_questions | |
| metrics = { | |
| "pipeline": pipeline_mode, | |
| "use_rerank": use_rerank, | |
| "use_verification": use_verification, | |
| "total_questions": total_questions, | |
| "recall_at_5": round(recall_at_5 * 100, 2), | |
| "recall_at_20": round(recall_at_20 * 100, 2), | |
| "answer_correctness": round(avg_correctness * 100, 2), | |
| "answer_faithfulness": round(avg_faithfulness * 100, 2), | |
| "hallucination_rate": round(hallucination_rate * 100, 2), | |
| "elapsed_sec": round(elapsed, 2), | |
| "details": detailed_results | |
| } | |
| return metrics | |
| def run_full_evaluation(): | |
| print("=" * 70) | |
| print("DOCUMENT QUESTION ANSWERING SYSTEM - 50 Q&A BENCHMARK EVALUATION") | |
| print("=" * 70) | |
| # 1. Ingest all 3 documents | |
| pipeline = DocumentIngestionPipeline(chunk_size=400, chunk_overlap=50) | |
| files = [ | |
| "data/sample_financial_report.pdf", | |
| "data/sample_cloud_architecture.docx", | |
| "data/sample_security_policy.txt" | |
| ] | |
| parsed_docs = [] | |
| total_chunks = 0 | |
| for f in files: | |
| if not os.path.exists(f): | |
| print(f"File {f} not found! Run create_sample_docs.py first.") | |
| return | |
| doc = pipeline.ingest_file(f) | |
| parsed_docs.append(doc) | |
| total_chunks += len(doc.chunks) | |
| print(f"Ingested {doc.file_name}: {doc.total_pages} pages, {len(doc.chunks)} chunks.") | |
| print(f"\nTotal indexed chunks: {total_chunks}") | |
| # 2. Setup Retriever & Generator | |
| retriever = HybridRetriever() | |
| retriever.index_documents(parsed_docs) | |
| verifier = AnswerVerifier(cross_encoder=retriever.reranker, threshold=0.55) | |
| generator = Generator(verifier=verifier, enable_verification=True) | |
| # 3. Run Evaluations across Configurations | |
| # Config 1: Baseline 1 - Plain Dense Vector Search (no BM25, no reranker) | |
| res_dense = evaluate_pipeline( | |
| pipeline_mode="dense_only", | |
| retriever=retriever, | |
| generator=generator, | |
| dataset=EVALUATION_DATASET, | |
| use_rerank=False, | |
| use_verification=False | |
| ) | |
| # Config 2: Baseline 2 - Plain BM25 Search (no Dense, no reranker) | |
| res_bm25 = evaluate_pipeline( | |
| pipeline_mode="bm25_only", | |
| retriever=retriever, | |
| generator=generator, | |
| dataset=EVALUATION_DATASET, | |
| use_rerank=False, | |
| use_verification=False | |
| ) | |
| # Config 3: Baseline 3 - Hybrid Search without Reranker (Dense + BM25, no cross-encoder) | |
| res_hybrid_no_rerank = evaluate_pipeline( | |
| pipeline_mode="hybrid_no_rerank", | |
| retriever=retriever, | |
| generator=generator, | |
| dataset=EVALUATION_DATASET, | |
| use_rerank=False, | |
| use_verification=False | |
| ) | |
| # Config 4: Proposed - Full Hybrid (top 20) + Cross-Encoder Reranker (top 5) + Verification | |
| res_proposed = evaluate_pipeline( | |
| pipeline_mode="hybrid_rerank", | |
| retriever=retriever, | |
| generator=generator, | |
| dataset=EVALUATION_DATASET, | |
| use_rerank=True, | |
| use_verification=True | |
| ) | |
| all_results = [res_dense, res_bm25, res_hybrid_no_rerank, res_proposed] | |
| # Save results to json | |
| report_data = { | |
| "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), | |
| "total_test_pairs": len(EVALUATION_DATASET), | |
| "configurations": [ | |
| {k: v for k, v in r.items() if k != "details"} for r in all_results | |
| ] | |
| } | |
| with open("evaluation_results.json", "w", encoding="utf-8") as f: | |
| json.dump(report_data, f, indent=2) | |
| # Print Formatted Markdown Comparison Table | |
| print("\n" + "=" * 70) | |
| print("BENCHMARK COMPARISON RESULTS") | |
| print("=" * 70) | |
| header = ( | |
| f"| {'Pipeline Configuration':<40} | {'Recall@5':<10} | {'Recall@20':<11} | " | |
| f"{'Correctness':<13} | {'Faithfulness':<14} | {'Hallucination Rate':<20} |" | |
| ) | |
| separator = ( | |
| f"|{'-' * 42}|{'-' * 12}|{'-' * 13}|{'-' * 15}|{'-' * 16}|{'-' * 22}|" | |
| ) | |
| print(header) | |
| print(separator) | |
| labels = [ | |
| "1. Baseline: Plain Dense (FAISS)", | |
| "2. Baseline: Plain BM25", | |
| "3. Baseline: Hybrid Search (No Rerank)", | |
| "4. Proposed: Hybrid + Reranker + Verify" | |
| ] | |
| for label, res in zip(labels, all_results): | |
| row = ( | |
| f"| {label:<40} | " | |
| f"{res['recall_at_5']:>8.1f}% | " | |
| f"{res['recall_at_20']:>9.1f}% | " | |
| f"{res['answer_correctness']:>11.1f}% | " | |
| f"{res['answer_faithfulness']:>12.1f}% | " | |
| f"{res['hallucination_rate']:>18.1f}% |" | |
| ) | |
| print(row) | |
| print(separator) | |
| print("\nResults successfully saved to evaluation_results.json") | |
| if __name__ == "__main__": | |
| run_full_evaluation() | |