File size: 34,264 Bytes
d3d09d6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
"""Comprehensive Evaluation Module for Document Question Answering.

Evaluates 50 curated Q&A pairs (35 answerable across PDF, DOCX, TXT + 15 unanswerable/negative controls).
Measures:
- Retrieval Recall@5 and Recall@20
- Answer Faithfulness (supported claims / total claims)
- Answer Correctness (semantic overlap with ground-truth)
- Hallucination Rate (% of responses containing unsupported claims)

Compares 4 configurations:
1. Baseline 1: Plain Dense Vector Search (FAISS only, top 5, no reranker)
2. Baseline 2: Plain BM25 Lexical Search (BM25 only, top 5, no reranker)
3. Baseline 3: Hybrid Search without Reranker (Dense + BM25 RRF, top 5)
4. Proposed: Full Hybrid Search (top 20) + Cross-Encoder Reranker (top 5) + Verification
"""

import json
import os
import re
import time
from dataclasses import dataclass, asdict
from typing import List, Dict, Any, Tuple

from ingest import DocumentIngestionPipeline, DocumentChunk
from retrieve import HybridRetriever
from generate import Generator
from verify import AnswerVerifier


@dataclass
class QAPair:
    id: int
    question: str
    expected_answer: str
    document: str
    expected_section: str
    expected_page: int
    is_answerable: bool
    keywords: List[str]


# 50 Curated Q&A pairs covering PDF, DOCX, and TXT (35 answerable + 15 negative controls)
EVALUATION_DATASET: List[Dict[str, Any]] = [
    # --- PDF: Financial Report (Questions 1 to 15) ---
    {
        "id": 1,
        "question": "What was the total consolidated revenue in Q3 fiscal year 2023?",
        "expected_answer": "Total consolidated revenue was $4.2 billion, representing a 14.8% increase year-over-year.",
        "document": "sample_financial_report.pdf",
        "expected_section": "1. Executive Summary",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["$4.2 billion", "14.8%", "consolidated revenue"]
    },
    {
        "id": 2,
        "question": "What was the net profit margin and net income in Q3?",
        "expected_answer": "Net income was $620 million, yielding a net profit margin of 18.4%.",
        "document": "sample_financial_report.pdf",
        "expected_section": "1. Executive Summary",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["18.4%", "$620 million", "net profit margin"]
    },
    {
        "id": 3,
        "question": "How much was spent on capital expenditures and what was the free cash flow in Q3?",
        "expected_answer": "Capital expenditures were $210 million, resulting in free cash flow of $710 million.",
        "document": "sample_financial_report.pdf",
        "expected_section": "1. Executive Summary",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["$210 million", "$710 million", "free cash flow"]
    },
    {
        "id": 4,
        "question": "What is the quarterly dividend amount declared per common share?",
        "expected_answer": "The Board declared a quarterly cash dividend of $0.45 per common share.",
        "document": "sample_financial_report.pdf",
        "expected_section": "1. Executive Summary",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["$0.45", "quarterly", "dividend"]
    },
    {
        "id": 5,
        "question": "What percentage of total revenue was contributed by North America?",
        "expected_answer": "North America contributed $2.18 billion or 52% of total revenue.",
        "document": "sample_financial_report.pdf",
        "expected_section": "2. Regional Performance Breakdown",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["52%", "$2.18 billion", "North America"]
    },
    {
        "id": 6,
        "question": "Which region recorded the highest year-over-year growth rate and what was that rate?",
        "expected_answer": "Asia-Pacific (APAC) recorded the highest regional growth rate at 22.4% year-over-year.",
        "document": "sample_financial_report.pdf",
        "expected_section": "2. Regional Performance Breakdown",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["APAC", "Asia-Pacific", "22.4%"]
    },
    {
        "id": 7,
        "question": "What was the gross margin and Q3 revenue for Cloud & AI Solutions?",
        "expected_answer": "Cloud & AI Solutions generated $2,150 million in revenue with a gross margin of 68.2%.",
        "document": "sample_financial_report.pdf",
        "expected_section": "3. Operating Segment Financials",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["68.2%", "2,150", "Cloud & AI Solutions"]
    },
    {
        "id": 8,
        "question": "What was the operating margin for Enterprise Hardware in Q3?",
        "expected_answer": "The operating margin for Enterprise Hardware was 14.1%.",
        "document": "sample_financial_report.pdf",
        "expected_section": "3. Operating Segment Financials",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["14.1%", "Enterprise Hardware", "operating margin"]
    },
    {
        "id": 9,
        "question": "How much revenue did Consulting & Managed Services generate in Q3?",
        "expected_answer": "Consulting & Managed Services generated $800 million in Q3 revenue.",
        "document": "sample_financial_report.pdf",
        "expected_section": "3. Operating Segment Financials",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["800", "Consulting & Managed Services"]
    },
    {
        "id": 10,
        "question": "How much was invested in Research and Development during Q3?",
        "expected_answer": "Acme invested $410 million in R&D, representing 9.8% of net revenue.",
        "document": "sample_financial_report.pdf",
        "expected_section": "4. Research & Development Investment",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["$410 million", "9.8%", "R&D"]
    },
    {
        "id": 11,
        "question": "How many patents were granted to Acme by the USPTO in Q3?",
        "expected_answer": "Acme was granted 47 new patents by the USPTO, bringing its global patent portfolio to over 1,280.",
        "document": "sample_financial_report.pdf",
        "expected_section": "4. Research & Development Investment",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["47", "USPTO", "patents"]
    },
    {
        "id": 12,
        "question": "How much are compliance preparations for the EU AI Act projected to cost across FY2024?",
        "expected_answer": "Compliance investments for the EU AI Act are projected at $25 million across FY2024.",
        "document": "sample_financial_report.pdf",
        "expected_section": "5. Risk Factors and Regulatory Environment",
        "expected_page": 3,
        "is_answerable": True,
        "keywords": ["$25 million", "EU AI Act", "compliance"]
    },
    {
        "id": 13,
        "question": "What is the full year 2024 revenue guidance range?",
        "expected_answer": "Full year 2024 revenue guidance is raised to $17.5 billion, with a range of $17.4B to $17.6B.",
        "document": "sample_financial_report.pdf",
        "expected_section": "6. Full Year Guidance and Capital Allocation",
        "expected_page": 3,
        "is_answerable": True,
        "keywords": ["$17.5 billion", "17.4B", "17.6B", "guidance"]
    },
    {
        "id": 14,
        "question": "Where are the new data center expansions planned according to the FY2024 capex budget?",
        "expected_answer": "Data center expansions are focused in Ireland and Singapore with an $850 million budget.",
        "document": "sample_financial_report.pdf",
        "expected_section": "6. Full Year Guidance and Capital Allocation",
        "expected_page": 3,
        "is_answerable": True,
        "keywords": ["Ireland", "Singapore", "data center"]
    },
    {
        "id": 15,
        "question": "How much is targeted for total share repurchases in FY2024?",
        "expected_answer": "Total share repurchases for FY2024 are targeted at $1.2 billion.",
        "document": "sample_financial_report.pdf",
        "expected_section": "6. Full Year Guidance and Capital Allocation",
        "expected_page": 3,
        "is_answerable": True,
        "keywords": ["$1.2 billion", "share repurchases"]
    },

    # --- DOCX: Cloud Architecture (Questions 16 to 26) ---
    {
        "id": 16,
        "question": "Across which cloud provider regions is the NexaScale Cloud Platform deployed?",
        "expected_answer": "It is deployed active-active across AWS (us-east-1 and us-west-2) and GCP (us-central1).",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "1. Architecture Overview",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["us-east-1", "us-west-2", "us-central1", "GCP", "AWS"]
    },
    {
        "id": 17,
        "question": "What technology is used for container orchestration?",
        "expected_answer": "The platform leverages Kubernetes (EKS and GKE) for stateless workload container orchestration.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "1. Architecture Overview",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["Kubernetes", "EKS", "GKE"]
    },
    {
        "id": 18,
        "question": "What is the CIDR block used for VPCs in us-east-1 and us-west-2?",
        "expected_answer": "us-east-1 uses 10.100.0.0/16 and us-west-2 uses 10.101.0.0/16.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "2. Network Topography and Interconnects",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["10.100.0.0/16", "10.101.0.0/16", "CIDR"]
    },
    {
        "id": 19,
        "question": "What bandwidth is provisioned for AWS Direct Connect dedicated circuits?",
        "expected_answer": "Dual redundant AWS Direct Connect 10 Gbps dedicated circuits are provisioned.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "2. Network Topography and Interconnects",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["10 Gbps", "Direct Connect", "redundant"]
    },
    {
        "id": 20,
        "question": "What database technology powers the transactional persistence tier?",
        "expected_answer": "Amazon Aurora PostgreSQL (Serverless v2) configured in Multi-Region Global Database mode.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "3. Database Tier and Caching Layer",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["Aurora PostgreSQL", "Serverless v2", "Multi-Region"]
    },
    {
        "id": 21,
        "question": "What is the average replication lag of the Aurora read replicas?",
        "expected_answer": "Average replication lag is less than 12 milliseconds.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "3. Database Tier and Caching Layer",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["12 milliseconds", "replication lag"]
    },
    {
        "id": 22,
        "question": "How many shards are configured in the Redis Cluster?",
        "expected_answer": "Redis Cluster (v7.2) is deployed across 6 shards (3 primary, 3 replica).",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "3. Database Tier and Caching Layer",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["6 shards", "3 primary", "Redis Cluster"]
    },
    {
        "id": 23,
        "question": "What is the target p99 API latency specified in the SLO?",
        "expected_answer": "The target API latency (p99) is less than 120 ms over a rolling 30 days window.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "4. Service Level Objectives (SLO) Specification",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["120 ms", "p99", "latency"]
    },
    {
        "id": 24,
        "question": "What is the target platform availability SLA and quarterly error budget?",
        "expected_answer": "Availability SLA is 99.99% with an error budget of 13.14 minutes downtime per calendar quarter.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "4. Service Level Objectives (SLO) Specification",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["99.99%", "13.14 minutes", "availability"]
    },
    {
        "id": 25,
        "question": "What is the Recovery Time Objective (RTO) and Recovery Point Objective (RPO)?",
        "expected_answer": "RTO is less than 30 minutes and RPO is less than 1 minute.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "4. Service Level Objectives (SLO) Specification",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["30 minutes", "1 minute", "RTO", "RPO"]
    },
    {
        "id": 26,
        "question": "How many consecutive synthetic canary probe failures trigger an automated DNS failover?",
        "expected_answer": "If 3 consecutive probes fail (evaluated every 10 seconds), ARC shifts DNS routing.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "5. Disaster Recovery and Failover Automation",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["3 consecutive", "10 seconds", "probe"]
    },

    # --- TXT: Security Policy (Questions 27 to 35) ---
    {
        "id": 27,
        "question": "What hardware authentication is required for corporate and production systems?",
        "expected_answer": "Mandatory MFA using FIDO2-compliant WebAuthn hardware security keys is required.",
        "document": "sample_security_policy.txt",
        "expected_section": "1. Access Control and Identity Management (IAM)",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["FIDO2", "WebAuthn", "hardware security keys"]
    },
    {
        "id": 28,
        "question": "What is the session timeout duration for inactive user console sessions?",
        "expected_answer": "Inactive user console sessions automatically disconnect after 15 minutes of inactivity.",
        "document": "sample_security_policy.txt",
        "expected_section": "1. Access Control and Identity Management (IAM)",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["15 minutes", "inactivity", "session"]
    },
    {
        "id": 29,
        "question": "How long do temporary privileged just-in-time (JIT) access credentials remain valid?",
        "expected_answer": "Just-in-time access approvals expire after 4 hours.",
        "document": "sample_security_policy.txt",
        "expected_section": "1. Access Control and Identity Management (IAM)",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["4 hours", "JIT", "credentials"]
    },
    {
        "id": 30,
        "question": "What TLS version and encryption algorithm are mandated for data in transit and data at rest?",
        "expected_answer": "Mandatory TLS version 1.3 in transit and AES-256 (AES-256-GCM) for data at rest.",
        "document": "sample_security_policy.txt",
        "expected_section": "2. Cryptographic and Encryption Standards",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["TLS 1.3", "AES-256", "encryption"]
    },
    {
        "id": 31,
        "question": "How frequently must master cryptographic encryption keys be rotated?",
        "expected_answer": "Master encryption keys must undergo automated cryptographic rotation every 90 days.",
        "document": "sample_security_policy.txt",
        "expected_section": "2. Cryptographic and Encryption Standards",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["90 days", "rotation", "keys"]
    },
    {
        "id": 32,
        "question": "What is the declaration SLA for a Severity 0 (P0) security incident?",
        "expected_answer": "Maximum declaration SLA is 5 minutes from initial detection for a P0 incident.",
        "document": "sample_security_policy.txt",
        "expected_section": "3. Incident Response and Declaration SLA",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["5 minutes", "Severity 0", "P0"]
    },
    {
        "id": 33,
        "question": "Within what timeframe must a comprehensive post-mortem analysis be published after a P0 incident?",
        "expected_answer": "A comprehensive post-mortem analysis must be published within 72 hours.",
        "document": "sample_security_policy.txt",
        "expected_section": "3. Incident Response and Declaration SLA",
        "expected_page": 1,
        "is_answerable": True,
        "keywords": ["72 hours", "post-mortem", "P0"]
    },
    {
        "id": 34,
        "question": "What is the remediation SLA for Critical vulnerabilities (CVSS 9.0 to 10.0)?",
        "expected_answer": "Remediation SLA is 48 hours from discovery for Critical vulnerabilities.",
        "document": "sample_security_policy.txt",
        "expected_section": "4. Vulnerability Management and Remediation Timelines",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["48 hours", "Critical", "CVSS"]
    },
    {
        "id": 35,
        "question": "Within what timeframe must a lost or stolen employee device be reported to security?",
        "expected_answer": "Any lost or stolen device must be reported to security@acme.corp within 2 hours.",
        "document": "sample_security_policy.txt",
        "expected_section": "5. Employee Endpoint and Device Security",
        "expected_page": 2,
        "is_answerable": True,
        "keywords": ["2 hours", "security@acme.corp", "device"]
    },

    # --- Negative / Out-of-Domain Controls (Questions 36 to 50) ---
    {
        "id": 36,
        "question": "What was the total CEO annual salary and executive bonus in fiscal year 2021?",
        "expected_answer": "Not found in the document.",
        "document": "sample_financial_report.pdf",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 37,
        "question": "What is the company policy regarding reimbursement for home office internet bills?",
        "expected_answer": "Not found in the document.",
        "document": "sample_security_policy.txt",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 38,
        "question": "Does the NexaScale Cloud Platform use Azure Cosmos DB for multi-master global writes?",
        "expected_answer": "Not found in the document.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 39,
        "question": "What was Acme's total revenue in Latin America during Q2 2022?",
        "expected_answer": "Not found in the document.",
        "document": "sample_financial_report.pdf",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 40,
        "question": "How many days of paid parental leave do full-time employees receive?",
        "expected_answer": "Not found in the document.",
        "document": "sample_security_policy.txt",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 41,
        "question": "What is the specific brand and model of the physical top-of-rack data center switches?",
        "expected_answer": "Not found in the document.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 42,
        "question": "What was the stock price of Acme Global Enterprises on December 31, 2023?",
        "expected_answer": "Not found in the document.",
        "document": "sample_financial_report.pdf",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 43,
        "question": "Which catering vendor provides cafeteria lunch services at the company headquarters?",
        "expected_answer": "Not found in the document.",
        "document": "sample_security_policy.txt",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 44,
        "question": "Does the system support Apache Cassandra for cold telemetry archiving?",
        "expected_answer": "Not found in the document.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 45,
        "question": "What is the budget for annual employee gym memberships and wellness stipends?",
        "expected_answer": "Not found in the document.",
        "document": "sample_financial_report.pdf",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 46,
        "question": "What are the rules for reserving visitor parking spots in the subterranean garage?",
        "expected_answer": "Not found in the document.",
        "document": "sample_security_policy.txt",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 47,
        "question": "Is Docker Swarm used as a backup container orchestrator in us-west-2?",
        "expected_answer": "Not found in the document.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 48,
        "question": "What was the total marketing spend on Super Bowl television commercials in 2023?",
        "expected_answer": "Not found in the document.",
        "document": "sample_financial_report.pdf",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 49,
        "question": "What is the mandatory dress code policy for employees working in customer-facing roles?",
        "expected_answer": "Not found in the document.",
        "document": "sample_security_policy.txt",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
    {
        "id": 50,
        "question": "Does the cloud architecture run on Oracle Cloud Infrastructure (OCI) in Frankfurt?",
        "expected_answer": "Not found in the document.",
        "document": "sample_cloud_architecture.docx",
        "expected_section": "None",
        "expected_page": 0,
        "is_answerable": False,
        "keywords": ["Not found in the document."]
    },
]


def compute_token_f1(predicted: str, ground_truth: str) -> float:
    """Computes token-level precision, recall, and F1 score."""
    pred_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", predicted.lower())
    gt_tokens = re.findall(r"\b[a-zA-Z0-9_%$]+\b", ground_truth.lower())

    if not pred_tokens or not gt_tokens:
        return 0.0

    common = set(pred_tokens) & set(gt_tokens)
    if not common:
        return 0.0

    precision = len(common) / len(pred_tokens)
    recall = len(common) / len(gt_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1


def evaluate_pipeline(
    pipeline_mode: str,
    retriever: HybridRetriever,
    generator: Generator,
    dataset: List[Dict[str, Any]],
    use_rerank: bool = True,
    use_verification: bool = True
) -> Dict[str, Any]:
    """Evaluates a specific pipeline configuration on the 50 Q&A benchmark."""
    print(f"\nEvaluating pipeline: {pipeline_mode.upper()} (rerank={use_rerank}, verifier={use_verification})...")
    start_time = time.time()

    total_questions = len(dataset)
    answerable_count = sum(1 for q in dataset if q["is_answerable"])
    unanswerable_count = total_questions - answerable_count

    recall_at_5_hits = 0
    recall_at_20_hits = 0
    correctness_scores = []
    faithfulness_scores = []
    hallucination_count = 0

    detailed_results = []

    for item in dataset:
        qid = item["id"]
        q_text = item["question"]
        is_ans = item["is_answerable"]
        exp_doc = item["document"]
        exp_sec = item["expected_section"]
        exp_page = item["expected_page"]
        keywords = item["keywords"]

        # 1. Retrieval
        retrieval_mode = pipeline_mode
        top_k_fetch = 20 if use_rerank else 5
        rerank_k = 5

        ret_res = retriever.retrieve(
            query=q_text,
            initial_top_k=top_k_fetch,
            rerank_top_k=rerank_k,
            mode=retrieval_mode
        )
        retrieved_chunks = ret_res["chunks"]

        # 2. Check Retrieval Recall (for answerable questions)
        is_hit_at_5 = False
        is_hit_at_20 = False

        if is_ans:
            # Check if relevant chunk is in top 5
            for c in retrieved_chunks[:5]:
                if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page):
                    is_hit_at_5 = True
                    break
            if is_hit_at_5:
                recall_at_5_hits += 1

            # Check if relevant chunk was in candidates (top 20)
            candidate_chunks = ret_res.get("chunks", [])
            for c in candidate_chunks:
                if exp_doc in c.file_name and (exp_sec.lower() in c.section_title.lower() or c.page_number == exp_page):
                    is_hit_at_20 = True
                    break
            if is_hit_at_20 or is_hit_at_5:
                recall_at_20_hits += 1

        # 3. Generation & Verification
        answer, verification = generator.generate_answer(
            query=q_text,
            context_chunks=retrieved_chunks[:5],
            verify=use_verification
        )

        # 4. Measure Correctness & Hallucination
        ans_lower = answer.lower()
        if not is_ans:
            # Unanswerable question: model should refuse with "Not found in the document."
            if "not found in the document" in ans_lower:
                correctness = 1.0
                is_hallucinated = False
                faithfulness = 1.0
            else:
                correctness = 0.0
                is_hallucinated = True
                faithfulness = 0.0
                hallucination_count += 1
        else:
            # Answerable question
            kw_hits = sum(1 for kw in keywords if kw.lower() in ans_lower)
            kw_ratio = kw_hits / len(keywords) if keywords else 0.0
            f1 = compute_token_f1(answer, item["expected_answer"])
            correctness = (kw_ratio * 0.7) + (f1 * 0.3)

            # Faithfulness
            if verification:
                faithfulness = verification.faithfulness_score
                is_hallucinated = verification.has_hallucination
            else:
                # Fallback faithfulness check against retrieved text
                claims = generator.verifier.extract_claims(answer)
                supported = sum(1 for c in claims if any(c.lower() in rc.text.lower() for rc in retrieved_chunks[:5]))
                faithfulness = supported / len(claims) if claims else 1.0
                is_hallucinated = faithfulness < 0.7

            if is_hallucinated:
                hallucination_count += 1

        correctness_scores.append(correctness)
        faithfulness_scores.append(faithfulness)

        detailed_results.append({
            "id": qid,
            "question": q_text,
            "answer": answer,
            "is_answerable": is_ans,
            "retrieval_hit_top5": is_hit_at_5,
            "correctness": round(correctness, 3),
            "faithfulness": round(faithfulness, 3),
            "is_hallucinated": is_hallucinated
        })

    elapsed = time.time() - start_time

    # Calculate aggregate metrics
    recall_at_5 = recall_at_5_hits / answerable_count if answerable_count > 0 else 0.0
    recall_at_20 = recall_at_20_hits / answerable_count if answerable_count > 0 else 0.0
    avg_correctness = sum(correctness_scores) / len(correctness_scores)
    avg_faithfulness = sum(faithfulness_scores) / len(faithfulness_scores)
    hallucination_rate = hallucination_count / total_questions

    metrics = {
        "pipeline": pipeline_mode,
        "use_rerank": use_rerank,
        "use_verification": use_verification,
        "total_questions": total_questions,
        "recall_at_5": round(recall_at_5 * 100, 2),
        "recall_at_20": round(recall_at_20 * 100, 2),
        "answer_correctness": round(avg_correctness * 100, 2),
        "answer_faithfulness": round(avg_faithfulness * 100, 2),
        "hallucination_rate": round(hallucination_rate * 100, 2),
        "elapsed_sec": round(elapsed, 2),
        "details": detailed_results
    }
    return metrics


def run_full_evaluation():
    print("=" * 70)
    print("DOCUMENT QUESTION ANSWERING SYSTEM - 50 Q&A BENCHMARK EVALUATION")
    print("=" * 70)

    # 1. Ingest all 3 documents
    pipeline = DocumentIngestionPipeline(chunk_size=400, chunk_overlap=50)
    files = [
        "data/sample_financial_report.pdf",
        "data/sample_cloud_architecture.docx",
        "data/sample_security_policy.txt"
    ]
    parsed_docs = []
    total_chunks = 0
    for f in files:
        if not os.path.exists(f):
            print(f"File {f} not found! Run create_sample_docs.py first.")
            return
        doc = pipeline.ingest_file(f)
        parsed_docs.append(doc)
        total_chunks += len(doc.chunks)
        print(f"Ingested {doc.file_name}: {doc.total_pages} pages, {len(doc.chunks)} chunks.")

    print(f"\nTotal indexed chunks: {total_chunks}")

    # 2. Setup Retriever & Generator
    retriever = HybridRetriever()
    retriever.index_documents(parsed_docs)
    verifier = AnswerVerifier(cross_encoder=retriever.reranker, threshold=0.55)
    generator = Generator(verifier=verifier, enable_verification=True)

    # 3. Run Evaluations across Configurations
    # Config 1: Baseline 1 - Plain Dense Vector Search (no BM25, no reranker)
    res_dense = evaluate_pipeline(
        pipeline_mode="dense_only",
        retriever=retriever,
        generator=generator,
        dataset=EVALUATION_DATASET,
        use_rerank=False,
        use_verification=False
    )

    # Config 2: Baseline 2 - Plain BM25 Search (no Dense, no reranker)
    res_bm25 = evaluate_pipeline(
        pipeline_mode="bm25_only",
        retriever=retriever,
        generator=generator,
        dataset=EVALUATION_DATASET,
        use_rerank=False,
        use_verification=False
    )

    # Config 3: Baseline 3 - Hybrid Search without Reranker (Dense + BM25, no cross-encoder)
    res_hybrid_no_rerank = evaluate_pipeline(
        pipeline_mode="hybrid_no_rerank",
        retriever=retriever,
        generator=generator,
        dataset=EVALUATION_DATASET,
        use_rerank=False,
        use_verification=False
    )

    # Config 4: Proposed - Full Hybrid (top 20) + Cross-Encoder Reranker (top 5) + Verification
    res_proposed = evaluate_pipeline(
        pipeline_mode="hybrid_rerank",
        retriever=retriever,
        generator=generator,
        dataset=EVALUATION_DATASET,
        use_rerank=True,
        use_verification=True
    )

    all_results = [res_dense, res_bm25, res_hybrid_no_rerank, res_proposed]

    # Save results to json
    report_data = {
        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
        "total_test_pairs": len(EVALUATION_DATASET),
        "configurations": [
            {k: v for k, v in r.items() if k != "details"} for r in all_results
        ]
    }
    with open("evaluation_results.json", "w", encoding="utf-8") as f:
        json.dump(report_data, f, indent=2)

    # Print Formatted Markdown Comparison Table
    print("\n" + "=" * 70)
    print("BENCHMARK COMPARISON RESULTS")
    print("=" * 70)

    header = (
        f"| {'Pipeline Configuration':<40} | {'Recall@5':<10} | {'Recall@20':<11} | "
        f"{'Correctness':<13} | {'Faithfulness':<14} | {'Hallucination Rate':<20} |"
    )
    separator = (
        f"|{'-' * 42}|{'-' * 12}|{'-' * 13}|{'-' * 15}|{'-' * 16}|{'-' * 22}|"
    )
    print(header)
    print(separator)

    labels = [
        "1. Baseline: Plain Dense (FAISS)",
        "2. Baseline: Plain BM25",
        "3. Baseline: Hybrid Search (No Rerank)",
        "4. Proposed: Hybrid + Reranker + Verify"
    ]

    for label, res in zip(labels, all_results):
        row = (
            f"| {label:<40} | "
            f"{res['recall_at_5']:>8.1f}% | "
            f"{res['recall_at_20']:>9.1f}% | "
            f"{res['answer_correctness']:>11.1f}% | "
            f"{res['answer_faithfulness']:>12.1f}% | "
            f"{res['hallucination_rate']:>18.1f}% |"
        )
        print(row)

    print(separator)
    print("\nResults successfully saved to evaluation_results.json")


if __name__ == "__main__":
    run_full_evaluation()