from __future__ import annotations import argparse import json import logging import math import os import sys import time from dataclasses import asdict, dataclass, field from typing import Any, Dict, List, Optional os.environ.setdefault("OFFLINE_EVAL", "1") from ..components.orchestration import RAGOrchestrator from ..components.registry import SystemRegistry from .dataset import EVALUATION_DATASET, EvalCase logger = logging.getLogger(__name__) # Ensure utf-8 output for terminal printing if sys.stdout and hasattr(sys.stdout, "reconfigure"): try: sys.stdout.reconfigure(encoding="utf-8") except Exception: pass _DOC_ID_ALIASES: Dict[str, str] = { "7725cf5b21f15913": "doc_1731f7ed91a5404ec0a6", "f27bfd499576a1fc": "doc_d87d482d5ae7960e2e43", "2510521858616bf9": "doc_1f3633afedabd9ee1998", "317e6c70bbb68d1a": "doc_5272452543608d401f61", } for _k, _v in list(_DOC_ID_ALIASES.items()): _DOC_ID_ALIASES[_v] = _k BENCHMARK_CASES: List[EvalCase] = EVALUATION_DATASET @dataclass class BenchmarkReport: total_cases: int = 0 mrr: float = 0.0 recall_at_k: float = 0.0 precision_at_k: float = 0.0 ndcg_at_k: float = 0.0 hit_rate_at_k: float = 0.0 abstention_accuracy: float = 0.0 citation_accuracy: float = 0.0 avg_latency_ms: float = 0.0 p50_latency_ms: float = 0.0 p90_latency_ms: float = 0.0 p95_latency_ms: float = 0.0 p99_latency_ms: float = 0.0 min_latency_ms: float = 0.0 max_latency_ms: float = 0.0 # Routing metrics fast_path_rate: float = 0.0 balanced_path_rate: float = 0.0 deep_path_rate: float = 0.0 escalation_rate: float = 0.0 rerank_rate: float = 0.0 multi_query_rate: float = 0.0 parent_expansion_rate: float = 0.0 abstention_rate: float = 0.0 category_metrics: Dict[str, Dict[str, float]] = field(default_factory=dict) details: List[Dict[str, Any]] = field(default_factory=list) def _compute_percentile(values: List[float], p: float) -> float: """Calculate percentile using linear interpolation.""" if not values: return 0.0 sorted_vals = sorted(values) k = (len(sorted_vals) - 1) * (p / 100.0) idx_floor = int(k) idx_ceil = min(idx_floor + 1, len(sorted_vals) - 1) weight = k - idx_floor return sorted_vals[idx_floor] + weight * (sorted_vals[idx_ceil] - sorted_vals[idx_floor]) def compute_ndcg_at_k(relevance_scores: List[float], k: int = 5) -> float: """ Compute Normalized Discounted Cumulative Gain at rank K. Supports binary (1/0) or graded relevance. """ if not relevance_scores: return 0.0 k_scores = relevance_scores[:k] dcg = sum((rel / math.log2(idx + 1)) for idx, rel in enumerate(k_scores, start=1)) ideal_scores = sorted(relevance_scores, reverse=True)[:k] idcg = sum((rel / math.log2(idx + 1)) for idx, rel in enumerate(ideal_scores, start=1)) if idcg <= 0.0: return 0.0 return dcg / idcg def run_benchmark( cases: Optional[List[EvalCase]] = None, mode: Optional[str] = None, top_k: int = 5, limit: Optional[int] = None, category: Optional[str] = None, orchestrator: Optional[RAGOrchestrator] = None, verbose: bool = False, trace: bool = False, ) -> BenchmarkReport: """ Execute evaluation benchmark against the active RAG orchestrator. Measures retrieval quality (MRR, Recall, Precision), citation accuracy, abstention reliability, latency distribution percentiles, and routing rates. """ selected_cases = cases or EVALUATION_DATASET if category: selected_cases = [c for c in selected_cases if c.category.lower() == category.lower()] if limit is not None and limit > 0: selected_cases = selected_cases[:limit] if not selected_cases: logger.warning("No evaluation cases selected for benchmark run.") return BenchmarkReport() # Reuse singleton instances from SystemRegistry if orchestrator is None: embedder = SystemRegistry.get_embedder() qdrant = SystemRegistry.get_qdrant() bm25 = SystemRegistry.get_bm25() doc_store = SystemRegistry.get_doc_store() reranker = SystemRegistry.get_reranker() if qdrant.count() == 0 and bm25.count() > 0: from ..components.chunking.models import Chunk loaded_chunks = [Chunk.from_dict(c) for c in bm25._all_chunks] vecs = embedder.embed_texts([c.text for c in loaded_chunks]) qdrant.upsert_chunks(loaded_chunks, vecs) orchestrator = RAGOrchestrator( embedder=embedder, qdrant_indexer=qdrant, bm25_indexer=bm25, document_store=doc_store, reranker=reranker, ) report = BenchmarkReport(total_cases=len(selected_cases)) reciprocal_ranks: List[float] = [] recalls: List[float] = [] precisions: List[float] = [] ndcgs: List[float] = [] hit_rates: List[float] = [] abstention_correct = 0 citation_valid_count = 0 answerable_count = 0 latencies: List[float] = [] fast_count = 0 balanced_count = 0 deep_count = 0 escalated_count = 0 reranked_count = 0 multi_query_count = 0 parent_exp_count = 0 abstained_total = 0 cat_recalls: Dict[str, List[float]] = {} cat_precisions: Dict[str, List[float]] = {} cat_mrr: Dict[str, List[float]] = {} cat_ndcg: Dict[str, List[float]] = {} cat_hit_rate: Dict[str, List[float]] = {} cat_abstention: Dict[str, List[int]] = {} cat_citations: Dict[str, List[int]] = {} cat_latencies: Dict[str, List[float]] = {} forced_mode = mode if mode and mode != "adaptive" else None for idx, case in enumerate(selected_cases, start=1): t0 = time.perf_counter() resp = orchestrator.query(case.query, mode=forced_mode) latency = (time.perf_counter() - t0) * 1000 latencies.append(latency) ret_meta = resp.retrieval if ret_meta.execution_mode == "fast": fast_count += 1 elif ret_meta.execution_mode == "deep": deep_count += 1 else: balanced_count += 1 if ret_meta.escalated: escalated_count += 1 if ret_meta.reranked: reranked_count += 1 if ret_meta.multi_query_used: multi_query_count += 1 if ret_meta.parent_expanded: parent_exp_count += 1 if resp.abstained: abstained_total += 1 cat = case.category if cat not in cat_recalls: cat_recalls[cat] = [] cat_precisions[cat] = [] cat_mrr[cat] = [] cat_ndcg[cat] = [] cat_hit_rate[cat] = [] cat_abstention[cat] = [] cat_citations[cat] = [] cat_latencies[cat] = [] cat_latencies[cat].append(latency) case_detail: Dict[str, Any] = { "query_id": case.query_id, "category": case.category, "query": case.query, "expected_answerable": case.answerable, "actual_abstained": resp.abstained, "confidence": resp.confidence, "execution_mode": resp.execution_mode, "stages": ret_meta.cascade_stages_executed, "escalated": ret_meta.escalated, "latency_ms": latency, "sources_count": len(resp.sources), } # Abstention accuracy is_abs_correct = (case.answerable and not resp.abstained) or (not case.answerable and resp.abstained) if is_abs_correct: abstention_correct += 1 cat_abstention[cat].append(1) else: cat_abstention[cat].append(0) # Retrieval accuracy for answerable cases if case.answerable: answerable_count += 1 retrieved_texts = [ (s.get("text", "") or s.get("snippet", "")).lower() for s in resp.sources ] first_rank = None hits_count = 0 rel_scores: List[float] = [] for rank, chunk_text in enumerate(retrieved_texts, start=1): src = resp.sources[rank - 1] if rank - 1 < len(resp.sources) else {} kw_match = any(kw.lower() in chunk_text for kw in case.expected_doc_keywords) src_did = str(src.get("document_id", "")).lower() doc_match = bool( getattr(case, "expected_document_ids", None) and any( str(did).lower() == src_did or _DOC_ID_ALIASES.get(str(did).lower()) == src_did for did in case.expected_document_ids ) ) chk_match = bool( getattr(case, "expected_chunk_ids", None) and any(str(cid).lower() == str(src.get("chunk_id", "")).lower() for cid in case.expected_chunk_ids) ) is_hit = kw_match or doc_match or chk_match rel_scores.append(1.0 if is_hit else 0.0) if is_hit: hits_count += 1 if first_rank is None: first_rank = rank rr = 1.0 / first_rank if first_rank is not None else 0.0 rec = 1.0 if hits_count > 0 else 0.0 prec = hits_count / max(1, len(retrieved_texts)) ndcg = compute_ndcg_at_k(rel_scores, k=top_k) hit_rate = 1.0 if any(r > 0 for r in rel_scores[:top_k]) else 0.0 reciprocal_ranks.append(rr) recalls.append(rec) precisions.append(prec) ndcgs.append(ndcg) hit_rates.append(hit_rate) cat_mrr[cat].append(rr) cat_recalls[cat].append(rec) cat_precisions[cat].append(prec) cat_ndcg[cat].append(ndcg) cat_hit_rate[cat].append(hit_rate) # Citation validation val = resp.citation_validation is_cit_valid = val.get("is_valid", False) and len(val.get("invalid_citations", [])) == 0 if is_cit_valid: citation_valid_count += 1 cat_citations[cat].append(1) else: cat_citations[cat].append(0) report.details.append(case_detail) if trace: print("-" * 65) print(f"QUERY [{case.query_id}] : {case.query}") print(f"ROUTE : {ret_meta.execution_mode.upper()} | STAGES: {ret_meta.cascade_stages_executed}") print(f"ESCALATED : {ret_meta.escalated} | RERANKED: {ret_meta.reranked}") print(f"LATENCY : {latency:.1f}ms | ABSTAINED: {resp.abstained}") if resp.sources: first_src = resp.sources[0] print(f"EVIDENCE #1: {first_src.get('filename')} ({first_src.get('snippet')[:70]}...)") print(f"ANSWER : {resp.answer[:100]}...") elif verbose: status_str = "ABSTAINED" if resp.abstained else "ANSWERED" print(f"[{idx:03d}/{len(selected_cases):03d}] {case.query_id:12s} | {case.category:12s} | {status_str:9s} | {latency:6.1f}ms | {case.query[:45]}") # Aggregate global metrics n = max(1, len(selected_cases)) report.mrr = sum(reciprocal_ranks) / max(1, len(reciprocal_ranks)) report.recall_at_k = sum(recalls) / max(1, len(recalls)) report.precision_at_k = sum(precisions) / max(1, len(precisions)) report.ndcg_at_k = sum(ndcgs) / max(1, len(ndcgs)) report.hit_rate_at_k = sum(hit_rates) / max(1, len(hit_rates)) report.abstention_accuracy = abstention_correct / n report.citation_accuracy = citation_valid_count / max(1, answerable_count) # Routing rates report.fast_path_rate = fast_count / n report.balanced_path_rate = balanced_count / n report.deep_path_rate = deep_count / n report.escalation_rate = escalated_count / n report.rerank_rate = reranked_count / n report.multi_query_rate = multi_query_count / n report.parent_expansion_rate = parent_exp_count / n report.abstention_rate = abstained_total / n # Aggregate latency distribution if latencies: report.avg_latency_ms = sum(latencies) / len(latencies) report.p50_latency_ms = _compute_percentile(latencies, 50.0) report.p90_latency_ms = _compute_percentile(latencies, 90.0) report.p95_latency_ms = _compute_percentile(latencies, 95.0) report.p99_latency_ms = _compute_percentile(latencies, 99.0) report.min_latency_ms = min(latencies) report.max_latency_ms = max(latencies) # Per-category metrics # FIX: unanswerable categories have no recall/precision/mrr data - default to None # (previously incorrectly defaulted to 1.0, inflating category averages) for cat in cat_latencies: cat_rec = sum(cat_recalls[cat]) / max(1, len(cat_recalls[cat])) if cat_recalls[cat] else None cat_prec = sum(cat_precisions[cat]) / max(1, len(cat_precisions[cat])) if cat_precisions[cat] else None cat_m = sum(cat_mrr[cat]) / max(1, len(cat_mrr[cat])) if cat_mrr[cat] else None cat_abs = sum(cat_abstention[cat]) / max(1, len(cat_abstention[cat])) cat_cit = sum(cat_citations[cat]) / max(1, len(cat_citations[cat])) if cat_citations[cat] else None cat_lats = cat_latencies[cat] report.category_metrics[cat] = { "cases": len(cat_lats), "recall_at_k": round(cat_rec, 4) if cat_rec is not None else None, "precision_at_k": round(cat_prec, 4) if cat_prec is not None else None, "mrr": round(cat_m, 4) if cat_m is not None else None, "abstention_accuracy": round(cat_abs, 4), "citation_accuracy": round(cat_cit, 4) if cat_cit is not None else None, "avg_latency_ms": round(sum(cat_lats) / len(cat_lats), 1), "p50_latency_ms": round(_compute_percentile(cat_lats, 50.0), 1), "p95_latency_ms": round(_compute_percentile(cat_lats, 95.0), 1), } return report def print_report(rep: BenchmarkReport, mode_label: str = "adaptive") -> None: """Print clean, structured benchmark results table.""" print("-" * 75) print(f"Nexus - Benchmark Evaluation Report [Mode: {mode_label.upper()}]") print("-" * 75) print(f"Total Test Cases : {rep.total_cases}") print(f"MRR (Mean Reciprocal) : {rep.mrr:.4f}") print(f"Recall@K : {rep.recall_at_k:.4f}") print(f"Precision@K : {rep.precision_at_k:.4f}") print(f"Hit Rate@K : {rep.hit_rate_at_k * 100:.1f}%") print(f"nDCG@K : {rep.ndcg_at_k:.4f}") print(f"Abstention Accuracy : {rep.abstention_accuracy * 100:.1f}%") print(f"Citation Accuracy : {rep.citation_accuracy * 100:.1f}%") print("-" * 75) print("Routing Distribution:") print(f" Fast Path Rate : {rep.fast_path_rate * 100:5.1f}%") print(f" Balanced Path Rate : {rep.balanced_path_rate * 100:5.1f}%") print(f" Deep Path Rate : {rep.deep_path_rate * 100:5.1f}%") print(f" Escalation Rate : {rep.escalation_rate * 100:5.1f}%") print(f" Rerank Rate : {rep.rerank_rate * 100:5.1f}%") print(f" Multi-Query Rate : {rep.multi_query_rate * 100:5.1f}%") print(f" Parent Expansion : {rep.parent_expansion_rate * 100:5.1f}%") print(f" Abstention Rate : {rep.abstention_rate * 100:5.1f}%") print("-" * 75) print("Latency Distribution (ms):") print(f" Min : {rep.min_latency_ms:6.1f} ms") print(f" P50 : {rep.p50_latency_ms:6.1f} ms") print(f" P90 : {rep.p90_latency_ms:6.1f} ms") print(f" P95 : {rep.p95_latency_ms:6.1f} ms") print(f" P99 : {rep.p99_latency_ms:6.1f} ms") print(f" Mean : {rep.avg_latency_ms:6.1f} ms") print(f" Max : {rep.max_latency_ms:6.1f} ms") print("-" * 75) if rep.category_metrics: print(f"{'Category':<15} | {'Cases':<5} | {'Recall':<6} | {'Prec':<6} | {'MRR':<6} | {'Citations':<9} | {'P50 (ms)':<8} | {'P95 (ms)':<8}") print("-" * 75) for cat, m in rep.category_metrics.items(): r = m['recall_at_k'] p = m['precision_at_k'] mrr = m['mrr'] c = m['citation_accuracy'] r_str = f"{r:<6.2f}" if r is not None else "N/A " p_str = f"{p:<6.2f}" if p is not None else "N/A " mrr_str = f"{mrr:<6.2f}" if mrr is not None else "N/A " c_str = f"{c * 100:<8.1f}%" if c is not None else "N/A " print( f"{cat:<15} | {m['cases']:<5} | {r_str} | " f"{p_str} | {mrr_str} | " f"{c_str} | {m['p50_latency_ms']:<8.1f} | {m['p95_latency_ms']:<8.1f}" ) print("-" * 75) def main() -> None: parser = argparse.ArgumentParser(description="Nexus - Production Benchmark Runner") parser.add_argument( "--mode", choices=["fast", "balanced", "deep", "adaptive"], default="adaptive", help="Pipeline execution mode (default: adaptive)", ) parser.add_argument("--limit", type=int, default=None, help="Limit number of evaluation cases") parser.add_argument("--category", type=str, default=None, help="Filter cases by category") parser.add_argument("--top-k", type=int, default=5, help="Top-K retrieval depth") parser.add_argument("--output", type=str, default="metadata/benchmark_production_frozen.json", help="Path to save JSON benchmark output") parser.add_argument("--verbose", action="store_true", help="Print per-case execution logs") parser.add_argument("--trace", action="store_true", help="Print per-query end-to-end trace view") args = parser.parse_args() rep = run_benchmark( mode=args.mode, top_k=args.top_k, limit=args.limit, category=args.category, verbose=args.verbose, trace=args.trace, ) print_report(rep, mode_label=args.mode) if args.output: out_dict = asdict(rep) with open(args.output, "w", encoding="utf-8") as f: json.dump(out_dict, f, indent=2) print(f"Benchmark report saved to: {args.output}") if __name__ == "__main__": main()