rag/evaluation.py

"""
rag/evaluation.py
-----------------
Phase 3 evaluation framework.

Architecture principle: retrieval evaluation and generation evaluation are
STRICTLY SEPARATED. This makes failure attribution unambiguous.

  Stage 1 — Retrieval-only (no LLM calls):
    Recall@k, MRR, Precision@k for all B0–B5 ablation configs.

  Stage 2 — Full pipeline (retrieval + generation + judge):
    Faithfulness (Gemini 1.5 Flash as judge), Answer Relevance.
    Run only on B2 (proposed) and B0 (dense baseline) to manage quota.

Eval dataset (50 items):
  - 35 synthetic: generated by Gemini from corpus documents, with
    verbatim_span used to locate ground-truth chunk(s).
  - 15 adversarial: hardcoded to stress failure modes.
  - Loaded from data/eval/eval_set.json if present; generated otherwise.

Config snapshot: frozen at eval start; written alongside results JSON so
every report is self-contained and reproducible.
"""

import json
import logging
import os
import random
import time
from dataclasses import asdict, dataclass, field
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any

import numpy as np

from rag.bm25_index import BM25Index
from rag.config import RAGConfig
from rag.embeddings import BGEEmbedder
from rag.index import FAISSIndex
from rag.models import ChunkRecord
from rag.retriever import HybridRetriever

logger = logging.getLogger(__name__)

# ── Faithfulness judge: prompt version tracking ───────────────────────────────
# Increment when the prompt text changes so results remain comparable across runs.
# LLM-as-judge caveat: Gemini 1.5 Flash is itself a language model and may exhibit
# systematic biases (e.g., awarding higher faithfulness to longer, confident-sounding
# answers regardless of factual grounding). Scores should be interpreted as
# approximate signal, not ground truth. Use consistent judge model + prompt version
# across all ablation runs to ensure internal comparability.
FAITHFULNESS_JUDGE_PROMPT_VERSION = "v1.0"
FAITHFULNESS_JUDGE_MODEL          = "gemini-1.5-flash"

# ─────────────────────────────────────────────────────────────────────────────
# Data types
# ─────────────────────────────────────────────────────────────────────────────

@dataclass
class EvalItem:
    qid:                str
    question:           str
    reference_answer:   str
    relevant_chunk_ids: list[str]   # empty → unanswerable / annotation pending
    tier:               str         # "synthetic" | "adversarial"
    source_doc:         str         # primary doc_id (empty string if N/A)


@dataclass
class RetrievalMetrics:
    recall_at_k:     float
    mrr:             float
    precision_at_k:  float
    k:               int
    n_queries:       int   # queries with non-empty relevant_chunk_ids


@dataclass
class GenerationMetrics:
    faithfulness:           float
    hallucination_free_rate: float
    answer_relevance:       float
    n_queries:              int


@dataclass
class FailureRecord:
    qid:                  str
    question:             str
    expected_chunk_ids:   list[str]
    retrieved_chunk_ids:  list[str]
    model_answer:         str
    error_type:           str    # "retrieval_miss" | "hallucination" | "both" | "unanswerable_correct" | "unanswerable_hallucinated"
    recall:               float
    faithfulness:         float  # -1.0 if not computed


@dataclass
class LatencyStats:
    mean_ms:   float
    p50_ms:    float
    p95_ms:    float
    n_queries: int


@dataclass
class RunResult:
    config_id:           str
    config_snapshot:     dict[str, Any]
    retrieval_metrics:   RetrievalMetrics
    generation_metrics:  GenerationMetrics | None
    latency:             LatencyStats
    failures:            list[FailureRecord]
    elapsed_seconds:     float


# ─────────────────────────────────────────────────────────────────────────────
# Retrieval metrics (pure functions — no LLM calls)
# ─────────────────────────────────────────────────────────────────────────────

def compute_recall_at_k(retrieved_ids: list[str], relevant_ids: list[str]) -> float:
    if not relevant_ids:
        return 0.0
    return len(set(retrieved_ids) & set(relevant_ids)) / len(relevant_ids)


def compute_mrr(retrieved_ids: list[str], relevant_ids: list[str]) -> float:
    relevant_set = set(relevant_ids)
    for rank, cid in enumerate(retrieved_ids, 1):
        if cid in relevant_set:
            return 1.0 / rank
    return 0.0


def compute_precision_at_k(retrieved_ids: list[str], relevant_ids: list[str]) -> float:
    if not retrieved_ids:
        return 0.0
    return len(set(retrieved_ids) & set(relevant_ids)) / len(retrieved_ids)


# ─────────────────────────────────────────────────────────────────────────────
# Generation metrics
# ─────────────────────────────────────────────────────────────────────────────

def compute_answer_relevance(
    query: str, answer: str, embedder: BGEEmbedder
) -> float:
    """Cosine similarity between query embedding and answer embedding."""
    q_emb = embedder.encode_query(query)          # (1, 768) L2-normalised
    a_emb = embedder.encode_corpus([answer], show_progress=False)  # (1, 768)
    return float(np.dot(q_emb, a_emb.T))


def judge_faithfulness(
    answer: str,
    source_texts: list[str],
    gemini_model: Any,
) -> tuple[float, list[dict]]:
    """
    Use Gemini as a strict claim-attribution judge.

    Returns (faithfulness_score, claims_list).
    faithfulness_score = supported_claims / total_claims.
    On parse failure returns (0.5, []) — conservative middle ground.
    """
    source_block = "\n\n".join(
        f"[Source {i}] {t}" for i, t in enumerate(source_texts, 1)
    )
    prompt = (
        "You are a strict fact-checker. Your ONLY job is claim attribution.\n\n"
        f"SOURCES:\n{source_block}\n\n"
        f"ANSWER:\n{answer}\n\n"
        "For each distinct factual claim in ANSWER, determine if it is:\n"
        "  SUPPORTED: directly stated or unambiguously implied by a source\n"
        "  UNSUPPORTED: relies on knowledge absent from the sources\n\n"
        "Output ONLY valid JSON, no other text:\n"
        '{"claims": [{"text": "...", "supported": true, "source_ref": "[Source N] or null"}], '
        '"faithfulness_score": <float 0-1>}'
    )
    try:
        resp = gemini_model.generate_content(prompt)
        raw  = resp.text.strip()
        # Strip markdown code fences if present
        if raw.startswith("```"):
            raw = "\n".join(raw.split("\n")[1:])
            if raw.endswith("```"):
                raw = raw[:-3]
        data = json.loads(raw)
        return float(data["faithfulness_score"]), data.get("claims", [])
    except Exception as exc:
        logger.warning("Faithfulness judge parse error: %s", exc)
        return 0.5, []


# ─────────────────────────────────────────────────────────────────────────────
# Eval dataset: load or generate
# ─────────────────────────────────────────────────────────────────────────────

def _find_relevant_chunks(
    verbatim_span: str, chunks: list[ChunkRecord], threshold: float = 0.70
) -> list[str]:
    """Locate chunk IDs containing or closely matching verbatim_span."""
    span_lower = verbatim_span.lower().strip()
    matches: list[str] = []
    for chunk in chunks:
        text_lower = chunk.text.lower()
        if span_lower in text_lower:
            matches.append(chunk.chunk_id)
        elif len(span_lower) >= 40:
            # Fuzzy match only for substantial spans (avoids false positives on short strings)
            window = text_lower[: len(span_lower) + 100]
            ratio  = SequenceMatcher(None, span_lower, window).ratio()
            if ratio >= threshold:
                matches.append(chunk.chunk_id)
    return matches


def _hardcoded_adversarial_items() -> list[EvalItem]:
    """
    15 manually crafted adversarial items covering the IndiaFinBench failure modes.
    relevant_chunk_ids is empty for unanswerable queries (correct behaviour =
    "insufficient context"); annotators should fill cross-doc and ref queries.
    """
    return [
        # ── Cross-document synthesis (4) ──────────────────────────────────────
        EvalItem(
            qid="adv_001",
            question="What KYC verification obligations apply to both SEBI-registered portfolio managers and RBI-regulated commercial banks?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_002",
            question="How do SEBI's anti-money laundering requirements for FPIs compare with RBI's AML obligations for NBFCs?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_003",
            question="Which SEBI and RBI circulars jointly govern the treatment of beneficial ownership disclosures?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_004",
            question="What reporting obligations exist under both SEBI and RBI frameworks for entities on UAPA designated lists?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        # ── Exact regulatory references (4) ───────────────────────────────────
        EvalItem(
            qid="adv_005",
            question="What does Section 51A of UAPA 1967 specifically require financial institutions to do?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_006",
            question="What are the conditions specified under Regulation 4(2)(b) of the FPI Regulations 2019?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_007",
            question="Under which master direction does RBI mandate unique identifiers for financial market participants?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_008",
            question="What was the cut-off rate announced for the 91-day Treasury Bill auction in March 2026?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        # ── Unanswerable / out-of-corpus (4) ──────────────────────────────────
        EvalItem(
            qid="adv_009",
            question="What is SEBI's regulatory framework for cryptocurrency derivative instruments?",
            reference_answer="The provided context does not contain sufficient information to answer this question.",
            relevant_chunk_ids=[],   # correct answer = "insufficient context"
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_010",
            question="What are RBI's guidelines on digital lending apps for fintech startups?",
            reference_answer="The provided context does not contain sufficient information to answer this question.",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_011",
            question="What is the minimum net worth requirement for a crypto exchange seeking SEBI registration?",
            reference_answer="The provided context does not contain sufficient information to answer this question.",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_012",
            question="What is RBI's position on issuing a retail Central Bank Digital Currency in India?",
            reference_answer="The provided context does not contain sufficient information to answer this question.",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        # ── Temporal / version conflict (3) ───────────────────────────────────
        EvalItem(
            qid="adv_013",
            question="When is the next Monetary Policy Committee meeting scheduled after April 2026?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_014",
            question="What was the outcome of the 622nd meeting of the RBI Central Board?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
        EvalItem(
            qid="adv_015",
            question="What SEBI circular superseded or amended the most recent FPI KYC guidelines?",
            reference_answer="ANNOTATION REQUIRED",
            relevant_chunk_ids=[],
            tier="adversarial",
            source_doc="",
        ),
    ]


def _generate_synthetic_items(
    docs: list,
    chunks: list[ChunkRecord],
    n: int = 35,
    api_key: str | None = None,
    seed: int = 42,
) -> list[EvalItem]:
    """
    Generate n synthetic QA items via Gemini 1.5 Flash.
    Each item's ground-truth chunk IDs are located via verbatim_span matching.
    Requires GEMINI_API_KEY env var or explicit api_key parameter.
    """
    import google.generativeai as genai  # type: ignore[import]

    key = api_key or os.environ.get("GEMINI_API_KEY")
    if not key:
        raise EnvironmentError("GEMINI_API_KEY not set. Cannot generate synthetic eval set.")

    genai.configure(api_key=key)
    model = genai.GenerativeModel(
        "gemini-1.5-flash",
        generation_config={"temperature": 0.3, "max_output_tokens": 512},
    )

    rng      = random.Random(seed)
    sampled  = rng.sample(docs, min(n, len(docs)))
    items:   list[EvalItem] = []
    failed   = 0

    for i, doc in enumerate(sampled):
        text_excerpt = doc.raw_text[:3000]
        prompt = (
            "Given the following regulatory text, write ONE specific factual question "
            "whose exact answer can be found in one or two consecutive paragraphs.\n\n"
            f"TEXT:\n{text_excerpt}\n\n"
            "Requirements:\n"
            "- The question must be answerable ONLY from this text.\n"
            "- The answer must be precise, not vague.\n"
            "- Include a verbatim_span: the first 60 characters of the exact answer text.\n\n"
            "Output ONLY valid JSON, no other text:\n"
            '{"question": "...", "answer": "...", "verbatim_span": "..."}'
        )
        try:
            resp = model.generate_content(prompt)
            raw  = resp.text.strip()
            if raw.startswith("```"):
                raw = "\n".join(raw.split("\n")[1:]).rstrip("` \n")
            data = json.loads(raw)

            relevant_ids = _find_relevant_chunks(data["verbatim_span"], chunks)
            items.append(EvalItem(
                qid                = f"syn_{i+1:03d}",
                question           = data["question"],
                reference_answer   = data["answer"],
                relevant_chunk_ids = relevant_ids,
                tier               = "synthetic",
                source_doc         = doc.doc_id,
            ))
            # Respect Gemini free-tier rate limits (~2 RPM for flash)
            time.sleep(0.5)

        except Exception as exc:
            logger.warning("Skipping doc %s: %s", doc.doc_id, exc)
            failed += 1

    logger.info(
        "Generated %d synthetic items (%d failed) from %d docs.",
        len(items), failed, len(sampled),
    )
    return items


def load_or_generate_eval_set(
    path: Path,
    docs: list | None = None,
    chunks: list[ChunkRecord] | None = None,
    n_synthetic: int = 35,
    api_key: str | None = None,
    seed: int = 42,
) -> list[EvalItem]:
    """
    Load from path if it exists; otherwise generate and save.
    docs and chunks required only for generation.
    """
    path = Path(path)
    if path.exists():
        raw   = json.loads(path.read_text(encoding="utf-8"))
        items = [EvalItem(**item) for item in raw]
        logger.info("Loaded %d eval items from %s", len(items), path)
        return items

    if docs is None or chunks is None:
        raise ValueError("docs and chunks required to generate eval set.")

    synthetic   = _generate_synthetic_items(docs, chunks, n_synthetic, api_key, seed)
    adversarial = _hardcoded_adversarial_items()
    items       = synthetic + adversarial

    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        json.dumps([asdict(i) for i in items], indent=2, ensure_ascii=False),
        encoding="utf-8",
    )
    logger.info("Saved %d eval items to %s", len(items), path)
    return items


# ─────────────────────────────────────────────────────────────────────────────
# Stage 1: Retrieval evaluation (no LLM)
# ─────────────────────────────────────────────────────────────────────────────

def evaluate_retrieval(
    retriever: HybridRetriever,
    eval_items: list[EvalItem],
    mode: str = "hybrid",
    k: int = 5,
) -> tuple[RetrievalMetrics, LatencyStats, list[FailureRecord]]:
    """
    Evaluate retriever on items that have ground-truth chunk IDs.
    Items with empty relevant_chunk_ids are skipped for metric aggregation
    (they still appear as failures if retrieval returns nothing useful).
    Also measures per-query wall-clock latency (retrieval only, no LLM).
    """
    recalls, mrrs, precisions = [], [], []
    latencies_ms: list[float] = []
    failures: list[FailureRecord] = []

    for item in eval_items:
        t0            = time.perf_counter()
        results       = retriever.retrieve(item.question, mode=mode)
        latencies_ms.append((time.perf_counter() - t0) * 1000)
        retrieved_ids = [r.chunk.chunk_id for r in results]

        if not item.relevant_chunk_ids:
            # Adversarial / unanswerable — skip metric computation
            continue

        recall    = compute_recall_at_k(retrieved_ids, item.relevant_chunk_ids)
        mrr       = compute_mrr(retrieved_ids, item.relevant_chunk_ids)
        precision = compute_precision_at_k(retrieved_ids, item.relevant_chunk_ids)

        recalls.append(recall)
        mrrs.append(mrr)
        precisions.append(precision)

        if recall < 1.0:
            failures.append(FailureRecord(
                qid                  = item.qid,
                question             = item.question,
                expected_chunk_ids   = item.relevant_chunk_ids,
                retrieved_chunk_ids  = retrieved_ids,
                model_answer         = "",   # not computed in retrieval-only pass
                error_type           = "retrieval_miss",
                recall               = recall,
                faithfulness         = -1.0,
            ))

    n = max(len(recalls), 1)
    lat_sorted = sorted(latencies_ms)
    p50 = lat_sorted[len(lat_sorted) // 2] if lat_sorted else 0.0
    p95 = lat_sorted[int(len(lat_sorted) * 0.95)] if lat_sorted else 0.0
    return (
        RetrievalMetrics(
            recall_at_k    = sum(recalls)    / n,
            mrr            = sum(mrrs)       / n,
            precision_at_k = sum(precisions) / n,
            k              = k,
            n_queries      = len(recalls),
        ),
        LatencyStats(
            mean_ms   = sum(latencies_ms) / max(len(latencies_ms), 1),
            p50_ms    = p50,
            p95_ms    = p95,
            n_queries = len(latencies_ms),
        ),
        failures,
    )


# ─────────────────────────────────────────────────────────────────────────────
# Stage 2: Generation evaluation (requires LLM + judge)
# ─────────────────────────────────────────────────────────────────────────────

def evaluate_generation(
    pipeline: Any,   # RAGPipeline
    eval_items: list[EvalItem],
    embedder: BGEEmbedder,
    gemini_model: Any,
    mode: str = "hybrid",
    max_items: int | None = None,
) -> tuple[GenerationMetrics, list[FailureRecord]]:
    """
    Run the full pipeline (retrieval + generation) and score each answer.
    Retrieval and generation failures are recorded separately in FailureRecord.error_type.
    """
    faithfulness_scores: list[float] = []
    hallucination_free:  list[bool]  = []
    relevance_scores:    list[float] = []
    failures:            list[FailureRecord] = []

    items_to_eval = eval_items[:max_items] if max_items else eval_items

    for item in items_to_eval:
        result = pipeline.ask(item.question, mode=mode)

        if "error" in result:
            logger.warning("Pipeline error for %s: %s", item.qid, result["error"])
            continue

        answer         = result["answer"]
        source_texts   = [s["text"] for s in result.get("sources", [])]
        retrieved_ids  = [s["chunk_id"] for s in result.get("sources", [])]

        # Retrieval quality (if ground truth available)
        recall = (
            compute_recall_at_k(retrieved_ids, item.relevant_chunk_ids)
            if item.relevant_chunk_ids else -1.0
        )

        # Faithfulness
        f_score, claims = judge_faithfulness(answer, source_texts, gemini_model)
        faithfulness_scores.append(f_score)
        all_supported = all(c.get("supported", True) for c in claims)
        hallucination_free.append(all_supported)

        # Answer relevance
        rel_score = compute_answer_relevance(item.question, answer, embedder)
        relevance_scores.append(rel_score)

        # Classify failure
        is_retrieval_miss   = bool(item.relevant_chunk_ids) and recall < 0.5
        is_hallucination    = f_score < 0.80
        is_unanswerable     = not item.relevant_chunk_ids

        if is_unanswerable:
            insufficient_phrase = "does not contain sufficient"
            error_type = (
                "unanswerable_correct"
                if insufficient_phrase in answer.lower()
                else "unanswerable_hallucinated"
            )
        elif is_retrieval_miss and is_hallucination:
            error_type = "both"
        elif is_retrieval_miss:
            error_type = "retrieval_miss"
        elif is_hallucination:
            error_type = "hallucination"
        else:
            continue  # no failure

        failures.append(FailureRecord(
            qid                  = item.qid,
            question             = item.question,
            expected_chunk_ids   = item.relevant_chunk_ids,
            retrieved_chunk_ids  = retrieved_ids,
            model_answer         = answer[:400],
            error_type           = error_type,
            recall               = recall,
            faithfulness         = f_score,
        ))

    n = max(len(faithfulness_scores), 1)
    return (
        GenerationMetrics(
            faithfulness            = sum(faithfulness_scores) / n,
            hallucination_free_rate = sum(hallucination_free)  / n,
            answer_relevance        = sum(relevance_scores)    / n,
            n_queries               = len(faithfulness_scores),
        ),
        failures,
    )


# ─────────────────────────────────────────────────────────────────────────────
# Ablation runner
# ─────────────────────────────────────────────────────────────────────────────

# Each config: id, retriever mode, top_k, optional alternative index dir
ABLATION_CONFIGS: list[dict] = [
    {"id": "B0_dense_only",  "mode": "dense",  "top_k": 5,  "index_dir": None, "chunk_chars": None},
    {"id": "B1_bm25_only",   "mode": "bm25",   "top_k": 5,  "index_dir": None, "chunk_chars": None},
    {"id": "B2_hybrid",      "mode": "hybrid", "top_k": 5,  "index_dir": None, "chunk_chars": None},
    {"id": "B3_small_chunks","mode": "hybrid", "top_k": 5,  "index_dir": "rag/index_800",  "chunk_chars": 800},
    {"id": "B4_large_chunks","mode": "hybrid", "top_k": 5,  "index_dir": "rag/index_2400", "chunk_chars": 2400},
    {"id": "B5_higher_k",    "mode": "hybrid", "top_k": 10, "index_dir": None, "chunk_chars": None},
]


def _load_retriever_for_config(
    cfg: dict,
    base_faiss: FAISSIndex,
    base_bm25:  BM25Index,
    embedder:   BGEEmbedder,
    base_cfg:   RAGConfig,
) -> HybridRetriever | None:
    """Build a HybridRetriever for an ablation config. Returns None if index missing."""
    if cfg["index_dir"] is not None:
        alt_dir = Path(cfg["index_dir"])
        if not (alt_dir / "faiss.index").exists():
            logger.warning(
                "Skipping %s: index not found at %s. "
                "Run: python -m rag.scripts.build_index --index-dir %s --chunk-size %s",
                cfg["id"], alt_dir, alt_dir, cfg["chunk_chars"],
            )
            return None
        faiss_idx = FAISSIndex.load(alt_dir, embedder.dim)
        bm25_idx  = BM25Index.load(alt_dir)
    else:
        faiss_idx = base_faiss
        bm25_idx  = base_bm25

    return HybridRetriever(
        faiss_index    = faiss_idx,
        bm25_index     = bm25_idx,
        embedder       = embedder,
        top_k          = cfg["top_k"],
        candidates     = base_cfg.candidates,
        rrf_k          = base_cfg.rrf_k,
        max_per_source = base_cfg.max_per_source,
    )


def run_ablation(
    base_faiss:  FAISSIndex,
    base_bm25:   BM25Index,
    embedder:    BGEEmbedder,
    base_cfg:    RAGConfig,
    eval_items:  list[EvalItem],
    configs:     list[dict] | None = None,
) -> list[RunResult]:
    configs = configs or ABLATION_CONFIGS
    results: list[RunResult] = []

    for cfg in configs:
        retriever = _load_retriever_for_config(
            cfg, base_faiss, base_bm25, embedder, base_cfg
        )
        if retriever is None:
            continue

        config_snapshot = {
            "config_id":    cfg["id"],
            "mode":         cfg["mode"],
            "top_k":        cfg["top_k"],
            "chunk_chars":  cfg["chunk_chars"] or base_cfg.target_chunk_chars,
            "overlap_chars": base_cfg.overlap_chars,
            "embedding_model": base_cfg.embedding_model,
            "rrf_k":        base_cfg.rrf_k,
            "candidates":   base_cfg.candidates,
        }

        logger.info("Running ablation: %s (mode=%s, k=%d)", cfg["id"], cfg["mode"], cfg["top_k"])
        t0 = time.perf_counter()

        metrics, latency, failures = evaluate_retrieval(
            retriever, eval_items, mode=cfg["mode"], k=cfg["top_k"]
        )

        results.append(RunResult(
            config_id          = cfg["id"],
            config_snapshot    = config_snapshot,
            retrieval_metrics  = metrics,
            generation_metrics = None,   # filled in separately for B0 and B2
            latency            = latency,
            failures           = failures,
            elapsed_seconds    = time.perf_counter() - t0,
        ))

    return results


# ─────────────────────────────────────────────────────────────────────────────
# Terminal report
# ─────────────────────────────────────────────────────────────────────────────

def print_terminal_report(
    ablation_results:   list[RunResult],
    gen_results_by_id:  dict[str, tuple[GenerationMetrics, list[FailureRecord]]],
    eval_items:         list[EvalItem],
) -> None:
    n_syn = sum(1 for i in eval_items if i.tier == "synthetic")
    n_adv = sum(1 for i in eval_items if i.tier == "adversarial")

    w = 72
    print()
    print("╔" + "═" * w + "╗")
    print("║" + " IndiaFinBench RAG — Phase 3 Evaluation Report".center(w) + "║")
    print("╚" + "═" * w + "╝")
    print(f"\n  Eval dataset : {len(eval_items)} queries  ({n_syn} synthetic + {n_adv} adversarial)")
    print(f"  Embedding    : BAAI/bge-base-en-v1.5 (768-dim, cosine, IndexFlatIP)")

    # ── Retrieval table ───────────────────────────────────────────────────────
    print()
    print("  RETRIEVAL METRICS")
    hdr = f"  {'Config':<22}  {'Recall@k':>9}  {'MRR':>8}  {'Prec@k':>8}  {'k':>3}  {'p50ms':>7}  {'p95ms':>7}  {'n':>4}"
    print(hdr)
    print("  " + "─" * (len(hdr) - 2))
    for r in ablation_results:
        m   = r.retrieval_metrics
        lat = r.latency
        tag = " ◄ proposed" if r.config_id == "B2_hybrid" else ""
        print(
            f"  {r.config_id:<22}  {m.recall_at_k:>9.4f}  {m.mrr:>8.4f}  "
            f"{m.precision_at_k:>8.4f}  {m.k:>3}  {lat.p50_ms:>7.1f}  {lat.p95_ms:>7.1f}  {m.n_queries:>4}{tag}"
        )

    # ── Thresholds ────────────────────────────────────────────────────────────
    print()
    print("  Targets: Recall@5 ≥ 0.80 | MRR ≥ 0.65 | Precision@5 ≥ 0.50")

    # ── Generation table ──────────────────────────────────────────────────────
    if gen_results_by_id:
        print()
        print(
            f"  GENERATION METRICS  "
            f"(judge: {FAITHFULNESS_JUDGE_MODEL}, prompt {FAITHFULNESS_JUDGE_PROMPT_VERSION})"
        )
        print("  NOTE: LLM-as-judge scores are approximate. Consistent judge model + prompt")
        print("  version across all runs ensures internal comparability, not absolute accuracy.")
        ghdr = f"  {'Config':<22}  {'Faithful':>9}  {'Halluc-free':>12}  {'Ans-Rel':>9}  {'n':>4}"
        print(ghdr)
        print("  " + "─" * (len(ghdr) - 2))
        for cfg_id, (gm, _) in gen_results_by_id.items():
            print(
                f"  {cfg_id:<22}  {gm.faithfulness:>9.4f}  "
                f"{gm.hallucination_free_rate:>12.4f}  {gm.answer_relevance:>9.4f}  {gm.n_queries:>4}"
            )
        print()
        print("  Targets: Faithfulness ≥ 0.85 | Halluc-free ≥ 0.90 | Ans-Rel ≥ 0.75")

    # ── Failure analysis ──────────────────────────────────────────────────────
    print()
    print("  FAILURE ANALYSIS (B2 Hybrid)")
    b2 = next((r for r in ablation_results if r.config_id == "B2_hybrid"), None)
    all_failures: list[FailureRecord] = list(b2.failures) if b2 else []
    if "B2_hybrid" in gen_results_by_id:
        all_failures.extend(gen_results_by_id["B2_hybrid"][1])

    if not all_failures:
        print("  No failures recorded.")
    else:
        from collections import Counter
        error_counts = Counter(f.error_type for f in all_failures)
        for etype, count in error_counts.most_common():
            pct = count / len(eval_items) * 100
            print(f"  [{etype:<28}]  {count:3d} queries  ({pct:.1f}%)")

        print()
        print("  Top 2 failure examples:")
        shown = 0
        for f in all_failures[:10]:
            if shown >= 2:
                break
            if f.error_type in ("retrieval_miss", "hallucination", "both"):
                print(f"    [{f.error_type}]  {f.qid}: {f.question[:70]}")
                if f.expected_chunk_ids:
                    print(f"      expected: {f.expected_chunk_ids[:2]}")
                    print(f"      got:      {f.retrieved_chunk_ids[:2]}")
                shown += 1

    print()


# ─────────────────────────────────────────────────────────────────────────────
# Persistence
# ─────────────────────────────────────────────────────────────────────────────

def save_report(
    ablation_results:  list[RunResult],
    gen_results:       dict[str, tuple[GenerationMetrics, list[FailureRecord]]],
    config_snapshot:   dict,
    path:              Path,
) -> None:
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)

    serialised_ablation = []
    for r in ablation_results:
        d = {
            "config_id":          r.config_id,
            "config_snapshot":    r.config_snapshot,
            "elapsed_seconds":    round(r.elapsed_seconds, 2),
            "retrieval_metrics":  asdict(r.retrieval_metrics),
            "generation_metrics": None,
            "failures":           [asdict(f) for f in r.failures],
        }
        if r.config_id in gen_results:
            gm, _ = gen_results[r.config_id]
            d["generation_metrics"] = asdict(gm)
        serialised_ablation.append(d)

    report = {
        "system_config":    config_snapshot,
        "judge_meta": {
            "model":          FAITHFULNESS_JUDGE_MODEL,
            "prompt_version": FAITHFULNESS_JUDGE_PROMPT_VERSION,
            "bias_note":      "LLM judge scores approximate; compare only within same version.",
        },
        "ablation_results": serialised_ablation,
    }
    path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
    logger.info("Report saved to %s", path)