"""RAG gold-set correctness (faithfulness / context precision style).""" import pytest from core.config import ProductionConfig from core.rag import GOLD_DOCUMENTS, answer_from_context, index_gold_documents, retrieve, log_llm_answer GOLD_CASES = [ { "id": "Q1", "query": "maximum lift for brownfield tamping NR/L2/TRK/2500", "must": ["30", "4.2.1", "2500"], "clause": "4.2.1", }, { "id": "Q2", "query": "minimum radius formula NR/L2/TRK/2102 speed cant", "must": ["30.6", "2102"], "clause": "5", }, { "id": "Q3", "query": "What does REPW4220A show NR60 turnout", "must": ["REPW4220A", "NR60"], "clause": "drawing", }, { "id": "Q4", "query": "Steps in complete_sop.txt survey design", "must": ["survey", "1.0"], "clause": "1.0", }, { "id": "Q5", "query": "cant deficiency limit NR/L2/TRK/2049 module 02", "must": ["110", "2049"], "clause": "mod02", }, ] @pytest.fixture(scope="module") def gold_index(): pytest.importorskip("chromadb") pytest.importorskip("sentence_transformers") col = index_gold_documents() if col is None: pytest.skip("RAG stack not installed") return col def _token_overlap(answer: str, context: str) -> float: a = set(answer.lower().split()) c = set(context.lower().split()) if not a: return 0.0 return len(a & c) / len(a) @pytest.mark.parametrize("case", GOLD_CASES, ids=[c["id"] for c in GOLD_CASES]) def test_gold_retrieval_and_faithfulness(gold_index, case): hits = retrieve( case["query"], n_results=ProductionConfig.SEARCH_TOP_K, threshold=0.0, collection_name="rag_gold_test", ) assert hits, f"No retrieval for {case['id']}" best = hits[0] assert best["score"] >= 0.7, f"{case['id']} similarity {best['score']} < 0.7" context = " ".join(h["content"] for h in hits) answer = answer_from_context(case["query"], hits) for token in case["must"]: assert token.lower() in context.lower(), f"{case['id']} context missing {token}" assert case["clause"] in answer or case["clause"] in context assert "hallucinate" not in answer.lower() or hits invented = "Section 99.99" in answer or "clause 9.9.9" in answer.lower() assert not invented faithfulness = _token_overlap(answer, context) assert faithfulness >= 0.5, f"{case['id']} faithfulness {faithfulness:.2f}" relevant = sum(1 for h in hits if any(t.lower() in h["content"].lower() for t in case["must"])) precision = relevant / len(hits) assert precision >= 0.2 log_llm_answer(best["metadata"].get("source", "gold"), case["clause"]) def test_gold_corpus_complete(): assert len(GOLD_DOCUMENTS) == 5 assert ProductionConfig.EMBEDDING_MODEL == "all-MiniLM-L6-v2" assert ProductionConfig.SEARCH_TOP_K == 5 assert ProductionConfig.EMBEDDING_DIM == 384