flowmarg / tests /test_rag_correctness.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
3.01 kB
"""RAG gold-set correctness (faithfulness / context precision style)."""
import pytest
from core.config import ProductionConfig
from core.rag import GOLD_DOCUMENTS, answer_from_context, index_gold_documents, retrieve, log_llm_answer
GOLD_CASES = [
{
"id": "Q1",
"query": "maximum lift for brownfield tamping NR/L2/TRK/2500",
"must": ["30", "4.2.1", "2500"],
"clause": "4.2.1",
},
{
"id": "Q2",
"query": "minimum radius formula NR/L2/TRK/2102 speed cant",
"must": ["30.6", "2102"],
"clause": "5",
},
{
"id": "Q3",
"query": "What does REPW4220A show NR60 turnout",
"must": ["REPW4220A", "NR60"],
"clause": "drawing",
},
{
"id": "Q4",
"query": "Steps in complete_sop.txt survey design",
"must": ["survey", "1.0"],
"clause": "1.0",
},
{
"id": "Q5",
"query": "cant deficiency limit NR/L2/TRK/2049 module 02",
"must": ["110", "2049"],
"clause": "mod02",
},
]
@pytest.fixture(scope="module")
def gold_index():
pytest.importorskip("chromadb")
pytest.importorskip("sentence_transformers")
col = index_gold_documents()
if col is None:
pytest.skip("RAG stack not installed")
return col
def _token_overlap(answer: str, context: str) -> float:
a = set(answer.lower().split())
c = set(context.lower().split())
if not a:
return 0.0
return len(a & c) / len(a)
@pytest.mark.parametrize("case", GOLD_CASES, ids=[c["id"] for c in GOLD_CASES])
def test_gold_retrieval_and_faithfulness(gold_index, case):
hits = retrieve(
case["query"],
n_results=ProductionConfig.SEARCH_TOP_K,
threshold=0.0,
collection_name="rag_gold_test",
)
assert hits, f"No retrieval for {case['id']}"
best = hits[0]
assert best["score"] >= 0.7, f"{case['id']} similarity {best['score']} < 0.7"
context = " ".join(h["content"] for h in hits)
answer = answer_from_context(case["query"], hits)
for token in case["must"]:
assert token.lower() in context.lower(), f"{case['id']} context missing {token}"
assert case["clause"] in answer or case["clause"] in context
assert "hallucinate" not in answer.lower() or hits
invented = "Section 99.99" in answer or "clause 9.9.9" in answer.lower()
assert not invented
faithfulness = _token_overlap(answer, context)
assert faithfulness >= 0.5, f"{case['id']} faithfulness {faithfulness:.2f}"
relevant = sum(1 for h in hits if any(t.lower() in h["content"].lower() for t in case["must"]))
precision = relevant / len(hits)
assert precision >= 0.2
log_llm_answer(best["metadata"].get("source", "gold"), case["clause"])
def test_gold_corpus_complete():
assert len(GOLD_DOCUMENTS) == 5
assert ProductionConfig.EMBEDDING_MODEL == "all-MiniLM-L6-v2"
assert ProductionConfig.SEARCH_TOP_K == 5
assert ProductionConfig.EMBEDDING_DIM == 384