Download tests/test_rag_correctness.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 3.01 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/tests/test_rag_correctness.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/tests/test_rag_correctness.py
-
curl -L -o test_rag_correctness.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/tests/test_rag_correctness.py
3.01 kB
| """RAG gold-set correctness (faithfulness / context precision style).""" | |
| import pytest | |
| from core.config import ProductionConfig | |
| from core.rag import GOLD_DOCUMENTS, answer_from_context, index_gold_documents, retrieve, log_llm_answer | |
| GOLD_CASES = [ | |
| { | |
| "id": "Q1", | |
| "query": "maximum lift for brownfield tamping NR/L2/TRK/2500", | |
| "must": ["30", "4.2.1", "2500"], | |
| "clause": "4.2.1", | |
| }, | |
| { | |
| "id": "Q2", | |
| "query": "minimum radius formula NR/L2/TRK/2102 speed cant", | |
| "must": ["30.6", "2102"], | |
| "clause": "5", | |
| }, | |
| { | |
| "id": "Q3", | |
| "query": "What does REPW4220A show NR60 turnout", | |
| "must": ["REPW4220A", "NR60"], | |
| "clause": "drawing", | |
| }, | |
| { | |
| "id": "Q4", | |
| "query": "Steps in complete_sop.txt survey design", | |
| "must": ["survey", "1.0"], | |
| "clause": "1.0", | |
| }, | |
| { | |
| "id": "Q5", | |
| "query": "cant deficiency limit NR/L2/TRK/2049 module 02", | |
| "must": ["110", "2049"], | |
| "clause": "mod02", | |
| }, | |
| ] | |
| def gold_index(): | |
| pytest.importorskip("chromadb") | |
| pytest.importorskip("sentence_transformers") | |
| col = index_gold_documents() | |
| if col is None: | |
| pytest.skip("RAG stack not installed") | |
| return col | |
| def _token_overlap(answer: str, context: str) -> float: | |
| a = set(answer.lower().split()) | |
| c = set(context.lower().split()) | |
| if not a: | |
| return 0.0 | |
| return len(a & c) / len(a) | |
| def test_gold_retrieval_and_faithfulness(gold_index, case): | |
| hits = retrieve( | |
| case["query"], | |
| n_results=ProductionConfig.SEARCH_TOP_K, | |
| threshold=0.0, | |
| collection_name="rag_gold_test", | |
| ) | |
| assert hits, f"No retrieval for {case['id']}" | |
| best = hits[0] | |
| assert best["score"] >= 0.7, f"{case['id']} similarity {best['score']} < 0.7" | |
| context = " ".join(h["content"] for h in hits) | |
| answer = answer_from_context(case["query"], hits) | |
| for token in case["must"]: | |
| assert token.lower() in context.lower(), f"{case['id']} context missing {token}" | |
| assert case["clause"] in answer or case["clause"] in context | |
| assert "hallucinate" not in answer.lower() or hits | |
| invented = "Section 99.99" in answer or "clause 9.9.9" in answer.lower() | |
| assert not invented | |
| faithfulness = _token_overlap(answer, context) | |
| assert faithfulness >= 0.5, f"{case['id']} faithfulness {faithfulness:.2f}" | |
| relevant = sum(1 for h in hits if any(t.lower() in h["content"].lower() for t in case["must"])) | |
| precision = relevant / len(hits) | |
| assert precision >= 0.2 | |
| log_llm_answer(best["metadata"].get("source", "gold"), case["clause"]) | |
| def test_gold_corpus_complete(): | |
| assert len(GOLD_DOCUMENTS) == 5 | |
| assert ProductionConfig.EMBEDDING_MODEL == "all-MiniLM-L6-v2" | |
| assert ProductionConfig.SEARCH_TOP_K == 5 | |
| assert ProductionConfig.EMBEDDING_DIM == 384 | |