Spaces:
Runtime error
Runtime error
Download eval_rag.py from progressive-intelligence-lab/docuflow: direct link, hf CLI and curl.
- Browser
- Download file 5.54 kB
-
https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/eval_rag.py
- Command line
-
hf download hf://spaces/progressive-intelligence-lab/docuflow/eval_rag.py
-
curl -L -o eval_rag.py https://huggingface.co/spaces/progressive-intelligence-lab/docuflow/resolve/main/eval_rag.py
5.54 kB
| """ | |
| RAG νμ΄νλΌμΈ νκ° μ€ν¬λ¦½νΈ (RAGnarok-AI + Ollama) | |
| μ¬μ©λ²: | |
| 1. μλ² μ€ν: pipenv run uvicorn app.main:app --reload | |
| 2. νκ° μ€ν: pipenv run python eval_rag.py | |
| """ | |
| import asyncio | |
| import requests as http_requests | |
| from ragnarok_ai.evaluators import ( | |
| precision_at_k, | |
| recall_at_k, | |
| mrr, | |
| ndcg_at_k, | |
| LLMJudge, | |
| ) | |
| # ββ μ€μ ββββββββββββββββββββββββββββββββββββββββββββββ | |
| API_BASE = "http://127.0.0.1:8000" | |
| OLLAMA_BASE = "http://localhost:11434" | |
| # ββ ν μ€νΈ μΌμ΄μ€ μ μ βββββββββββββββββββββββββββββββββ | |
| TEST_CASES = [ | |
| { | |
| "question": "νλμΊνΌνμ μ§λΆμ¨μ?", | |
| "expected": "νλμΊνΌνμ μ§λΆμ¨μ 59.72%μ λλ€.", | |
| }, | |
| { | |
| "question": "νλμΉ΄λμ μ§λΆμ¨μ?", | |
| "expected": "νλμΉ΄λμ μ§λΆμ¨μ 36.24%μ λλ€.", | |
| }, | |
| { | |
| "question": "νλλ‘ν μ μ§λΆμ¨μ?", | |
| "expected": "νλλ‘ν μ μ§λΆμ¨μ 50.50%μ λλ€.", | |
| }, | |
| ] | |
| def call_rag(question: str, document_id: str) -> dict: | |
| """RAG APIλ₯Ό νΈμΆνμ¬ λ΅λ³κ³Ό sourcesλ₯Ό λ°ν""" | |
| resp = http_requests.post( | |
| f"{API_BASE}/qa", | |
| json={"query": question, "document_id": document_id}, | |
| timeout=120, | |
| ) | |
| resp.raise_for_status() | |
| return resp.json() | |
| def get_document_id() -> str: | |
| """μ λ‘λλ λ¬Έμμ document_idλ₯Ό μ λ ₯λ°λλ€.""" | |
| doc_id = input("\nνκ°ν document_idλ₯Ό μ λ ₯νμΈμ: ").strip() | |
| if not doc_id: | |
| print("document_idκ° νμν©λλ€.") | |
| exit(1) | |
| return doc_id | |
| async def main(): | |
| document_id = get_document_id() | |
| print(f"\n{'='*60}") | |
| print(f" RAG νκ° μμ (μ΄ {len(TEST_CASES)}건)") | |
| print(f"{'='*60}\n") | |
| # LLM-as-Judge μ΄κΈ°ν (Ollama llama3.1) | |
| judge = LLMJudge(model="llama3.1", base_url=OLLAMA_BASE) | |
| # κ° ν μ€νΈ μΌμ΄μ€μ λν΄ RAG API νΈμΆ λ° νκ° | |
| all_results = [] | |
| for i, tc in enumerate(TEST_CASES, 1): | |
| question = tc["question"] | |
| expected = tc["expected"] | |
| print(f" [{i}/{len(TEST_CASES)}] μ§λ¬Έ: {question}") | |
| result = call_rag(question, document_id) | |
| answer = result["answer"] | |
| sources = result["sources"] | |
| context = "\n".join(sources) | |
| print(f" λ΅λ³: {answer[:80]}...") | |
| print(f" μμ€: {len(sources)}건") | |
| # Retrieval μ§ν (sourcesλ₯Ό λ¬Έμ IDλ‘ μ¬μ©) | |
| retrieved_ids = [f"doc_{j}" for j in range(len(sources))] | |
| relevant_ids = [f"doc_0"] # μ΅μ 첫 λ²μ§Έ λ¬Έμκ° κ΄λ ¨ μλ€κ³ κ°μ | |
| p = precision_at_k(retrieved_ids, relevant_ids, k=len(sources)) | |
| r = recall_at_k(retrieved_ids, relevant_ids, k=len(sources)) | |
| m = mrr(retrieved_ids, relevant_ids) | |
| # LLM-as-Judge νκ° (faithfulness, relevance, hallucination) | |
| print(f" νκ° μ€... (Ollama llama3.1, μκ°μ΄ 걸릴 μ μμ΅λλ€)") | |
| faithfulness = await judge.evaluate_faithfulness( | |
| context=context, question=question, answer=answer | |
| ) | |
| relevance = await judge.evaluate_relevance(question=question, answer=answer) | |
| hallucination = await judge.detect_hallucination(context=context, answer=answer) | |
| case_result = { | |
| "question": question, | |
| "answer": answer, | |
| "expected": expected, | |
| "retrieval": {"precision": p, "recall": r, "mrr": m}, | |
| "faithfulness": { | |
| "verdict": faithfulness.verdict, | |
| "score": faithfulness.score, | |
| "explanation": faithfulness.explanation, | |
| }, | |
| "relevance": { | |
| "verdict": relevance.verdict, | |
| "score": relevance.score, | |
| "explanation": relevance.explanation, | |
| }, | |
| "hallucination": { | |
| "verdict": hallucination.verdict, | |
| "score": hallucination.score, | |
| "explanation": hallucination.explanation, | |
| }, | |
| } | |
| all_results.append(case_result) | |
| print( | |
| f" Faithfulness: {faithfulness.verdict} ({faithfulness.score:.2f})" | |
| ) | |
| print(f" Relevance: {relevance.verdict} ({relevance.score:.2f})") | |
| print( | |
| f" Hallucination: {hallucination.verdict} ({hallucination.score:.2f})" | |
| ) | |
| print() | |
| # μμ½ μΆλ ₯ | |
| print(f"{'='*60}") | |
| print(f" νκ° μλ£! κ²°κ³Ό μμ½") | |
| print(f"{'='*60}\n") | |
| avg_faith = sum(r["faithfulness"]["score"] for r in all_results) / len(all_results) | |
| avg_rel = sum(r["relevance"]["score"] for r in all_results) / len(all_results) | |
| avg_hall = sum(r["hallucination"]["score"] for r in all_results) / len(all_results) | |
| print(f" νκ· Faithfulness : {avg_faith:.2f}") | |
| print(f" νκ· Relevance : {avg_rel:.2f}") | |
| print(f" νκ· Hallucination: {avg_hall:.2f}") | |
| print() | |
| for i, r in enumerate(all_results, 1): | |
| print(f" [{i}] {r['question']}") | |
| print(f" λ΅λ³: {r['answer'][:60]}...") | |
| print( | |
| f" Faithfulness={r['faithfulness']['score']:.2f} " | |
| f"Relevance={r['relevance']['score']:.2f} " | |
| f"Hallucination={r['hallucination']['score']:.2f}" | |
| ) | |
| print() | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |