""" RAG 파이프라인 평가 스크립트 (RAGnarok-AI + Ollama) 사용법: 1. 서버 실행: pipenv run uvicorn app.main:app --reload 2. 평가 실행: pipenv run python eval_rag.py """ import asyncio import requests as http_requests from ragnarok_ai.evaluators import ( precision_at_k, recall_at_k, mrr, ndcg_at_k, LLMJudge, ) # ── 설정 ────────────────────────────────────────────── API_BASE = "http://127.0.0.1:8000" OLLAMA_BASE = "http://localhost:11434" # ── 테스트 케이스 정의 ───────────────────────────────── TEST_CASES = [ { "question": "현대캐피탈의 지분율은?", "expected": "현대캐피탈의 지분율은 59.72%입니다.", }, { "question": "현대카드의 지분율은?", "expected": "현대카드의 지분율은 36.24%입니다.", }, { "question": "현대로템의 지분율은?", "expected": "현대로템의 지분율은 50.50%입니다.", }, ] def call_rag(question: str, document_id: str) -> dict: """RAG API를 호출하여 답변과 sources를 반환""" resp = http_requests.post( f"{API_BASE}/qa", json={"query": question, "document_id": document_id}, timeout=120, ) resp.raise_for_status() return resp.json() def get_document_id() -> str: """업로드된 문서의 document_id를 입력받는다.""" doc_id = input("\n평가할 document_id를 입력하세요: ").strip() if not doc_id: print("document_id가 필요합니다.") exit(1) return doc_id async def main(): document_id = get_document_id() print(f"\n{'='*60}") print(f" RAG 평가 시작 (총 {len(TEST_CASES)}건)") print(f"{'='*60}\n") # LLM-as-Judge 초기화 (Ollama llama3.1) judge = LLMJudge(model="llama3.1", base_url=OLLAMA_BASE) # 각 테스트 케이스에 대해 RAG API 호출 및 평가 all_results = [] for i, tc in enumerate(TEST_CASES, 1): question = tc["question"] expected = tc["expected"] print(f" [{i}/{len(TEST_CASES)}] 질문: {question}") result = call_rag(question, document_id) answer = result["answer"] sources = result["sources"] context = "\n".join(sources) print(f" 답변: {answer[:80]}...") print(f" 소스: {len(sources)}건") # Retrieval 지표 (sources를 문서 ID로 사용) retrieved_ids = [f"doc_{j}" for j in range(len(sources))] relevant_ids = [f"doc_0"] # 최소 첫 번째 문서가 관련 있다고 가정 p = precision_at_k(retrieved_ids, relevant_ids, k=len(sources)) r = recall_at_k(retrieved_ids, relevant_ids, k=len(sources)) m = mrr(retrieved_ids, relevant_ids) # LLM-as-Judge 평가 (faithfulness, relevance, hallucination) print(f" 평가 중... (Ollama llama3.1, 시간이 걸릴 수 있습니다)") faithfulness = await judge.evaluate_faithfulness( context=context, question=question, answer=answer ) relevance = await judge.evaluate_relevance(question=question, answer=answer) hallucination = await judge.detect_hallucination(context=context, answer=answer) case_result = { "question": question, "answer": answer, "expected": expected, "retrieval": {"precision": p, "recall": r, "mrr": m}, "faithfulness": { "verdict": faithfulness.verdict, "score": faithfulness.score, "explanation": faithfulness.explanation, }, "relevance": { "verdict": relevance.verdict, "score": relevance.score, "explanation": relevance.explanation, }, "hallucination": { "verdict": hallucination.verdict, "score": hallucination.score, "explanation": hallucination.explanation, }, } all_results.append(case_result) print( f" Faithfulness: {faithfulness.verdict} ({faithfulness.score:.2f})" ) print(f" Relevance: {relevance.verdict} ({relevance.score:.2f})") print( f" Hallucination: {hallucination.verdict} ({hallucination.score:.2f})" ) print() # 요약 출력 print(f"{'='*60}") print(f" 평가 완료! 결과 요약") print(f"{'='*60}\n") avg_faith = sum(r["faithfulness"]["score"] for r in all_results) / len(all_results) avg_rel = sum(r["relevance"]["score"] for r in all_results) / len(all_results) avg_hall = sum(r["hallucination"]["score"] for r in all_results) / len(all_results) print(f" 평균 Faithfulness : {avg_faith:.2f}") print(f" 평균 Relevance : {avg_rel:.2f}") print(f" 평균 Hallucination: {avg_hall:.2f}") print() for i, r in enumerate(all_results, 1): print(f" [{i}] {r['question']}") print(f" 답변: {r['answer'][:60]}...") print( f" Faithfulness={r['faithfulness']['score']:.2f} " f"Relevance={r['relevance']['score']:.2f} " f"Hallucination={r['hallucination']['score']:.2f}" ) print() if __name__ == "__main__": asyncio.run(main())