docuflow / eval_rag.py
woongmin's picture
[Delta-02] Ollamaμ—μ„œ OpenAI둜 LLM/μž„λ² λ”© μ „ν™˜
0c6fd3a
Raw History Blame Contribute Delete
5.54 kB
"""
RAG νŒŒμ΄ν”„λΌμΈ 평가 슀크립트 (RAGnarok-AI + Ollama)
μ‚¬μš©λ²•:
1. μ„œλ²„ μ‹€ν–‰: pipenv run uvicorn app.main:app --reload
2. 평가 μ‹€ν–‰: pipenv run python eval_rag.py
"""
import asyncio
import requests as http_requests
from ragnarok_ai.evaluators import (
precision_at_k,
recall_at_k,
mrr,
ndcg_at_k,
LLMJudge,
)
# ── μ„€μ • ──────────────────────────────────────────────
API_BASE = "http://127.0.0.1:8000"
OLLAMA_BASE = "http://localhost:11434"
# ── ν…ŒμŠ€νŠΈ μΌ€μ΄μŠ€ μ •μ˜ ─────────────────────────────────
TEST_CASES = [
{
"question": "ν˜„λŒ€μΊν”Όνƒˆμ˜ μ§€λΆ„μœ¨μ€?",
"expected": "ν˜„λŒ€μΊν”Όνƒˆμ˜ μ§€λΆ„μœ¨μ€ 59.72%μž…λ‹ˆλ‹€.",
},
{
"question": "ν˜„λŒ€μΉ΄λ“œμ˜ μ§€λΆ„μœ¨μ€?",
"expected": "ν˜„λŒ€μΉ΄λ“œμ˜ μ§€λΆ„μœ¨μ€ 36.24%μž…λ‹ˆλ‹€.",
},
{
"question": "ν˜„λŒ€λ‘œν…œμ˜ μ§€λΆ„μœ¨μ€?",
"expected": "ν˜„λŒ€λ‘œν…œμ˜ μ§€λΆ„μœ¨μ€ 50.50%μž…λ‹ˆλ‹€.",
},
]
def call_rag(question: str, document_id: str) -> dict:
"""RAG APIλ₯Ό ν˜ΈμΆœν•˜μ—¬ λ‹΅λ³€κ³Ό sourcesλ₯Ό λ°˜ν™˜"""
resp = http_requests.post(
f"{API_BASE}/qa",
json={"query": question, "document_id": document_id},
timeout=120,
)
resp.raise_for_status()
return resp.json()
def get_document_id() -> str:
"""μ—…λ‘œλ“œλœ λ¬Έμ„œμ˜ document_idλ₯Ό μž…λ ₯λ°›λŠ”λ‹€."""
doc_id = input("\n평가할 document_idλ₯Ό μž…λ ₯ν•˜μ„Έμš”: ").strip()
if not doc_id:
print("document_idκ°€ ν•„μš”ν•©λ‹ˆλ‹€.")
exit(1)
return doc_id
async def main():
document_id = get_document_id()
print(f"\n{'='*60}")
print(f" RAG 평가 μ‹œμž‘ (총 {len(TEST_CASES)}건)")
print(f"{'='*60}\n")
# LLM-as-Judge μ΄ˆκΈ°ν™” (Ollama llama3.1)
judge = LLMJudge(model="llama3.1", base_url=OLLAMA_BASE)
# 각 ν…ŒμŠ€νŠΈ μΌ€μ΄μŠ€μ— λŒ€ν•΄ RAG API 호좜 및 평가
all_results = []
for i, tc in enumerate(TEST_CASES, 1):
question = tc["question"]
expected = tc["expected"]
print(f" [{i}/{len(TEST_CASES)}] 질문: {question}")
result = call_rag(question, document_id)
answer = result["answer"]
sources = result["sources"]
context = "\n".join(sources)
print(f" λ‹΅λ³€: {answer[:80]}...")
print(f" μ†ŒμŠ€: {len(sources)}건")
# Retrieval μ§€ν‘œ (sourcesλ₯Ό λ¬Έμ„œ ID둜 μ‚¬μš©)
retrieved_ids = [f"doc_{j}" for j in range(len(sources))]
relevant_ids = [f"doc_0"] # μ΅œμ†Œ 첫 번째 λ¬Έμ„œκ°€ κ΄€λ ¨ μžˆλ‹€κ³  κ°€μ •
p = precision_at_k(retrieved_ids, relevant_ids, k=len(sources))
r = recall_at_k(retrieved_ids, relevant_ids, k=len(sources))
m = mrr(retrieved_ids, relevant_ids)
# LLM-as-Judge 평가 (faithfulness, relevance, hallucination)
print(f" 평가 쀑... (Ollama llama3.1, μ‹œκ°„μ΄ 걸릴 수 μžˆμŠ΅λ‹ˆλ‹€)")
faithfulness = await judge.evaluate_faithfulness(
context=context, question=question, answer=answer
)
relevance = await judge.evaluate_relevance(question=question, answer=answer)
hallucination = await judge.detect_hallucination(context=context, answer=answer)
case_result = {
"question": question,
"answer": answer,
"expected": expected,
"retrieval": {"precision": p, "recall": r, "mrr": m},
"faithfulness": {
"verdict": faithfulness.verdict,
"score": faithfulness.score,
"explanation": faithfulness.explanation,
},
"relevance": {
"verdict": relevance.verdict,
"score": relevance.score,
"explanation": relevance.explanation,
},
"hallucination": {
"verdict": hallucination.verdict,
"score": hallucination.score,
"explanation": hallucination.explanation,
},
}
all_results.append(case_result)
print(
f" Faithfulness: {faithfulness.verdict} ({faithfulness.score:.2f})"
)
print(f" Relevance: {relevance.verdict} ({relevance.score:.2f})")
print(
f" Hallucination: {hallucination.verdict} ({hallucination.score:.2f})"
)
print()
# μš”μ•½ 좜λ ₯
print(f"{'='*60}")
print(f" 평가 μ™„λ£Œ! κ²°κ³Ό μš”μ•½")
print(f"{'='*60}\n")
avg_faith = sum(r["faithfulness"]["score"] for r in all_results) / len(all_results)
avg_rel = sum(r["relevance"]["score"] for r in all_results) / len(all_results)
avg_hall = sum(r["hallucination"]["score"] for r in all_results) / len(all_results)
print(f" 평균 Faithfulness : {avg_faith:.2f}")
print(f" 평균 Relevance : {avg_rel:.2f}")
print(f" 평균 Hallucination: {avg_hall:.2f}")
print()
for i, r in enumerate(all_results, 1):
print(f" [{i}] {r['question']}")
print(f" λ‹΅λ³€: {r['answer'][:60]}...")
print(
f" Faithfulness={r['faithfulness']['score']:.2f} "
f"Relevance={r['relevance']['score']:.2f} "
f"Hallucination={r['hallucination']['score']:.2f}"
)
print()
if __name__ == "__main__":
asyncio.run(main())