SAP-ERP-AI-Agent / src /evaluation /eval_worker_b.py
daisysooyeon's picture
deploy: SAP ERP AI Agent (HF Spaces docker)
50efdc6
Raw History Blame Contribute Delete
21.5 kB
"""
src/evaluation/eval_worker_b.py
RAG ํ’ˆ์งˆ ํ‰๊ฐ€ โ€” ์ž์ฒด ๊ตฌํ˜„ ๋ฉ”ํŠธ๋ฆญ (RAGAS ๋Œ€์‹ )
ํ‰๊ฐ€ ๋ฉ”ํŠธ๋ฆญ:
Hit Rate : top-k ์•ˆ์— ์ •๋‹ต source_chunk๊ฐ€ ํฌํ•จ๋œ ๋น„์œจ (๋ชฉํ‘œ โ‰ฅ 85%)
NDCG@3 : ์ƒ์œ„ 3๊ฐœ ์ฒญํฌ ์ˆœ์œ„ ํ’ˆ์งˆ, ์ •๋‹ต์ด ๋†’์„์ˆ˜๋ก ๋†’์€ ์ ์ˆ˜ (๋ชฉํ‘œ โ‰ฅ 0.75)
MRR : Mean Reciprocal Rank, ์ฒซ ์ •๋‹ต ๋ฌธ์„œ ์—ญ์ˆœ์œ„ ํ‰๊ท  (๋ชฉํ‘œ โ‰ฅ 0.70)
Context Recall: top-k ์ปจํ…์ŠคํŠธ๊ฐ€ ์ •๋‹ต ์ƒ์„ฑ์— ํ•„์š”ํ•œ ์ •๋ณด๋ฅผ ํฌํ•จํ•˜๋Š”์ง€ (๋ชฉํ‘œ โ‰ฅ 0.90)
Faithfulness : ์ƒ์„ฑ ๋‹ต๋ณ€์ด retrieved ์ปจํ…์ŠคํŠธ์— ๊ทผ๊ฑฐํ•˜๋Š”์ง€ LLM judge (๋ชฉํ‘œ โ‰ฅ 0.85)
์ž…๋ ฅ ๋ฐ์ดํ„ฐ์…‹ ํ˜•์‹ (router_test_cases_gen.json, QA_ONLY + BOTH ํ•„ํ„ฐ):
{
"id": "r_001",
"user_input": "customer email text",
"label": "QA_ONLY",
"rag_evidence": "exact chunk text (str or list[str] for multi-chunk)",
"qa_question": "extracted knowledge question"
}
CLI ์‚ฌ์šฉ๋ฒ•:
python -m src.evaluation.eval_worker_b # ์ „์ฒด ํ‰๊ฐ€
python -m src.evaluation.eval_worker_b --skip-llm-judge # retrieval ๋ฉ”ํŠธ๋ฆญ๋งŒ (๋น ๋ฆ„)
python -m src.evaluation.eval_worker_b --report reports/worker_b_eval_result.json
"""
from __future__ import annotations
import logging
import math
import os
import time
from dotenv import load_dotenv
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI
from src.config import get_config
from src.rag.retriever import build_hybrid_retriever
from src.rag.reranker import rerank
def _serialize_docs(docs: list) -> list[dict]:
return [
{
"content": doc.page_content,
"source": doc.metadata.get("source", ""),
"chunk_id": doc.metadata.get("chunk_id", ""),
"page": doc.metadata.get("page", -1),
}
for doc in docs
]
load_dotenv()
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# LLM for judge (gemini-flash)
# ---------------------------------------------------------------------------
_judge_llm: ChatOpenAI | None = None
def _get_judge_llm() -> ChatOpenAI:
global _judge_llm
if _judge_llm is not None:
return _judge_llm
cfg = get_config()
api_key = os.getenv("OPENROUTER_API_KEY", "")
judge_cfg = cfg.models.eval_judge
_judge_llm = ChatOpenAI(
model=judge_cfg.name,
temperature=0.0,
openai_api_key=api_key,
openai_api_base=cfg.openrouter.base_url,
default_headers={
"HTTP-Referer": "https://github.com/daisysooyeon/SAP-ERP-AI-Agent",
"X-Title": "SAP-ERP-AI-Agent RAG-Eval",
},
)
return _judge_llm
# ---------------------------------------------------------------------------
# ๋งค์นญ ํ—ฌํผ
# ---------------------------------------------------------------------------
def _chunk_in_docs(source_chunks: str | list[str], docs: list[dict], threshold: float = 0.5) -> int | None:
"""
source_chunks(๋ฌธ์ž์—ด ๋˜๋Š” ๋ฆฌ์ŠคํŠธ) ์ค‘ ํ•˜๋‚˜๋ผ๋„ retrieved docs ์ค‘ ์–ด๋А ์œ„์น˜(0-indexed)์— ์žˆ๋Š”์ง€ ๋ฐ˜ํ™˜.
์ •ํ™•ํ•œ substring์ด ์—†์œผ๋ฉด 50% ์ด์ƒ ํ† ํฐ ๊ฒน์นจ์œผ๋กœ ํŒ๋‹จ.
์—ฌ๋Ÿฌ ๊ฐœ ๋งค์นญ๋  ๊ฒฝ์šฐ ๊ฐ€์žฅ ๋†’์€ ์ˆœ์œ„(์ž‘์€ ์ธ๋ฑ์Šค) ๋ฐ˜ํ™˜.
์—†์œผ๋ฉด None ๋ฐ˜ํ™˜.
"""
if not source_chunks:
return None
if isinstance(source_chunks, str):
chunks_to_check = [source_chunks]
else:
chunks_to_check = source_chunks
best_pos = None
for chunk in chunks_to_check:
gt_tokens = set(chunk.lower().split())
for i, doc in enumerate(docs):
content = doc["content"]
# 1์ˆœ์œ„: substring
if chunk[:80].strip() in content:
if best_pos is None or i < best_pos:
best_pos = i
break
# 2์ˆœ์œ„: ํ† ํฐ overlap
doc_tokens = set(content.lower().split())
if gt_tokens and doc_tokens:
overlap = len(gt_tokens & doc_tokens) / len(gt_tokens)
if overlap >= threshold:
if best_pos is None or i < best_pos:
best_pos = i
break
return best_pos
def _match_by_id(evidence_ids: str | list[str], docs: list[dict]) -> int | None:
"""retrieved docs ์ค‘ metadata chunk_id ๊ฐ€ ์ •๋‹ต id ์ง‘ํ•ฉ์— ์†ํ•œ ์ฒซ ์œ„์น˜(0-indexed)๋ฅผ ๋ฐ˜ํ™˜."""
if not evidence_ids:
return None
id_set = {evidence_ids} if isinstance(evidence_ids, str) else set(evidence_ids)
for i, doc in enumerate(docs):
if doc.get("chunk_id") in id_set:
return i
return None
def _locate_evidence(case: dict, docs: list[dict], threshold: float = 0.5) -> int | None:
"""
์ •๋‹ต ์ฒญํฌ๊ฐ€ docs ์ค‘ ์–ด๋А ์œ„์น˜์— ์žˆ๋Š”์ง€ ๋ฐ˜ํ™˜.
case ์— ์ •๋‹ต chunk_id(evidence_chunk_id)๊ฐ€ ์žˆ์œผ๋ฉด id ์ •ํ™• ๋งค์นญ,
์—†์œผ๋ฉด rag_evidence ํ…์ŠคํŠธ fuzzy ๋งค์นญ์œผ๋กœ ํด๋ฐฑ. (build_eval_ids.py ๋กœ id ๋ถ€์—ฌ ๊ฐ€๋Šฅ)
"""
evidence_ids = case.get("evidence_chunk_id")
if evidence_ids:
return _match_by_id(evidence_ids, docs)
return _chunk_in_docs(case.get("rag_evidence"), docs, threshold)
# ---------------------------------------------------------------------------
# Retrieval ๋ฉ”ํŠธ๋ฆญ (Hit Rate, NDCG@3, MRR)
# ---------------------------------------------------------------------------
def compute_hit_rate(cases: list[dict], all_retrieved: list[list[dict]]) -> float:
"""top-k ์•ˆ์— ์ •๋‹ต ์ฒญํฌ๊ฐ€ ํฌํ•จ๋œ ๋น„์œจ"""
hits = sum(
1 for case, docs in zip(cases, all_retrieved)
if _locate_evidence(case, docs) is not None
)
return hits / len(cases) if cases else 0.0
def compute_ndcg_at_3(cases: list[dict], all_retrieved: list[list[dict]]) -> float:
"""
NDCG@3: ์ •๋‹ต์ด top-3 ์•ˆ์—์„œ ๋ช‡ ๋ฒˆ์งธ์— ์žˆ๋Š”์ง€์— ๋”ฐ๋ผ ์ ์ˆ˜ ์ฐจ๋“ฑ
IDCG = 1/log2(2) = 1.0 (์ •๋‹ต์ด 1์œ„์ธ ์ด์ƒ์  ์ผ€์ด์Šค)
"""
ndcg_scores = []
for case, docs in zip(cases, all_retrieved):
top3 = docs[:3]
pos = _locate_evidence(case, top3)
if pos is None:
ndcg_scores.append(0.0)
else:
dcg = 1.0 / math.log2(pos + 2) # pos๋Š” 0-indexed, log2(rank+1)
idcg = 1.0 / math.log2(2) # ์ด์ƒ์ : rank=1
ndcg_scores.append(dcg / idcg)
return sum(ndcg_scores) / len(ndcg_scores) if ndcg_scores else 0.0
def compute_mrr(cases: list[dict], all_retrieved: list[list[dict]]) -> float:
"""Mean Reciprocal Rank: ์ฒซ ๋ฒˆ์งธ ์ •๋‹ต ๋ฌธ์„œ์˜ ์—ญ์ˆœ์œ„ ํ‰๊ท """
rr_scores = []
for case, docs in zip(cases, all_retrieved):
pos = _locate_evidence(case, docs)
if pos is None:
rr_scores.append(0.0)
else:
rr_scores.append(1.0 / (pos + 1)) # 0-indexed โ†’ 1-indexed
return sum(rr_scores) / len(rr_scores) if rr_scores else 0.0
# ---------------------------------------------------------------------------
# LLM Judge ๋ฉ”ํŠธ๋ฆญ (Context Recall, Faithfulness)
# ---------------------------------------------------------------------------
_CONTEXT_RECALL_PROMPT = ChatPromptTemplate.from_messages([
("system", (
"You are an evaluation assistant. "
"Given a question and a context, "
"determine whether the context contains enough information to answer the question correctly.\n"
"Respond with ONLY '1' (yes, context is sufficient) or '0' (no, context is insufficient)."
)),
("human", (
"Question: {question}\n\n"
"Context:\n{context}\n\n"
"Does the context contain sufficient information? (1 or 0):"
)),
])
_FAITHFULNESS_PROMPT = ChatPromptTemplate.from_messages([
("system", (
"You are an evaluation assistant. "
"Given an answer and the context it was generated from, "
"determine whether the answer is fully supported by the context (no hallucination).\n"
"Respond with ONLY '1' (yes, answer is faithful) or '0' (no, answer contains hallucination)."
)),
("human", (
"Context:\n{context}\n\n"
"Answer: {rag_answer}\n\n"
"Is the answer fully supported by the context? (1 or 0):"
)),
])
def _llm_judge(
prompt_template: ChatPromptTemplate,
variables: dict,
metric: str = "unknown",
delay: float = 0.5,
) -> tuple[int, bool]:
"""LLM judge ํ˜ธ์ถœ โ†’ (score, is_invalid) ๋ฐ˜ํ™˜. score๋Š” 0 ๋˜๋Š” 1."""
try:
llm = _get_judge_llm()
chain = prompt_template | llm
result = chain.invoke(variables)
# qwen3-8b๋Š” <think>...</think> ๋ธ”๋ก ๋˜๋Š” ์ค„๋ฐ”๊ฟˆ ํ›„ ์ˆซ์ž๋ฅผ ์ถœ๋ ฅํ•˜๋Š” ๊ฒฝ์šฐ๊ฐ€ ์žˆ์Œ
import re as _re
raw_val = result.content.strip()
# <think>...</think> ์ œ๊ฑฐ
val = _re.sub(r"<think>.*?</think>", "", raw_val, flags=_re.DOTALL).strip()
# ์ฒซ ๋ฒˆ์งธ ๋“ฑ์žฅํ•˜๋Š” 0 ๋˜๋Š” 1 ์ถ”์ถœ
m = _re.search(r"[01]", val)
val = m.group() if m else val
time.sleep(delay)
if val.startswith("1"):
return 1, False
elif val.startswith("0"):
return 0, False
else:
logger.warning(
"[eval_rag] LLM judge [%s] unexpected response: %r",
metric, val[:80],
)
return 0, True # conservative: unexpected ์‘๋‹ต์€ 0์  ์ฒ˜๋ฆฌ
except Exception as e:
logger.warning("[eval_rag] LLM judge [%s] failed: %s", metric, e)
return 0, False
def compute_context_recall(
cases: list[dict],
all_retrieved: list[list[dict]],
) -> tuple[float, int]:
"""Context Recall: retrieved context๊ฐ€ ์ •๋‹ต ์ƒ์„ฑ์— ์ถฉ๋ถ„ํ•œ์ง€ LLM judge (๋ชฉํ‘œ >= 0.90)"""
scores = []
invalid = 0
for case, docs in zip(cases, all_retrieved):
context = "\n\n".join(d["content"] for d in docs)
score, is_invalid = _llm_judge(
_CONTEXT_RECALL_PROMPT,
{"question": case.get("qa_question") or case.get("question"), "context": context},
metric="context_recall",
)
if is_invalid:
invalid += 1
scores.append(score)
logger.debug("[eval_rag] Context Recall [%s]: %d", case.get("id"), score)
return (sum(scores) / len(scores) if scores else 0.0), invalid
def compute_faithfulness(
cases: list[dict],
all_retrieved: list[list[dict]],
rag_answers: list[str],
) -> tuple[float, int]:
"""Faithfulness: ์ƒ์„ฑ ๋‹ต๋ณ€์ด ์ปจํ…์ŠคํŠธ์— ๊ทผ๊ฑฐํ•˜๋Š”์ง€ LLM judge (๋ชฉํ‘œ >= 0.85)"""
scores = []
invalid = 0
for case, docs, answer in zip(cases, all_retrieved, rag_answers):
if not answer or not docs:
scores.append(0)
continue
context = "\n\n".join(d["content"] for d in docs)
score, is_invalid = _llm_judge(
_FAITHFULNESS_PROMPT,
{"context": context, "rag_answer": answer},
metric="faithfulness",
)
if is_invalid:
invalid += 1
scores.append(score)
logger.debug("[eval_rag] Faithfulness [%s]: %d", case.get("id"), score)
return (sum(scores) / len(scores) if scores else 0.0), invalid
# ---------------------------------------------------------------------------
# ๋ฉ”์ธ ํ‰๊ฐ€ ํ•จ์ˆ˜
# ---------------------------------------------------------------------------
def run_rag_evaluation(
test_cases: list[dict],
rag_answers: list[str] | None = None,
all_retrieved: list[list[dict]] | None = None,
skip_llm_judge: bool = False,
) -> dict:
"""
์ „์ฒด RAG ํ‰๊ฐ€ ์‹คํ–‰.
Args:
test_cases: rag_test_cases.json ๋กœ๋“œ ๊ฒฐ๊ณผ
rag_answers: ๊ฐ ์ผ€์ด์Šค์— ๋Œ€ํ•œ ์ƒ์„ฑ ๋‹ต๋ณ€ (์—†์œผ๋ฉด retrieval ๋ฉ”ํŠธ๋ฆญ๋งŒ ๊ณ„์‚ฐ)
all_retrieved: ๊ฐ ์ผ€์ด์Šค์˜ retrieved docs ๋ชฉ๋ก
skip_llm_judge: True์ด๋ฉด Context Recall / Faithfulness ์Šคํ‚ต
Returns:
{
"hit_rate": float,
"ndcg_at_3": float,
"mrr": float,
"context_recall": float | None,
"faithfulness": float | None,
"n_cases": int,
}
"""
n = len(test_cases)
logger.info("[eval_rag] Evaluating %d cases ...", n)
hit_rate = compute_hit_rate(test_cases, all_retrieved)
ndcg_at_3 = compute_ndcg_at_3(test_cases, all_retrieved)
mrr = compute_mrr(test_cases, all_retrieved)
context_recall: float | None = None
faithfulness: float | None = None
invalid_cr: int = 0
invalid_fa: int = 0
if not skip_llm_judge and rag_answers:
logger.info("[eval_rag] Running LLM judges (Context Recall + Faithfulness) ...")
context_recall, invalid_cr = compute_context_recall(test_cases, all_retrieved)
faithfulness, invalid_fa = compute_faithfulness(test_cases, all_retrieved, rag_answers)
if invalid_cr > 0:
logger.warning("[eval_rag] Context Recall judge invalid: %d / %d", invalid_cr, n)
if invalid_fa > 0:
logger.warning("[eval_rag] Faithfulness judge invalid: %d / %d", invalid_fa, n)
return {
"hit_rate": round(hit_rate, 4),
"ndcg_at_3": round(ndcg_at_3, 4),
"mrr": round(mrr, 4),
"context_recall": round(context_recall, 4) if context_recall is not None else None,
"faithfulness": round(faithfulness, 4) if faithfulness is not None else None,
"n_cases": n,
"judge_invalid_context_recall": invalid_cr,
"judge_invalid_faithfulness": invalid_fa,
}
# ---------------------------------------------------------------------------
# CLI ์ง„์ž…์  (python -m src.evaluation.eval_rag ๋˜๋Š” run_eval_rag.sh ์—์„œ ํ˜ธ์ถœ)
# ---------------------------------------------------------------------------
TARGETS = {
"hit_rate": 0.85,
"ndcg_at_3": 0.75,
"mrr": 0.70,
"context_recall": 0.90,
"faithfulness": 0.85,
}
METRIC_LABELS = {
"hit_rate": "Hit Rate ",
"ndcg_at_3": "NDCG@3 ",
"mrr": "MRR ",
"context_recall": "Ctx Recall ",
"faithfulness": "Faithfulness",
}
def _main():
import argparse
import json
import sys
from pathlib import Path
# Windows cp949 ํ„ฐ๋ฏธ๋„ ์ธ์ฝ”๋”ฉ ๋Œ€์‘
try:
sys.stdout.reconfigure(encoding="utf-8")
except Exception:
pass
from src.logging_config import setup_logging
setup_logging()
parser = argparse.ArgumentParser(description="RAG Quality Evaluation")
parser.add_argument("--report", default="reports/worker_b_eval_result.json", help="์ถœ๋ ฅ ๋ฆฌํฌํŠธ ๊ฒฝ๋กœ")
parser.add_argument("--dataset", default="data/eval/router_test_cases_gen.json")
parser.add_argument("--skip-llm-judge", action="store_true", help="LLM judge ๋ฉ”ํŠธ๋ฆญ ์Šคํ‚ต")
parser.add_argument(
"--no-preprocess", action="store_true",
help="Skip email preprocessing (cleaned_body extraction) and extract queries from raw user_input "
"(default: use preprocessed cleaned_body, matching production worker_b_node)",
)
args = parser.parse_args()
# ๋ฐ์ดํ„ฐ์…‹ ๋กœ๋“œ ๋ฐ ํ•„ํ„ฐ๋ง (QA_ONLY, BOTH)
dataset_path = Path(args.dataset)
all_cases: list[dict] = json.loads(dataset_path.read_text(encoding="utf-8"))
test_cases = [c for c in all_cases if c.get("label") in ("QA_ONLY", "BOTH")]
logger.info("Loaded %d test cases for Worker B from %s", len(test_cases), dataset_path)
# ๋ฆฌํŠธ๋ฆฌ๋ฒ„ / ๋ฆฌ๋žญ์ปค ์ดˆ๊ธฐํ™”
logger.info("Initializing retriever (bge-m3 loading may take a moment) ...")
retriever = build_hybrid_retriever()
cfg = get_config()
top_n = cfg.rag.top_k_rerank
# ์ด๋ฉ”์ผ ์ „์ฒ˜๋ฆฌ โ€” ํ”„๋กœ๋•์…˜(worker_b_node)์€ ์›๋ณธ ๋Œ€์‹  cleaned_body๋กœ ์ฟผ๋ฆฌ๋ฅผ ์ถ”์ถœํ•œ๋‹ค.
# eval๋„ ๋™์ผ ์กฐ๊ฑด์„ ๋งž์ถ”๊ธฐ ์œ„ํ•ด ์ผ€์ด์Šค๋ณ„๋กœ ์ „์ฒ˜๋ฆฌํ•ด cleaned_body๋ฅผ ์ฟผ๋ฆฌ ์ถ”์ถœ ์ž…๋ ฅ์œผ๋กœ ์“ด๋‹ค.
# (--no-preprocess ๋ฉด ๊ธฐ์กด์ฒ˜๋Ÿผ ์›๋ณธ user_input ์‚ฌ์šฉ)
if not args.no_preprocess:
from src.preprocess.email_preprocessor import preprocess_email
logger.info("Email preprocessing ENABLED (using cleaned_body for query extraction, matching production)")
else:
logger.info("Email preprocessing DISABLED (using raw user_input)")
# ๊ฐ ์ผ€์ด์Šค ์‹คํ–‰
results = []
all_retrieved: list[list[dict]] = []
rag_answers: list[str] = []
print(f"\n{'='*60}")
print(f" RAG Evaluation - {len(test_cases)} cases")
print(f"{'='*60}\n")
for i, case in enumerate(test_cases):
cid = case.get("id", f"case_{i}")
# router_test_cases_gen.json์˜ user_input (์ด๋ฉ”์ผ ์›๋ฌธ) ์‚ฌ์šฉ
raw_input = case.get("user_input") or case.get("qa_question") or case.get("question", "")
t0 = time.perf_counter()
query_extraction_input = raw_input
if not args.no_preprocess:
try:
ctx = preprocess_email(raw_input)
if ctx.preprocess_ok:
query_extraction_input = (
ctx.question_summary or ctx.cleaned_body or raw_input
)
except Exception as e:
logger.warning("[%s] preprocess failed: %s โ€” using raw user_input", cid, e)
# ์ฟผ๋ฆฌ ์ถ”์ถœ โ€” Query Expansion (worker_b์˜ ์‹ค์ œ ๋กœ์ง๊ณผ ๋™์ผํ•˜๊ฒŒ)
from src.graph.worker_b import _extract_rag_queries
rag_queries = _extract_rag_queries(query_extraction_input)
if not rag_queries:
rag_queries = [case.get("qa_question") or raw_input] # qa_question์ด ์ด๋ฉ”์ผ ์ „๋ฌธ๋ณด๋‹ค ์ •ํ™•ํ•œ ์ฟผ๋ฆฌ
rag_query = rag_queries[0] # ๋Œ€ํ‘œ ์ฟผ๋ฆฌ (reranking ๊ธฐ์ค€)
# ๊ฒ€์ƒ‰ โ€” ์ฟผ๋ฆฌ๋ณ„ ๊ฒ€์ƒ‰ ํ›„ ์ค‘๋ณต ์ œ๊ฑฐ ๋ณ‘ํ•ฉ
try:
seen_contents: set[str] = set()
candidate_docs = []
for q in rag_queries:
for doc in retriever.invoke(q):
if doc.page_content not in seen_contents:
seen_contents.add(doc.page_content)
candidate_docs.append(doc)
except Exception as e:
logger.error("[%s] Retrieval failed: %s", cid, e)
candidate_docs = []
# ๋ฆฌ๋žญํ‚น (multi-query)
try:
top_docs = rerank(rag_query, candidate_docs, top_n=top_n, queries=rag_queries)
except Exception as e:
logger.warning("[%s] Rerank failed: %s. Using top-k.", cid, e)
top_docs = candidate_docs[:top_n]
# ์ง๋ ฌํ™”
docs_serial = _serialize_docs(top_docs)
# ๋‹ต๋ณ€ ์ƒ์„ฑ (LLM judge ํ•„์š” ์‹œ)
answer = ""
if not args.skip_llm_judge:
from src.graph.worker_b import _generate_answer
answer = _generate_answer(rag_query, top_docs)
elapsed = (time.perf_counter() - t0) * 1000
# ์ •๋‹ต ์œ„์น˜ (evidence_chunk_id ์žˆ์œผ๋ฉด id ์ •ํ™• ๋งค์นญ, ์—†์œผ๋ฉด ํ…์ŠคํŠธ ํด๋ฐฑ)
hit_pos = _locate_evidence(case, docs_serial)
result = {
"id": cid,
"question": rag_query,
"raw_input": raw_input[:100],
"label": case.get("label", ""),
"hit": hit_pos is not None,
"hit_position": hit_pos,
"rag_answer": answer,
"retrieved_docs": docs_serial,
"elapsed_ms": round(elapsed, 1),
}
results.append(result)
all_retrieved.append(docs_serial)
rag_answers.append(answer)
status = f"HIT@{hit_pos+1}" if hit_pos is not None else "MISS"
print(f" [{cid}] {status} | {elapsed:.0f}ms | {rag_query[:60]!r}")
# ์ง‘๊ณ„
print(f"\n{'='*60}")
print(" Computing metrics ...")
summary = run_rag_evaluation(
test_cases=test_cases,
rag_answers=rag_answers if not args.skip_llm_judge else None,
all_retrieved=all_retrieved,
skip_llm_judge=args.skip_llm_judge,
)
# ๊ฒฐ๊ณผ ์ถœ๋ ฅ
print(f"\n{'='*60}")
print(" RAG Evaluation Summary")
print(f"{'='*60}")
for key, label in METRIC_LABELS.items():
val = summary.get(key)
if val is None:
print(f" {label}: (skipped)")
continue
target = TARGETS.get(key, 0)
pct = f"{val*100:.1f}%"
flag = "OK" if val >= target else "FAIL"
print(f" {label}: {pct:>7} (target: {target*100:.0f}%) [{flag}]")
print(f" {'Cases':12}: {summary['n_cases']}")
if not args.skip_llm_judge:
inv_cr = summary.get("judge_invalid_context_recall", 0)
inv_fa = summary.get("judge_invalid_faithfulness", 0)
print(f" {'Ctx Invalid':12}: {inv_cr:>3} [{'OK' if inv_cr == 0 else 'WARN'}]")
print(f" {'Faith Invalid':12}: {inv_fa:>3} [{'OK' if inv_fa == 0 else 'WARN'}]")
print(f"{'='*60}\n")
# ๋ฆฌํฌํŠธ ์ €์žฅ
report_path = Path(args.report)
report_path.parent.mkdir(parents=True, exist_ok=True)
report = {"summary": summary, "targets": TARGETS, "results": results}
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Report saved to: {report_path}\n")
if __name__ == "__main__":
_main()