Spaces:
Sleeping
Sleeping
Download src/evaluation/eval_worker_b.py from daisysooyeon/SAP-ERP-AI-Agent: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/spaces/daisysooyeon/SAP-ERP-AI-Agent/resolve/main/src/evaluation/eval_worker_b.py
- Command line
-
hf download hf://spaces/daisysooyeon/SAP-ERP-AI-Agent/src/evaluation/eval_worker_b.py
-
curl -L -o eval_worker_b.py https://huggingface.co/spaces/daisysooyeon/SAP-ERP-AI-Agent/resolve/main/src/evaluation/eval_worker_b.py
21.5 kB
| """ | |
| src/evaluation/eval_worker_b.py | |
| RAG ํ์ง ํ๊ฐ โ ์์ฒด ๊ตฌํ ๋ฉํธ๋ฆญ (RAGAS ๋์ ) | |
| ํ๊ฐ ๋ฉํธ๋ฆญ: | |
| Hit Rate : top-k ์์ ์ ๋ต source_chunk๊ฐ ํฌํจ๋ ๋น์จ (๋ชฉํ โฅ 85%) | |
| NDCG@3 : ์์ 3๊ฐ ์ฒญํฌ ์์ ํ์ง, ์ ๋ต์ด ๋์์๋ก ๋์ ์ ์ (๋ชฉํ โฅ 0.75) | |
| MRR : Mean Reciprocal Rank, ์ฒซ ์ ๋ต ๋ฌธ์ ์ญ์์ ํ๊ท (๋ชฉํ โฅ 0.70) | |
| Context Recall: top-k ์ปจํ ์คํธ๊ฐ ์ ๋ต ์์ฑ์ ํ์ํ ์ ๋ณด๋ฅผ ํฌํจํ๋์ง (๋ชฉํ โฅ 0.90) | |
| Faithfulness : ์์ฑ ๋ต๋ณ์ด retrieved ์ปจํ ์คํธ์ ๊ทผ๊ฑฐํ๋์ง LLM judge (๋ชฉํ โฅ 0.85) | |
| ์ ๋ ฅ ๋ฐ์ดํฐ์ ํ์ (router_test_cases_gen.json, QA_ONLY + BOTH ํํฐ): | |
| { | |
| "id": "r_001", | |
| "user_input": "customer email text", | |
| "label": "QA_ONLY", | |
| "rag_evidence": "exact chunk text (str or list[str] for multi-chunk)", | |
| "qa_question": "extracted knowledge question" | |
| } | |
| CLI ์ฌ์ฉ๋ฒ: | |
| python -m src.evaluation.eval_worker_b # ์ ์ฒด ํ๊ฐ | |
| python -m src.evaluation.eval_worker_b --skip-llm-judge # retrieval ๋ฉํธ๋ฆญ๋ง (๋น ๋ฆ) | |
| python -m src.evaluation.eval_worker_b --report reports/worker_b_eval_result.json | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import math | |
| import os | |
| import time | |
| from dotenv import load_dotenv | |
| from langchain_core.prompts import ChatPromptTemplate | |
| from langchain_openai import ChatOpenAI | |
| from src.config import get_config | |
| from src.rag.retriever import build_hybrid_retriever | |
| from src.rag.reranker import rerank | |
| def _serialize_docs(docs: list) -> list[dict]: | |
| return [ | |
| { | |
| "content": doc.page_content, | |
| "source": doc.metadata.get("source", ""), | |
| "chunk_id": doc.metadata.get("chunk_id", ""), | |
| "page": doc.metadata.get("page", -1), | |
| } | |
| for doc in docs | |
| ] | |
| load_dotenv() | |
| logger = logging.getLogger(__name__) | |
| # --------------------------------------------------------------------------- | |
| # LLM for judge (gemini-flash) | |
| # --------------------------------------------------------------------------- | |
| _judge_llm: ChatOpenAI | None = None | |
| def _get_judge_llm() -> ChatOpenAI: | |
| global _judge_llm | |
| if _judge_llm is not None: | |
| return _judge_llm | |
| cfg = get_config() | |
| api_key = os.getenv("OPENROUTER_API_KEY", "") | |
| judge_cfg = cfg.models.eval_judge | |
| _judge_llm = ChatOpenAI( | |
| model=judge_cfg.name, | |
| temperature=0.0, | |
| openai_api_key=api_key, | |
| openai_api_base=cfg.openrouter.base_url, | |
| default_headers={ | |
| "HTTP-Referer": "https://github.com/daisysooyeon/SAP-ERP-AI-Agent", | |
| "X-Title": "SAP-ERP-AI-Agent RAG-Eval", | |
| }, | |
| ) | |
| return _judge_llm | |
| # --------------------------------------------------------------------------- | |
| # ๋งค์นญ ํฌํผ | |
| # --------------------------------------------------------------------------- | |
| def _chunk_in_docs(source_chunks: str | list[str], docs: list[dict], threshold: float = 0.5) -> int | None: | |
| """ | |
| source_chunks(๋ฌธ์์ด ๋๋ ๋ฆฌ์คํธ) ์ค ํ๋๋ผ๋ retrieved docs ์ค ์ด๋ ์์น(0-indexed)์ ์๋์ง ๋ฐํ. | |
| ์ ํํ substring์ด ์์ผ๋ฉด 50% ์ด์ ํ ํฐ ๊ฒน์นจ์ผ๋ก ํ๋จ. | |
| ์ฌ๋ฌ ๊ฐ ๋งค์นญ๋ ๊ฒฝ์ฐ ๊ฐ์ฅ ๋์ ์์(์์ ์ธ๋ฑ์ค) ๋ฐํ. | |
| ์์ผ๋ฉด None ๋ฐํ. | |
| """ | |
| if not source_chunks: | |
| return None | |
| if isinstance(source_chunks, str): | |
| chunks_to_check = [source_chunks] | |
| else: | |
| chunks_to_check = source_chunks | |
| best_pos = None | |
| for chunk in chunks_to_check: | |
| gt_tokens = set(chunk.lower().split()) | |
| for i, doc in enumerate(docs): | |
| content = doc["content"] | |
| # 1์์: substring | |
| if chunk[:80].strip() in content: | |
| if best_pos is None or i < best_pos: | |
| best_pos = i | |
| break | |
| # 2์์: ํ ํฐ overlap | |
| doc_tokens = set(content.lower().split()) | |
| if gt_tokens and doc_tokens: | |
| overlap = len(gt_tokens & doc_tokens) / len(gt_tokens) | |
| if overlap >= threshold: | |
| if best_pos is None or i < best_pos: | |
| best_pos = i | |
| break | |
| return best_pos | |
| def _match_by_id(evidence_ids: str | list[str], docs: list[dict]) -> int | None: | |
| """retrieved docs ์ค metadata chunk_id ๊ฐ ์ ๋ต id ์งํฉ์ ์ํ ์ฒซ ์์น(0-indexed)๋ฅผ ๋ฐํ.""" | |
| if not evidence_ids: | |
| return None | |
| id_set = {evidence_ids} if isinstance(evidence_ids, str) else set(evidence_ids) | |
| for i, doc in enumerate(docs): | |
| if doc.get("chunk_id") in id_set: | |
| return i | |
| return None | |
| def _locate_evidence(case: dict, docs: list[dict], threshold: float = 0.5) -> int | None: | |
| """ | |
| ์ ๋ต ์ฒญํฌ๊ฐ docs ์ค ์ด๋ ์์น์ ์๋์ง ๋ฐํ. | |
| case ์ ์ ๋ต chunk_id(evidence_chunk_id)๊ฐ ์์ผ๋ฉด id ์ ํ ๋งค์นญ, | |
| ์์ผ๋ฉด rag_evidence ํ ์คํธ fuzzy ๋งค์นญ์ผ๋ก ํด๋ฐฑ. (build_eval_ids.py ๋ก id ๋ถ์ฌ ๊ฐ๋ฅ) | |
| """ | |
| evidence_ids = case.get("evidence_chunk_id") | |
| if evidence_ids: | |
| return _match_by_id(evidence_ids, docs) | |
| return _chunk_in_docs(case.get("rag_evidence"), docs, threshold) | |
| # --------------------------------------------------------------------------- | |
| # Retrieval ๋ฉํธ๋ฆญ (Hit Rate, NDCG@3, MRR) | |
| # --------------------------------------------------------------------------- | |
| def compute_hit_rate(cases: list[dict], all_retrieved: list[list[dict]]) -> float: | |
| """top-k ์์ ์ ๋ต ์ฒญํฌ๊ฐ ํฌํจ๋ ๋น์จ""" | |
| hits = sum( | |
| 1 for case, docs in zip(cases, all_retrieved) | |
| if _locate_evidence(case, docs) is not None | |
| ) | |
| return hits / len(cases) if cases else 0.0 | |
| def compute_ndcg_at_3(cases: list[dict], all_retrieved: list[list[dict]]) -> float: | |
| """ | |
| NDCG@3: ์ ๋ต์ด top-3 ์์์ ๋ช ๋ฒ์งธ์ ์๋์ง์ ๋ฐ๋ผ ์ ์ ์ฐจ๋ฑ | |
| IDCG = 1/log2(2) = 1.0 (์ ๋ต์ด 1์์ธ ์ด์์ ์ผ์ด์ค) | |
| """ | |
| ndcg_scores = [] | |
| for case, docs in zip(cases, all_retrieved): | |
| top3 = docs[:3] | |
| pos = _locate_evidence(case, top3) | |
| if pos is None: | |
| ndcg_scores.append(0.0) | |
| else: | |
| dcg = 1.0 / math.log2(pos + 2) # pos๋ 0-indexed, log2(rank+1) | |
| idcg = 1.0 / math.log2(2) # ์ด์์ : rank=1 | |
| ndcg_scores.append(dcg / idcg) | |
| return sum(ndcg_scores) / len(ndcg_scores) if ndcg_scores else 0.0 | |
| def compute_mrr(cases: list[dict], all_retrieved: list[list[dict]]) -> float: | |
| """Mean Reciprocal Rank: ์ฒซ ๋ฒ์งธ ์ ๋ต ๋ฌธ์์ ์ญ์์ ํ๊ท """ | |
| rr_scores = [] | |
| for case, docs in zip(cases, all_retrieved): | |
| pos = _locate_evidence(case, docs) | |
| if pos is None: | |
| rr_scores.append(0.0) | |
| else: | |
| rr_scores.append(1.0 / (pos + 1)) # 0-indexed โ 1-indexed | |
| return sum(rr_scores) / len(rr_scores) if rr_scores else 0.0 | |
| # --------------------------------------------------------------------------- | |
| # LLM Judge ๋ฉํธ๋ฆญ (Context Recall, Faithfulness) | |
| # --------------------------------------------------------------------------- | |
| _CONTEXT_RECALL_PROMPT = ChatPromptTemplate.from_messages([ | |
| ("system", ( | |
| "You are an evaluation assistant. " | |
| "Given a question and a context, " | |
| "determine whether the context contains enough information to answer the question correctly.\n" | |
| "Respond with ONLY '1' (yes, context is sufficient) or '0' (no, context is insufficient)." | |
| )), | |
| ("human", ( | |
| "Question: {question}\n\n" | |
| "Context:\n{context}\n\n" | |
| "Does the context contain sufficient information? (1 or 0):" | |
| )), | |
| ]) | |
| _FAITHFULNESS_PROMPT = ChatPromptTemplate.from_messages([ | |
| ("system", ( | |
| "You are an evaluation assistant. " | |
| "Given an answer and the context it was generated from, " | |
| "determine whether the answer is fully supported by the context (no hallucination).\n" | |
| "Respond with ONLY '1' (yes, answer is faithful) or '0' (no, answer contains hallucination)." | |
| )), | |
| ("human", ( | |
| "Context:\n{context}\n\n" | |
| "Answer: {rag_answer}\n\n" | |
| "Is the answer fully supported by the context? (1 or 0):" | |
| )), | |
| ]) | |
| def _llm_judge( | |
| prompt_template: ChatPromptTemplate, | |
| variables: dict, | |
| metric: str = "unknown", | |
| delay: float = 0.5, | |
| ) -> tuple[int, bool]: | |
| """LLM judge ํธ์ถ โ (score, is_invalid) ๋ฐํ. score๋ 0 ๋๋ 1.""" | |
| try: | |
| llm = _get_judge_llm() | |
| chain = prompt_template | llm | |
| result = chain.invoke(variables) | |
| # qwen3-8b๋ <think>...</think> ๋ธ๋ก ๋๋ ์ค๋ฐ๊ฟ ํ ์ซ์๋ฅผ ์ถ๋ ฅํ๋ ๊ฒฝ์ฐ๊ฐ ์์ | |
| import re as _re | |
| raw_val = result.content.strip() | |
| # <think>...</think> ์ ๊ฑฐ | |
| val = _re.sub(r"<think>.*?</think>", "", raw_val, flags=_re.DOTALL).strip() | |
| # ์ฒซ ๋ฒ์งธ ๋ฑ์ฅํ๋ 0 ๋๋ 1 ์ถ์ถ | |
| m = _re.search(r"[01]", val) | |
| val = m.group() if m else val | |
| time.sleep(delay) | |
| if val.startswith("1"): | |
| return 1, False | |
| elif val.startswith("0"): | |
| return 0, False | |
| else: | |
| logger.warning( | |
| "[eval_rag] LLM judge [%s] unexpected response: %r", | |
| metric, val[:80], | |
| ) | |
| return 0, True # conservative: unexpected ์๋ต์ 0์ ์ฒ๋ฆฌ | |
| except Exception as e: | |
| logger.warning("[eval_rag] LLM judge [%s] failed: %s", metric, e) | |
| return 0, False | |
| def compute_context_recall( | |
| cases: list[dict], | |
| all_retrieved: list[list[dict]], | |
| ) -> tuple[float, int]: | |
| """Context Recall: retrieved context๊ฐ ์ ๋ต ์์ฑ์ ์ถฉ๋ถํ์ง LLM judge (๋ชฉํ >= 0.90)""" | |
| scores = [] | |
| invalid = 0 | |
| for case, docs in zip(cases, all_retrieved): | |
| context = "\n\n".join(d["content"] for d in docs) | |
| score, is_invalid = _llm_judge( | |
| _CONTEXT_RECALL_PROMPT, | |
| {"question": case.get("qa_question") or case.get("question"), "context": context}, | |
| metric="context_recall", | |
| ) | |
| if is_invalid: | |
| invalid += 1 | |
| scores.append(score) | |
| logger.debug("[eval_rag] Context Recall [%s]: %d", case.get("id"), score) | |
| return (sum(scores) / len(scores) if scores else 0.0), invalid | |
| def compute_faithfulness( | |
| cases: list[dict], | |
| all_retrieved: list[list[dict]], | |
| rag_answers: list[str], | |
| ) -> tuple[float, int]: | |
| """Faithfulness: ์์ฑ ๋ต๋ณ์ด ์ปจํ ์คํธ์ ๊ทผ๊ฑฐํ๋์ง LLM judge (๋ชฉํ >= 0.85)""" | |
| scores = [] | |
| invalid = 0 | |
| for case, docs, answer in zip(cases, all_retrieved, rag_answers): | |
| if not answer or not docs: | |
| scores.append(0) | |
| continue | |
| context = "\n\n".join(d["content"] for d in docs) | |
| score, is_invalid = _llm_judge( | |
| _FAITHFULNESS_PROMPT, | |
| {"context": context, "rag_answer": answer}, | |
| metric="faithfulness", | |
| ) | |
| if is_invalid: | |
| invalid += 1 | |
| scores.append(score) | |
| logger.debug("[eval_rag] Faithfulness [%s]: %d", case.get("id"), score) | |
| return (sum(scores) / len(scores) if scores else 0.0), invalid | |
| # --------------------------------------------------------------------------- | |
| # ๋ฉ์ธ ํ๊ฐ ํจ์ | |
| # --------------------------------------------------------------------------- | |
| def run_rag_evaluation( | |
| test_cases: list[dict], | |
| rag_answers: list[str] | None = None, | |
| all_retrieved: list[list[dict]] | None = None, | |
| skip_llm_judge: bool = False, | |
| ) -> dict: | |
| """ | |
| ์ ์ฒด RAG ํ๊ฐ ์คํ. | |
| Args: | |
| test_cases: rag_test_cases.json ๋ก๋ ๊ฒฐ๊ณผ | |
| rag_answers: ๊ฐ ์ผ์ด์ค์ ๋ํ ์์ฑ ๋ต๋ณ (์์ผ๋ฉด retrieval ๋ฉํธ๋ฆญ๋ง ๊ณ์ฐ) | |
| all_retrieved: ๊ฐ ์ผ์ด์ค์ retrieved docs ๋ชฉ๋ก | |
| skip_llm_judge: True์ด๋ฉด Context Recall / Faithfulness ์คํต | |
| Returns: | |
| { | |
| "hit_rate": float, | |
| "ndcg_at_3": float, | |
| "mrr": float, | |
| "context_recall": float | None, | |
| "faithfulness": float | None, | |
| "n_cases": int, | |
| } | |
| """ | |
| n = len(test_cases) | |
| logger.info("[eval_rag] Evaluating %d cases ...", n) | |
| hit_rate = compute_hit_rate(test_cases, all_retrieved) | |
| ndcg_at_3 = compute_ndcg_at_3(test_cases, all_retrieved) | |
| mrr = compute_mrr(test_cases, all_retrieved) | |
| context_recall: float | None = None | |
| faithfulness: float | None = None | |
| invalid_cr: int = 0 | |
| invalid_fa: int = 0 | |
| if not skip_llm_judge and rag_answers: | |
| logger.info("[eval_rag] Running LLM judges (Context Recall + Faithfulness) ...") | |
| context_recall, invalid_cr = compute_context_recall(test_cases, all_retrieved) | |
| faithfulness, invalid_fa = compute_faithfulness(test_cases, all_retrieved, rag_answers) | |
| if invalid_cr > 0: | |
| logger.warning("[eval_rag] Context Recall judge invalid: %d / %d", invalid_cr, n) | |
| if invalid_fa > 0: | |
| logger.warning("[eval_rag] Faithfulness judge invalid: %d / %d", invalid_fa, n) | |
| return { | |
| "hit_rate": round(hit_rate, 4), | |
| "ndcg_at_3": round(ndcg_at_3, 4), | |
| "mrr": round(mrr, 4), | |
| "context_recall": round(context_recall, 4) if context_recall is not None else None, | |
| "faithfulness": round(faithfulness, 4) if faithfulness is not None else None, | |
| "n_cases": n, | |
| "judge_invalid_context_recall": invalid_cr, | |
| "judge_invalid_faithfulness": invalid_fa, | |
| } | |
| # --------------------------------------------------------------------------- | |
| # CLI ์ง์ ์ (python -m src.evaluation.eval_rag ๋๋ run_eval_rag.sh ์์ ํธ์ถ) | |
| # --------------------------------------------------------------------------- | |
| TARGETS = { | |
| "hit_rate": 0.85, | |
| "ndcg_at_3": 0.75, | |
| "mrr": 0.70, | |
| "context_recall": 0.90, | |
| "faithfulness": 0.85, | |
| } | |
| METRIC_LABELS = { | |
| "hit_rate": "Hit Rate ", | |
| "ndcg_at_3": "NDCG@3 ", | |
| "mrr": "MRR ", | |
| "context_recall": "Ctx Recall ", | |
| "faithfulness": "Faithfulness", | |
| } | |
| def _main(): | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| # Windows cp949 ํฐ๋ฏธ๋ ์ธ์ฝ๋ฉ ๋์ | |
| try: | |
| sys.stdout.reconfigure(encoding="utf-8") | |
| except Exception: | |
| pass | |
| from src.logging_config import setup_logging | |
| setup_logging() | |
| parser = argparse.ArgumentParser(description="RAG Quality Evaluation") | |
| parser.add_argument("--report", default="reports/worker_b_eval_result.json", help="์ถ๋ ฅ ๋ฆฌํฌํธ ๊ฒฝ๋ก") | |
| parser.add_argument("--dataset", default="data/eval/router_test_cases_gen.json") | |
| parser.add_argument("--skip-llm-judge", action="store_true", help="LLM judge ๋ฉํธ๋ฆญ ์คํต") | |
| parser.add_argument( | |
| "--no-preprocess", action="store_true", | |
| help="Skip email preprocessing (cleaned_body extraction) and extract queries from raw user_input " | |
| "(default: use preprocessed cleaned_body, matching production worker_b_node)", | |
| ) | |
| args = parser.parse_args() | |
| # ๋ฐ์ดํฐ์ ๋ก๋ ๋ฐ ํํฐ๋ง (QA_ONLY, BOTH) | |
| dataset_path = Path(args.dataset) | |
| all_cases: list[dict] = json.loads(dataset_path.read_text(encoding="utf-8")) | |
| test_cases = [c for c in all_cases if c.get("label") in ("QA_ONLY", "BOTH")] | |
| logger.info("Loaded %d test cases for Worker B from %s", len(test_cases), dataset_path) | |
| # ๋ฆฌํธ๋ฆฌ๋ฒ / ๋ฆฌ๋ญ์ปค ์ด๊ธฐํ | |
| logger.info("Initializing retriever (bge-m3 loading may take a moment) ...") | |
| retriever = build_hybrid_retriever() | |
| cfg = get_config() | |
| top_n = cfg.rag.top_k_rerank | |
| # ์ด๋ฉ์ผ ์ ์ฒ๋ฆฌ โ ํ๋ก๋์ (worker_b_node)์ ์๋ณธ ๋์ cleaned_body๋ก ์ฟผ๋ฆฌ๋ฅผ ์ถ์ถํ๋ค. | |
| # eval๋ ๋์ผ ์กฐ๊ฑด์ ๋ง์ถ๊ธฐ ์ํด ์ผ์ด์ค๋ณ๋ก ์ ์ฒ๋ฆฌํด cleaned_body๋ฅผ ์ฟผ๋ฆฌ ์ถ์ถ ์ ๋ ฅ์ผ๋ก ์ด๋ค. | |
| # (--no-preprocess ๋ฉด ๊ธฐ์กด์ฒ๋ผ ์๋ณธ user_input ์ฌ์ฉ) | |
| if not args.no_preprocess: | |
| from src.preprocess.email_preprocessor import preprocess_email | |
| logger.info("Email preprocessing ENABLED (using cleaned_body for query extraction, matching production)") | |
| else: | |
| logger.info("Email preprocessing DISABLED (using raw user_input)") | |
| # ๊ฐ ์ผ์ด์ค ์คํ | |
| results = [] | |
| all_retrieved: list[list[dict]] = [] | |
| rag_answers: list[str] = [] | |
| print(f"\n{'='*60}") | |
| print(f" RAG Evaluation - {len(test_cases)} cases") | |
| print(f"{'='*60}\n") | |
| for i, case in enumerate(test_cases): | |
| cid = case.get("id", f"case_{i}") | |
| # router_test_cases_gen.json์ user_input (์ด๋ฉ์ผ ์๋ฌธ) ์ฌ์ฉ | |
| raw_input = case.get("user_input") or case.get("qa_question") or case.get("question", "") | |
| t0 = time.perf_counter() | |
| query_extraction_input = raw_input | |
| if not args.no_preprocess: | |
| try: | |
| ctx = preprocess_email(raw_input) | |
| if ctx.preprocess_ok: | |
| query_extraction_input = ( | |
| ctx.question_summary or ctx.cleaned_body or raw_input | |
| ) | |
| except Exception as e: | |
| logger.warning("[%s] preprocess failed: %s โ using raw user_input", cid, e) | |
| # ์ฟผ๋ฆฌ ์ถ์ถ โ Query Expansion (worker_b์ ์ค์ ๋ก์ง๊ณผ ๋์ผํ๊ฒ) | |
| from src.graph.worker_b import _extract_rag_queries | |
| rag_queries = _extract_rag_queries(query_extraction_input) | |
| if not rag_queries: | |
| rag_queries = [case.get("qa_question") or raw_input] # qa_question์ด ์ด๋ฉ์ผ ์ ๋ฌธ๋ณด๋ค ์ ํํ ์ฟผ๋ฆฌ | |
| rag_query = rag_queries[0] # ๋ํ ์ฟผ๋ฆฌ (reranking ๊ธฐ์ค) | |
| # ๊ฒ์ โ ์ฟผ๋ฆฌ๋ณ ๊ฒ์ ํ ์ค๋ณต ์ ๊ฑฐ ๋ณํฉ | |
| try: | |
| seen_contents: set[str] = set() | |
| candidate_docs = [] | |
| for q in rag_queries: | |
| for doc in retriever.invoke(q): | |
| if doc.page_content not in seen_contents: | |
| seen_contents.add(doc.page_content) | |
| candidate_docs.append(doc) | |
| except Exception as e: | |
| logger.error("[%s] Retrieval failed: %s", cid, e) | |
| candidate_docs = [] | |
| # ๋ฆฌ๋ญํน (multi-query) | |
| try: | |
| top_docs = rerank(rag_query, candidate_docs, top_n=top_n, queries=rag_queries) | |
| except Exception as e: | |
| logger.warning("[%s] Rerank failed: %s. Using top-k.", cid, e) | |
| top_docs = candidate_docs[:top_n] | |
| # ์ง๋ ฌํ | |
| docs_serial = _serialize_docs(top_docs) | |
| # ๋ต๋ณ ์์ฑ (LLM judge ํ์ ์) | |
| answer = "" | |
| if not args.skip_llm_judge: | |
| from src.graph.worker_b import _generate_answer | |
| answer = _generate_answer(rag_query, top_docs) | |
| elapsed = (time.perf_counter() - t0) * 1000 | |
| # ์ ๋ต ์์น (evidence_chunk_id ์์ผ๋ฉด id ์ ํ ๋งค์นญ, ์์ผ๋ฉด ํ ์คํธ ํด๋ฐฑ) | |
| hit_pos = _locate_evidence(case, docs_serial) | |
| result = { | |
| "id": cid, | |
| "question": rag_query, | |
| "raw_input": raw_input[:100], | |
| "label": case.get("label", ""), | |
| "hit": hit_pos is not None, | |
| "hit_position": hit_pos, | |
| "rag_answer": answer, | |
| "retrieved_docs": docs_serial, | |
| "elapsed_ms": round(elapsed, 1), | |
| } | |
| results.append(result) | |
| all_retrieved.append(docs_serial) | |
| rag_answers.append(answer) | |
| status = f"HIT@{hit_pos+1}" if hit_pos is not None else "MISS" | |
| print(f" [{cid}] {status} | {elapsed:.0f}ms | {rag_query[:60]!r}") | |
| # ์ง๊ณ | |
| print(f"\n{'='*60}") | |
| print(" Computing metrics ...") | |
| summary = run_rag_evaluation( | |
| test_cases=test_cases, | |
| rag_answers=rag_answers if not args.skip_llm_judge else None, | |
| all_retrieved=all_retrieved, | |
| skip_llm_judge=args.skip_llm_judge, | |
| ) | |
| # ๊ฒฐ๊ณผ ์ถ๋ ฅ | |
| print(f"\n{'='*60}") | |
| print(" RAG Evaluation Summary") | |
| print(f"{'='*60}") | |
| for key, label in METRIC_LABELS.items(): | |
| val = summary.get(key) | |
| if val is None: | |
| print(f" {label}: (skipped)") | |
| continue | |
| target = TARGETS.get(key, 0) | |
| pct = f"{val*100:.1f}%" | |
| flag = "OK" if val >= target else "FAIL" | |
| print(f" {label}: {pct:>7} (target: {target*100:.0f}%) [{flag}]") | |
| print(f" {'Cases':12}: {summary['n_cases']}") | |
| if not args.skip_llm_judge: | |
| inv_cr = summary.get("judge_invalid_context_recall", 0) | |
| inv_fa = summary.get("judge_invalid_faithfulness", 0) | |
| print(f" {'Ctx Invalid':12}: {inv_cr:>3} [{'OK' if inv_cr == 0 else 'WARN'}]") | |
| print(f" {'Faith Invalid':12}: {inv_fa:>3} [{'OK' if inv_fa == 0 else 'WARN'}]") | |
| print(f"{'='*60}\n") | |
| # ๋ฆฌํฌํธ ์ ์ฅ | |
| report_path = Path(args.report) | |
| report_path.parent.mkdir(parents=True, exist_ok=True) | |
| report = {"summary": summary, "targets": TARGETS, "results": results} | |
| report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") | |
| print(f"Report saved to: {report_path}\n") | |
| if __name__ == "__main__": | |
| _main() | |