rag-backend / src /ops /evaluator.py
Harshilforworks's picture
Upload 99 files
4244db2 verified
Raw History Blame Contribute Delete
2.58 kB
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
def _safe_rate(numerator: int, denominator: int) -> float:
if denominator <= 0:
return 0.0
return round((numerator / denominator) * 100.0, 2)
def run_eval(eval_set_path: str, actual_results: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
path = Path(eval_set_path)
if not path.exists():
return {
"total_queries": 0,
"citation_coverage_rate": 0.0,
"eligibility_correctness_rate": 0.0,
"abstention_accuracy_rate": 0.0,
"details": [],
}
actual_results = actual_results or {}
eval_set = json.loads(path.read_text(encoding="utf-8"))
total = len(eval_set)
citation_hits = 0
prereq_total = 0
prereq_correct = 0
abstention_total = 0
abstention_correct = 0
details: list[dict[str, Any]] = []
for item in eval_set:
query_id = str(item.get("id", ""))
question = str(item.get("question", "") or "")
expected_decision = item.get("expected_decision")
expected_citations = bool(item.get("expected_citations_present", False))
expected_abstained = item.get("expected_abstained")
actual = actual_results.get(query_id, {})
actual_citations = actual.get("citations", [])
actual_decision = actual.get("decision")
actual_abstained = actual.get("abstained")
actual_answer = str(actual.get("answer", "") or "")
has_citation = bool(actual_citations)
if expected_citations and has_citation:
citation_hits += 1
if expected_decision is not None:
prereq_total += 1
if actual_decision == expected_decision:
prereq_correct += 1
if expected_abstained is not None:
abstention_total += 1
if bool(actual_abstained) == bool(expected_abstained):
abstention_correct += 1
details.append(
{
"id": query_id,
"type": item.get("type"),
"question": question,
"actual_answer": actual_answer,
"expected_decision": expected_decision,
"actual_decision": actual_decision,
"expected_abstained": expected_abstained,
"actual_abstained": actual_abstained,
"expected_citations_present": expected_citations,
"actual_citations_present": has_citation,
}
)
return {
"total_queries": total,
"citation_coverage_rate": _safe_rate(citation_hits, len([q for q in eval_set if q.get("expected_citations_present")])),
"eligibility_correctness_rate": _safe_rate(prereq_correct, prereq_total),
"abstention_accuracy_rate": _safe_rate(abstention_correct, abstention_total),
"details": details,
}