Spaces:
Sleeping
Sleeping
File size: 3,314 Bytes
af383cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Completeness evaluator - Does the answer cover all aspects?"""
import re
import json
from ..types import (
QAPair,
SystemOutput,
EvaluationMetric,
)
from .base import BaseEvaluator
class CompletenessEvaluator(BaseEvaluator):
"""Evaluates whether the answer addresses all aspects of the question.
Completeness measures if the answer:
- Covers all parts of multi-part questions
- Provides sufficient detail
- Doesn't omit important information
- Is thorough without being excessive
Example:
Question: "What are the benefits and drawbacks of Python?"
Incomplete: "Python is easy to learn" (only mentions benefits)
Complete: "Benefits: readable, large ecosystem. Drawbacks: slower than C++"
"""
@property
def metric(self) -> EvaluationMetric:
return EvaluationMetric.COMPLETENESS
@property
def system_prompt(self) -> str:
return """You are an expert evaluator assessing answer completeness.
Completeness means the answer addresses all aspects of the question:
- All sub-questions are answered
- Sufficient detail for each aspect
- Key information is not omitted
- Balanced coverage (not over-focused on one aspect)
You will be given:
- Question asked
- Reference answer (example of complete answer)
- System's answer
Your task: Rate how complete the system answer is.
Score 1.0: All question aspects thoroughly addressed
Score 0.8: All aspects covered but could use more detail
Score 0.5: Some aspects missed or under-explored
Score 0.2: Many important aspects missing
Score 0.0: Severely incomplete, major gaps
For multi-part questions, check:
- Part 1 addressed?
- Part 2 addressed?
- Part 3 addressed?
- Sufficient depth for each?
Respond with JSON:
{
"score": <float 0-1>,
"question_aspects": [<list of aspects in question>],
"covered_aspects": [<aspects fully addressed>],
"partially_covered": [<aspects mentioned but lacking detail>],
"missing_aspects": [<aspects not addressed>],
"reasoning": "<explanation>"
}"""
def format_prompt(
self,
qa_pair: QAPair,
system_output: SystemOutput,
) -> str:
return f"""QUESTION:
{qa_pair.question}
REFERENCE ANSWER (example of complete answer):
{qa_pair.answer}
SYSTEM ANSWER (to evaluate):
{system_output.answer}
How complete is the system answer? Does it address all aspects?"""
async def parse_judge_response(self, response: str) -> tuple[float, str]:
"""Parse JSON response from judge."""
try:
json_match = re.search(r'\{.*\}', response, re.DOTALL)
if json_match:
data = json.loads(json_match.group())
else:
data = json.loads(response)
score = float(data.get("score", 0.5))
reasoning = data.get("reasoning", "No reasoning provided")
return max(0, min(1, score)), reasoning
except json.JSONDecodeError:
score_match = re.search(r'score["\s:]*(\d+\.?\d*)', response.lower())
if score_match:
score = float(score_match.group(1))
score = score / 100 if score > 1 else score
return max(0, min(1, score)), response[:200]
return 0.5, response[:200]
|