File size: 1,774 Bytes
be0c348 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Construct one DDI judge request and validate its response; no network calls."""
import json
from inference.evaluation.protocol import REPO_ROOT, prompt_text
def build_judge_request(reference_diagnosis, model_output):
if not isinstance(reference_diagnosis, str) or not reference_diagnosis.strip():
raise ValueError('A nonempty reference diagnosis is required for scoring.')
if not isinstance(model_output, str):
raise ValueError('The generated response must be a string.')
config = json.loads((REPO_ROOT / 'configs/ddi_judge.json').read_text())
tail = model_output[-config['batch_evaluator_defaults']['model_output_tail_characters']:]
user = prompt_text('ddi_judge_user.txt').format(
ground_truth=reference_diagnosis, model_output=tail
)
return {
key: config[key] for key in ['model', 'temperature', 'max_tokens', 'response_format', 'extra_body']
} | {'messages': [
{'role': 'system', 'content': prompt_text('ddi_judge_system.txt')},
{'role': 'user', 'content': user},
]}
def validate_judge_response(content):
result = json.loads(content)
if not isinstance(result, dict) or set(result) != {'predicted_diagnosis', 'correct', 'reason'}:
raise ValueError('Unexpected judge response fields.')
if not isinstance(result['predicted_diagnosis'], str) or type(result['correct']) is not bool:
raise ValueError('Invalid diagnosis or correctness type.')
if not isinstance(result['reason'], str) or not result['reason'].strip():
raise ValueError('A nonempty reason is required.')
if result['correct'] and not result['predicted_diagnosis'].strip():
raise ValueError('A correct response requires an identified final diagnosis.')
return result
|