fev-4b / fev4b /metrics.py
murabcd's picture
feat: add compiled serving shared training and benchmarks
5adf069 verified
Raw History Blame Contribute Delete
3.89 kB
"""Distribution metrics, with all failed questions retained in the denominator."""
import math
from .data import option_keys, target_index
def distribution(question, answer):
if answer.get('type') != question['type']:
raise ValueError('Answer type mismatch')
keys = option_keys(question)
if question['type'] == 'noul':
p = answer['noul']
values = [1-p, p]
else:
if set(answer['probabilities']) != set(keys):
raise ValueError('Wrong probability keys')
values = [answer['probabilities'][k] for k in keys]
if (any(type(p) not in (int, float) or not math.isfinite(p) or not 0 <= p <= 1 for p in values)
or not math.isclose(sum(values), 1, abs_tol=.02)):
raise ValueError('Invalid distribution')
# Hosted APIs serialize rounded probabilities. Normalize only within the
# API tolerance; keep the original response in predictions.jsonl.
return [p/sum(values) for p in values]
def rows_for(record, response=None, error=None):
rows = []
if response is not None and set(response['answers']) != set(record['questions']):
raise ValueError('Incomplete response')
for key, question in record['questions'].items():
row = {'id': record['id'], 'question': key, 'type': question['type'],
'source': record.get('source', record.get('domain', 'unspecified')), 'error': error}
if error is None:
values = distribution(question, response['answers'][key])
target = target_index(question)
predicted = max(range(len(values)), key=values.__getitem__)
row.update(correct=predicted == target, confidence=max(values),
probabilities=values, target=target,
brier=sum((p - int(i == target))**2 for i, p in enumerate(values)),
nll=-math.log(max(values[target], 1e-12)))
if question['type'] == 'score':
row['ordinalAbsoluteError'] = abs(sum(i*p for i, p in enumerate(values)) - target)
rows.append(row)
return rows
def summarize(rows):
good = [r for r in rows if r['error'] is None]
bins = [[] for _ in range(10)]
for row in good:
bins[min(9, int(row['confidence']*10))].append(row)
ece = sum(len(b) * abs(sum(r['correct'] for r in b)/len(b) - sum(r['confidence'] for r in b)/len(b))
for b in bins if b) / len(good) if good else None
coverage = []
for threshold in (.5, .7, .8, .9, .95, .99):
accepted = [r for r in good if r['confidence'] >= threshold]
coverage.append({'threshold': threshold, 'accepted': len(accepted),
'coverage': len(accepted)/len(rows) if rows else 0,
'errorRate': sum(not r['correct'] for r in accepted)/len(accepted) if accepted else None})
return {'questions': len(rows), 'answered': len(good), 'errors': len(rows)-len(good),
'correct': sum(r['correct'] for r in good),
'accuracyAllQuestions': sum(r['correct'] for r in good)/len(rows) if rows else None,
'brierAnswered': sum(r['brier'] for r in good)/len(good) if good else None,
'nllAnswered': sum(r['nll'] for r in good)/len(good) if good else None,
'ece10Answered': ece,
'confidentWrongAt09': sum(not r['correct'] and r['confidence'] >= .9 for r in good),
'coverageByFixedThreshold': coverage,
'coverageInterpretation': 'Observed on this dataset; not a future error guarantee. Thresholds not tuned here.'}
def report_for(rows):
return {'overall': summarize(rows),
'byType': {k: summarize([r for r in rows if r['type'] == k]) for k in sorted({r['type'] for r in rows})},
'bySource': {k: summarize([r for r in rows if r['source'] == k]) for k in sorted({r['source'] for r in rows})}}