SCOPD / evaluation /scopd_eval /scoring.py
enmingzhangzz's picture
Publish evaluation-and-model-card
ab18ce0 verified
Raw History Blame Contribute Delete
3.84 kB
"""Aggregate correctness without erasing dataset-specific paired scoring."""
from collections import defaultdict
from statistics import mean
def summarize(rows):
groups = defaultdict(list)
for row in rows:
groups[row['dataset']].append(row)
reports = {}
for dataset, items in groups.items():
report = {'rows': len(items), 'accuracy_percent': 100 * mean(r['correct'] for r in items),
'deterministic_match_percent': 100 * mean(r['exact_match'] for r in items)}
categories = defaultdict(list)
for row in items:
categories[str(row.get('metadata', {}).get('category', 'all'))].append(row['correct'])
report['category_accuracy_percent'] = {k: 100 * mean(v) for k, v in categories.items()}
if dataset == 'MME':
pairs = defaultdict(list)
for row in items:
meta = row['metadata']
pairs[(meta['category'], meta['image_path'])].append(row['correct'])
if any(len(v) != 2 for v in pairs.values()):
report['mme_score'] = None
report['note'] = 'Incomplete MME image pairs: no official-style total reported'
else:
by_category = defaultdict(list)
for (category, _), values in pairs.items():
by_category[category].append(all(values))
report['mme_category_score'] = {k: 100 * (mean(categories[k]) + mean(v))
for k, v in by_category.items()}
report['mme_score'] = sum(report['mme_category_score'].values())
if dataset == 'MMVP':
pairs = defaultdict(list)
for row in items:
idx = int(row['sample_id'])
if idx < 1:
raise ValueError('MMVP official indices start at 1')
pairs[(idx - 1) // 2].append(row['correct'])
report['pair_accuracy_percent'] = (100 * mean(all(v) for v in pairs.values())
if all(len(v) == 2 for v in pairs.values()) else None)
if dataset == 'HallusionBench':
figures, questions = defaultdict(list), defaultdict(list)
for row in items:
parts = str(row['sample_id']).split('_')
if len(parts) < 6:
raise ValueError('HallusionBench IDs lack set/figure/question fields')
cat = row['metadata']['l2-category']
figures[(cat, parts[3], parts[4])].append(row['correct'])
questions[(cat, parts[3], parts[5])].append(row['correct'])
report.update(aAcc=report['accuracy_percent'],
fAcc=100 * mean(all(v) for v in figures.values()),
qAcc=100 * mean(all(v) for v in questions.values()))
if dataset == 'POPE':
report['metric'] = 'accuracy; not F1'
if dataset == 'CVBench':
two_d, three_d = defaultdict(list), []
for row in items:
meta = row['metadata']
if meta['split'] == '2D':
two_d[meta['source']].append(row['correct'])
elif meta['split'] == '3D':
three_d.append(row['correct'])
else:
raise ValueError('CVBench split must be 2D or 3D')
report['2d_source_macro_percent'] = 100 * mean(mean(v) for v in two_d.values()) if two_d else None
report['3d_accuracy_percent'] = 100 * mean(three_d) if three_d else None
report['cvbench_macro_percent'] = (mean([report['2d_source_macro_percent'], report['3d_accuracy_percent']])
if two_d and three_d else None)
reports[dataset] = report
return reports