SCOPD / evaluation /scopd_eval /datasets.py
enmingzhangzz's picture
Publish evaluation-and-model-card
ab18ce0 verified
Raw History Blame Contribute Delete
3.49 kB
"""The image benchmarks used in the main tables, with explicit subset names."""
import ast
import math
import re
import string
DATASETS = {
'mme': 'MME', 'mmstar': 'MMStar', 'mathvista': 'MathVista_MINI',
'mathverse': 'MathVerse_MINI_Vision_Only', 'mmmupro': 'MMMU_Pro_4c',
'hallusion': 'HallusionBench', 'cvbench': 'CVBench', 'logicvista': 'LogicVista',
'blink': 'BLINK', 'visonly': 'VisOnlyQA-VLMEvalKit', 'hr4k': 'HRBench4K',
'mmvp': 'MMVP', 'mmereal_lite': 'MME-RealWorld-Lite', 'realworldqa': 'RealWorldQA',
'pope': 'POPE', 'mathvision': 'MathVision_MINI',
}
def present(value):
return value is not None and not (isinstance(value, float) and math.isnan(value)) and str(value).strip() != ''
def options_for(row):
options = {key: str(row[key]) for key in string.ascii_uppercase if key in row and present(row[key])}
if options:
return options
if present(row.get('choices')):
choices = row['choices']
if isinstance(choices, str):
choices = ast.literal_eval(choices)
if isinstance(choices, list):
return {string.ascii_uppercase[i]: str(v) for i, v in enumerate(choices)}
return {}
def make_record(dataset, row, response, images):
reference = str(row['answer'])
options = options_for(row)
original_options = str(row.get('multi-choice options', ''))
if dataset.startswith('MME-RealWorld'):
labels = re.findall(r'(?m)^\s*\(?([A-E])[).]', original_options)
if len(labels) != len(set(labels)):
# Preserve the original duplicate labels; do not silently reinterpret
# the positional E column as a repaired option in the judge prompt.
options = {key: key for key in 'ABCDE'}
if dataset == 'MathVista_MINI' and present(row.get('answer_option')):
reference = str(row['answer_option'])
if dataset == 'HallusionBench':
reference = {'0': 'No', '1': 'Yes', '0.0': 'No', '1.0': 'Yes'}.get(reference, reference)
if isinstance(response, dict):
extra = response.get('extra_records', {})
raw = extra.get('raw_prediction')
if raw is None:
raise ValueError('Raw model output was not preserved')
prediction = response['prediction']
tokens = extra.get('generated_token_len')
else:
raw, prediction, tokens = str(response), str(response), None
metadata = {}
for field in ('category', 'l2-category', 'task', 'answer_type', 'precision', 'split', 'type', 'source'):
if present(row.get(field)):
metadata[field] = str(row[field])
if dataset == 'MME':
metadata['image_path'] = str(row.get('image_path', images[0]))
return {'sample_id': str(row['index']), 'dataset': dataset, 'question': str(row['question']),
'reference': reference, 'options': options, 'images': [str(p) for p in images],
'raw_prediction': raw, 'prediction': prediction, 'generated_tokens': tokens,
'metadata': metadata, 'original_options_text': original_options}
def load(key):
from vlmeval.dataset import build_dataset, ConcatDataset
if key == 'cvbench':
ConcatDataset.DATASET_SETS['CVBench'] = ['CV-Bench-2D', 'CV-Bench-3D']
dataset = build_dataset(DATASETS[key])
if dataset is None:
raise ValueError(f'Unknown pinned dataset: {DATASETS[key]}')
if not dataset.data['index'].astype(str).is_unique:
raise ValueError('Duplicate benchmark IDs')
return dataset