import base64
import json
from pathlib import Path
import pytest
from scopd_eval import postprocess
from scopd_eval.datasets import make_record
from scopd_eval.infer import visionzip_settings
from scopd_eval.protocol import DEFAULT_MODEL, candidate_answer, exact_match, make_request
from scopd_eval.scoring import summarize
IMAGE = 'data:image/png;base64,' + base64.b64encode(b'test-image-bytes').decode()
def record(answer='B', candidate='A is temptingB', **extra):
return {'dataset': 'MMStar', 'sample_id': '1', 'question': 'Which object?',
'options': {'A': 'red cube', 'B': 'blue sphere'}, 'reference': answer,
'images': [IMAGE], 'raw_prediction': candidate, **extra}
@pytest.mark.parametrize('candidate,match', [
('AB', True),
('A', False), ('blue sphere', True),
('B or A', False), ('I think A, not B', False),
('the answer is B', False), ('B', False),
('AB', False),
('B Actually A', False),
])
def test_match_is_strict(candidate, match):
assert exact_match(record(candidate=candidate)) is match
def test_numeric_equivalence_without_substring_matching():
assert exact_match(record(answer='0.5', candidate='1/2', options={}))
assert not exact_match(record(answer='0.5', candidate='0.5 or 0.6', options={}))
assert not exact_match(record(answer='5', candidate='15', options={}))
def test_judge_sees_question_gt_candidate_and_all_images():
row = record(candidate='the sphere', images=[IMAGE, IMAGE])
payload, key, identity = make_request(row, '.', revision='revision1')
assert payload['model'] == DEFAULT_MODEL
content = payload['messages'][1]['content']
context = json.loads(content[0]['text'])
assert context['reference_answer'] == 'B' and context['candidate_final_answer'] == 'the sphere'
assert context['question'] == row['question'] and context['options'] == row['options']
assert [x['type'] for x in content] == ['text', 'image_url', 'image_url']
assert 'A is tempting' not in content[0]['text']
assert len(key) == 64 and len(identity['image_sha256']) == 2
assert payload['chat_template_kwargs']['enable_thinking'] is False
def test_cache_changes_with_gt_image_revision_and_candidate():
row = record()
key = make_request(row, '.', revision='a')[1]
for changed in [dict(row, reference='A'), dict(row, raw_prediction='A'),
dict(row, images=['data:image/png;base64,' + base64.b64encode(b'other').decode()])]:
assert make_request(changed, '.', revision='a')[1] != key
assert make_request(row, '.', revision='b')[1] != key
def test_missing_image_fails_closed(tmp_path):
with pytest.raises(FileNotFoundError):
make_request(record(images=['missing.png']), tmp_path, revision='a')
with pytest.raises(ValueError):
make_request(record(images=[]), tmp_path, revision='a')
def test_only_mismatches_call_judge_and_resume(tmp_path, monkeypatch):
rows = [record(), record(candidate='the sphere', sample_id='2'),
record(candidate='A', sample_id='3')]
path = tmp_path / 'predictions.jsonl'
path.write_text(''.join(json.dumps(row) + '\n' for row in rows))
calls = []
def query(url, payload):
text = json.loads(payload['messages'][1]['content'][0]['text'])['candidate_final_answer']
calls.append(text)
return text == 'the sphere', {'mock': True}
monkeypatch.setattr(postprocess, 'query', query)
kwargs = dict(image_root=tmp_path, base_url='http://localhost', model=DEFAULT_MODEL, revision='pin')
result = postprocess.run(path, tmp_path / 'out', **kwargs)
assert sorted(calls) == ['A', 'the sphere']
assert result['fallback_total'] == 2
assert result['scores']['MMStar']['accuracy_percent'] == pytest.approx(200 / 3)
postprocess.run(path, tmp_path / 'out', **kwargs)
assert len(calls) == 2
with pytest.raises(ValueError):
postprocess.run(path, tmp_path / 'out', **dict(kwargs, revision='other'))
def test_failure_not_converted_to_incorrect_score(tmp_path, monkeypatch):
path = tmp_path / 'predictions.jsonl'
path.write_text(json.dumps(record(candidate='unknown')) + '\n')
def query(*args):
raise TimeoutError('test failure')
monkeypatch.setattr(postprocess, 'query', query)
with pytest.raises(RuntimeError):
postprocess.run(path, tmp_path / 'out', image_root=tmp_path, base_url='http://localhost',
model=DEFAULT_MODEL, revision='pin')
summary = json.loads((tmp_path / 'out/summary.json').read_text())
assert summary['status'] == 'incomplete' and 'scores' not in summary
def test_mme_pair_and_mmvp_pair_scoring():
rows = [dict(record(sample_id=str(i), dataset='MME'), correct=i != 2, exact_match=i != 2,
metadata={'category': 'count', 'image_path': str(i // 2)}) for i in range(4)]
assert summarize(rows)['MME']['mme_score'] == 125
pairs = [dict(r, dataset='MMVP', sample_id=str(i + 1)) for i, r in enumerate(rows)]
assert summarize(pairs)['MMVP']['pair_accuracy_percent'] == 50
@pytest.mark.parametrize('retention,backend', [(.1, .95), (.2, .85), (.3, .75), (1., 0.)])
def test_retention_is_not_backend_ratio(retention, backend):
assert visionzip_settings(retention)['visionzip_ratio'] == backend
def test_hallusion_boolean_reference():
row = make_record('HallusionBench', {'index': 'x', 'question': 'Is it blue?', 'answer': 1}, 'Yes', ['x.png'])
assert row['reference'] == 'Yes' and exact_match(row)
def test_cvbench_keeps_source_macro_then_2d_3d_macro():
rows = []
for i, (source, split, correct) in enumerate([('a', '2D', True), ('b', '2D', False),
('b', '2D', False), ('b', '2D', False),
('x', '3D', True)]):
rows.append(dict(record(dataset='CVBench', sample_id=str(i)), correct=correct, exact_match=correct,
metadata={'source': source, 'split': split}))
report = summarize(rows)['CVBench']
assert report['2d_source_macro_percent'] == 50
assert report['3d_accuracy_percent'] == 100
assert report['cvbench_macro_percent'] == 75
def test_mmereal_duplicate_option_labels_are_not_repaired():
source = {'index': 1, 'question': 'Which?', 'answer': 'E',
'A': 'first', 'B': 'second', 'C': 'third', 'D': 'fourth', 'E': 'fifth',
'multi-choice options': '(A) first\n(B) second\n(C) third\n(D) fourth\n(D) fifth'}
row = make_record('MME-RealWorld-Lite', source, 'fifth', [IMAGE])
assert row['options']['E'] == 'E' and not exact_match(row)
payload, _, _ = make_request(row, '.', revision='pin')
assert '(D) fifth' in payload['messages'][1]['content'][0]['text']