File size: 1,537 Bytes
e8b8483 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Every committed artifact matches the schema its generator declares."""
import pytest
from common.artifacts import REGISTRY
from conftest import REPO, load
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_exists(rel):
assert (REPO / rel).exists(), f'{rel} is registered but missing'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_opens_with_provenance(rel):
doc = load(rel)
assert next(iter(doc)) == 'provenance', f'{rel} does not open with provenance'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_payload_keys(rel):
spec = REGISTRY[rel]
got = tuple(k for k in load(rel) if k != 'provenance')
assert got == spec.payload_keys, f'{rel} payload {got} != declared {spec.payload_keys}'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_generator(rel):
assert load(rel)['provenance']['generator'] == REGISTRY[rel].generator
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_pool(rel):
spec = REGISTRY[rel]
pool = load(rel)['provenance'].get('pool')
if spec.pool is not None:
assert pool == spec.pool, f'{rel} names pool {pool!r}, registry says {spec.pool!r}'
def test_selection_and_evaluation_read_different_splits():
"""No figure may be measured on the images that chose it."""
sel = load('rules.json')['provenance']
ev = load('eval.json')['provenance']
assert sel['split'] == 'train2017'
assert ev['split'] == 'val2017'
assert sel['split'] != ev['split']
|