tests/test_artifacts.py

"""Every committed artifact matches the schema its generator declares."""
import pytest

from common.artifacts import REGISTRY
from conftest import REPO, load


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_exists(rel):
    assert (REPO / rel).exists(), f'{rel} is registered but missing'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_opens_with_provenance(rel):
    doc = load(rel)
    assert next(iter(doc)) == 'provenance', f'{rel} does not open with provenance'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_payload_keys(rel):
    spec = REGISTRY[rel]
    got = tuple(k for k in load(rel) if k != 'provenance')
    assert got == spec.payload_keys, f'{rel} payload {got} != declared {spec.payload_keys}'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_generator(rel):
    assert load(rel)['provenance']['generator'] == REGISTRY[rel].generator


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_pool(rel):
    spec = REGISTRY[rel]
    pool = load(rel)['provenance'].get('pool')
    if spec.pool is not None:
        assert pool == spec.pool, f'{rel} names pool {pool!r}, registry says {spec.pool!r}'


def test_selection_and_evaluation_read_different_splits():
    """No figure may be measured on the images that chose it."""
    sel = load('rules.json')['provenance']
    ev = load('eval.json')['provenance']
    assert sel['split'] == 'train2017'
    assert ev['split'] == 'val2017'
    assert sel['split'] != ev['split']