tests/test_rules.py

"""The rules are well formed and say the same thing everywhere they appear."""
import pytest

from conftest import load

RULE_NAMES = sorted(load('rules.json')['rules'])


@pytest.mark.parametrize('name', RULE_NAMES)
def test_sets_are_disjoint_and_balanced(name, rules):
    r = rules['rules'][name]
    pos, neg = r['pos_dims'], r['neg_dims']
    assert len(pos) == len(neg), f'{name}: the two sums must have equal term counts'
    assert not set(pos) & set(neg), f'{name}: a dim appears on both sides'
    assert len(set(pos)) == len(pos) and len(set(neg)) == len(neg)
    assert r['n_dims'] == len(pos) + len(neg)


@pytest.mark.parametrize('name', RULE_NAMES)
def test_dims_are_in_range(name, rules):
    for d in rules['rules'][name]['pos_dims'] + rules['rules'][name]['neg_dims']:
        assert 0 <= d < 768


@pytest.mark.parametrize('name', RULE_NAMES)
def test_no_free_parameters(name, rules):
    assert rules['rules'][name]['free_parameters'] == 0


@pytest.mark.parametrize('name', RULE_NAMES)
def test_name_matches_size(name, rules):
    assert name == f'd{rules["rules"][name]["n_dims"]}'


def test_rules_nest(rules):
    """Greedy selection grows the sets, so each rule extends the one below it."""
    by_size = sorted(rules['rules'].values(), key=lambda r: r['n_dims'])
    for small, large in zip(by_size, by_size[1:]):
        assert small['pos_dims'] == large['pos_dims'][:len(small['pos_dims'])]
        assert small['neg_dims'] == large['neg_dims'][:len(small['neg_dims'])]


@pytest.mark.parametrize('name', RULE_NAMES)
def test_eval_carries_the_same_dims(name, rules, evaluation):
    r, e = rules['rules'][name], evaluation['rules'][name]
    assert e['pos_dims'] == r['pos_dims']
    assert e['neg_dims'] == r['neg_dims']
    assert e['F1_train'] == r['F1_train']


def test_more_dims_do_not_score_worse(evaluation):
    by_size = sorted(evaluation['rules'].values(), key=lambda r: r['n_dims'])
    f1 = [r['F1'] for r in by_size]
    assert f1 == sorted(f1), f'F1 is not monotone in dim count: {f1}'


def test_val_tracks_train(evaluation):
    """Selection on 118k images should not overfit; val must stay close to train."""
    for name, r in evaluation['rules'].items():
        assert abs(r['F1'] - r['F1_train']) < 0.02, \
            f'{name}: train {r["F1_train"]} against val {r["F1"]}'