File size: 2,324 Bytes
e8b8483 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """The rules are well formed and say the same thing everywhere they appear."""
import pytest
from conftest import load
RULE_NAMES = sorted(load('rules.json')['rules'])
@pytest.mark.parametrize('name', RULE_NAMES)
def test_sets_are_disjoint_and_balanced(name, rules):
r = rules['rules'][name]
pos, neg = r['pos_dims'], r['neg_dims']
assert len(pos) == len(neg), f'{name}: the two sums must have equal term counts'
assert not set(pos) & set(neg), f'{name}: a dim appears on both sides'
assert len(set(pos)) == len(pos) and len(set(neg)) == len(neg)
assert r['n_dims'] == len(pos) + len(neg)
@pytest.mark.parametrize('name', RULE_NAMES)
def test_dims_are_in_range(name, rules):
for d in rules['rules'][name]['pos_dims'] + rules['rules'][name]['neg_dims']:
assert 0 <= d < 768
@pytest.mark.parametrize('name', RULE_NAMES)
def test_no_free_parameters(name, rules):
assert rules['rules'][name]['free_parameters'] == 0
@pytest.mark.parametrize('name', RULE_NAMES)
def test_name_matches_size(name, rules):
assert name == f'd{rules["rules"][name]["n_dims"]}'
def test_rules_nest(rules):
"""Greedy selection grows the sets, so each rule extends the one below it."""
by_size = sorted(rules['rules'].values(), key=lambda r: r['n_dims'])
for small, large in zip(by_size, by_size[1:]):
assert small['pos_dims'] == large['pos_dims'][:len(small['pos_dims'])]
assert small['neg_dims'] == large['neg_dims'][:len(small['neg_dims'])]
@pytest.mark.parametrize('name', RULE_NAMES)
def test_eval_carries_the_same_dims(name, rules, evaluation):
r, e = rules['rules'][name], evaluation['rules'][name]
assert e['pos_dims'] == r['pos_dims']
assert e['neg_dims'] == r['neg_dims']
assert e['F1_train'] == r['F1_train']
def test_more_dims_do_not_score_worse(evaluation):
by_size = sorted(evaluation['rules'].values(), key=lambda r: r['n_dims'])
f1 = [r['F1'] for r in by_size]
assert f1 == sorted(f1), f'F1 is not monotone in dim count: {f1}'
def test_val_tracks_train(evaluation):
"""Selection on 118k images should not overfit; val must stay close to train."""
for name, r in evaluation['rules'].items():
assert abs(r['F1'] - r['F1_train']) < 0.02, \
f'{name}: train {r["F1_train"]} against val {r["F1"]}'
|