File size: 3,960 Bytes
4e09985
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
"""The scores the README reports, checked against values worked out by hand."""

from __future__ import annotations

import unittest

from banking77 import metrics


class WilsonTests(unittest.TestCase):
    def test_half_of_ten(self):
        lo, hi = metrics.wilson(5, 10)
        self.assertAlmostEqual(lo, 0.2366, places=4)
        self.assertAlmostEqual(hi, 0.7634, places=4)

    def test_stays_inside_zero_and_one(self):
        self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
        self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
        self.assertGreater(metrics.wilson(20, 20)[0], 0.8)

    def test_refuses_bad_counts(self):
        with self.assertRaises(ValueError):
            metrics.wilson(3, 0)
        with self.assertRaises(ValueError):
            metrics.wilson(5, 4)


class EceTests(unittest.TestCase):
    def test_honest_confidence_scores_zero(self):
        conf = [0.75] * 4
        correct = [True, True, True, False]
        self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)

    def test_overconfidence_shows_as_the_gap(self):
        conf = [0.9] * 10
        correct = [True] * 6 + [False] * 4
        self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)

    def test_confidence_one_falls_in_the_last_bin(self):
        self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)


class SelectiveTests(unittest.TestCase):
    def test_threshold_keeps_only_confident_answers(self):
        conf = [0.99, 0.95, 0.6, 0.4]
        correct = [True, True, False, True]
        s = metrics.selective(conf, correct, 0.9)
        self.assertEqual(s["answered"], 2)
        self.assertEqual(s["coverage"], 0.5)
        self.assertEqual(s["accuracy"], 1.0)
        self.assertEqual(s["errors"], 0)

    def test_nothing_answered_has_no_accuracy(self):
        self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])


class CoverageTests(unittest.TestCase):
    def test_largest_prefix_at_target(self):
        conf = [0.99, 0.98, 0.97, 0.5, 0.4]
        correct = [True, True, True, False, True]
        c = metrics.coverage_at_accuracy(conf, correct, 0.95)
        self.assertEqual(c["answered"], 3)
        self.assertEqual(c["threshold"], 0.97)

    def test_a_later_prefix_can_recover(self):
        conf = [0.9, 0.8, 0.7, 0.6, 0.5]
        correct = [True, False, True, True, True]
        self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)

    def test_ties_are_kept_or_dropped_together(self):
        conf = [0.9, 0.9, 0.9, 0.5]
        correct = [True, True, False, True]
        self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
        self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)

    def test_none_when_unreachable(self):
        self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))


class McNemarTests(unittest.TestCase):
    def test_no_disagreement(self):
        self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)

    def test_lopsided_disagreement(self):
        self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)

    def test_symmetric(self):
        self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))


class CompareTests(unittest.TestCase):
    def test_counts_only_the_disagreements(self):
        from banking77.compare import compare

        a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
        b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
        r = compare(a, b)
        self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))

    def test_refuses_runs_on_different_tests(self):
        from banking77.compare import compare

        with self.assertRaises(ValueError):
            compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])


if __name__ == "__main__":
    unittest.main()