"""The scores the README reports, checked against values worked out by hand.""" from __future__ import annotations import unittest from banking77 import metrics class WilsonTests(unittest.TestCase): def test_half_of_ten(self): lo, hi = metrics.wilson(5, 10) self.assertAlmostEqual(lo, 0.2366, places=4) self.assertAlmostEqual(hi, 0.7634, places=4) def test_stays_inside_zero_and_one(self): self.assertEqual(metrics.wilson(0, 20)[0], 0.0) self.assertEqual(metrics.wilson(20, 20)[1], 1.0) self.assertGreater(metrics.wilson(20, 20)[0], 0.8) def test_refuses_bad_counts(self): with self.assertRaises(ValueError): metrics.wilson(3, 0) with self.assertRaises(ValueError): metrics.wilson(5, 4) class EceTests(unittest.TestCase): def test_honest_confidence_scores_zero(self): conf = [0.75] * 4 correct = [True, True, True, False] self.assertAlmostEqual(metrics.ece(conf, correct), 0.0) def test_overconfidence_shows_as_the_gap(self): conf = [0.9] * 10 correct = [True] * 6 + [False] * 4 self.assertAlmostEqual(metrics.ece(conf, correct), 0.3) def test_confidence_one_falls_in_the_last_bin(self): self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0) class SelectiveTests(unittest.TestCase): def test_threshold_keeps_only_confident_answers(self): conf = [0.99, 0.95, 0.6, 0.4] correct = [True, True, False, True] s = metrics.selective(conf, correct, 0.9) self.assertEqual(s["answered"], 2) self.assertEqual(s["coverage"], 0.5) self.assertEqual(s["accuracy"], 1.0) self.assertEqual(s["errors"], 0) def test_nothing_answered_has_no_accuracy(self): self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"]) class CoverageTests(unittest.TestCase): def test_largest_prefix_at_target(self): conf = [0.99, 0.98, 0.97, 0.5, 0.4] correct = [True, True, True, False, True] c = metrics.coverage_at_accuracy(conf, correct, 0.95) self.assertEqual(c["answered"], 3) self.assertEqual(c["threshold"], 0.97) def test_a_later_prefix_can_recover(self): conf = [0.9, 0.8, 0.7, 0.6, 0.5] correct = [True, False, True, True, True] self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5) def test_ties_are_kept_or_dropped_together(self): conf = [0.9, 0.9, 0.9, 0.5] correct = [True, True, False, True] self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95)) self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4) def test_none_when_unreachable(self): self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5)) class McNemarTests(unittest.TestCase): def test_no_disagreement(self): self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0) def test_lopsided_disagreement(self): self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024) def test_symmetric(self): self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3)) class CompareTests(unittest.TestCase): def test_counts_only_the_disagreements(self): from banking77.compare import compare a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}] b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}] r = compare(a, b) self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1)) def test_refuses_runs_on_different_tests(self): from banking77.compare import compare with self.assertRaises(ValueError): compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}]) if __name__ == "__main__": unittest.main()