laya-banking77 / code /tests /test_metrics.py
VibeHash's picture
Add laya-banking77: Laya fine-tuned on Banking77, its scores and the code that made it
4e09985 verified
Raw History Blame Contribute Delete
3.96 kB
"""The scores the README reports, checked against values worked out by hand."""
from __future__ import annotations
import unittest
from banking77 import metrics
class WilsonTests(unittest.TestCase):
def test_half_of_ten(self):
lo, hi = metrics.wilson(5, 10)
self.assertAlmostEqual(lo, 0.2366, places=4)
self.assertAlmostEqual(hi, 0.7634, places=4)
def test_stays_inside_zero_and_one(self):
self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
self.assertGreater(metrics.wilson(20, 20)[0], 0.8)
def test_refuses_bad_counts(self):
with self.assertRaises(ValueError):
metrics.wilson(3, 0)
with self.assertRaises(ValueError):
metrics.wilson(5, 4)
class EceTests(unittest.TestCase):
def test_honest_confidence_scores_zero(self):
conf = [0.75] * 4
correct = [True, True, True, False]
self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)
def test_overconfidence_shows_as_the_gap(self):
conf = [0.9] * 10
correct = [True] * 6 + [False] * 4
self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)
def test_confidence_one_falls_in_the_last_bin(self):
self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)
class SelectiveTests(unittest.TestCase):
def test_threshold_keeps_only_confident_answers(self):
conf = [0.99, 0.95, 0.6, 0.4]
correct = [True, True, False, True]
s = metrics.selective(conf, correct, 0.9)
self.assertEqual(s["answered"], 2)
self.assertEqual(s["coverage"], 0.5)
self.assertEqual(s["accuracy"], 1.0)
self.assertEqual(s["errors"], 0)
def test_nothing_answered_has_no_accuracy(self):
self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])
class CoverageTests(unittest.TestCase):
def test_largest_prefix_at_target(self):
conf = [0.99, 0.98, 0.97, 0.5, 0.4]
correct = [True, True, True, False, True]
c = metrics.coverage_at_accuracy(conf, correct, 0.95)
self.assertEqual(c["answered"], 3)
self.assertEqual(c["threshold"], 0.97)
def test_a_later_prefix_can_recover(self):
conf = [0.9, 0.8, 0.7, 0.6, 0.5]
correct = [True, False, True, True, True]
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)
def test_ties_are_kept_or_dropped_together(self):
conf = [0.9, 0.9, 0.9, 0.5]
correct = [True, True, False, True]
self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)
def test_none_when_unreachable(self):
self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))
class McNemarTests(unittest.TestCase):
def test_no_disagreement(self):
self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)
def test_lopsided_disagreement(self):
self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)
def test_symmetric(self):
self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))
class CompareTests(unittest.TestCase):
def test_counts_only_the_disagreements(self):
from banking77.compare import compare
a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
r = compare(a, b)
self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))
def test_refuses_runs_on_different_tests(self):
from banking77.compare import compare
with self.assertRaises(ValueError):
compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])
if __name__ == "__main__":
unittest.main()