Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 3,960 Bytes
4e09985 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | """The scores the README reports, checked against values worked out by hand."""
from __future__ import annotations
import unittest
from banking77 import metrics
class WilsonTests(unittest.TestCase):
def test_half_of_ten(self):
lo, hi = metrics.wilson(5, 10)
self.assertAlmostEqual(lo, 0.2366, places=4)
self.assertAlmostEqual(hi, 0.7634, places=4)
def test_stays_inside_zero_and_one(self):
self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
self.assertGreater(metrics.wilson(20, 20)[0], 0.8)
def test_refuses_bad_counts(self):
with self.assertRaises(ValueError):
metrics.wilson(3, 0)
with self.assertRaises(ValueError):
metrics.wilson(5, 4)
class EceTests(unittest.TestCase):
def test_honest_confidence_scores_zero(self):
conf = [0.75] * 4
correct = [True, True, True, False]
self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)
def test_overconfidence_shows_as_the_gap(self):
conf = [0.9] * 10
correct = [True] * 6 + [False] * 4
self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)
def test_confidence_one_falls_in_the_last_bin(self):
self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)
class SelectiveTests(unittest.TestCase):
def test_threshold_keeps_only_confident_answers(self):
conf = [0.99, 0.95, 0.6, 0.4]
correct = [True, True, False, True]
s = metrics.selective(conf, correct, 0.9)
self.assertEqual(s["answered"], 2)
self.assertEqual(s["coverage"], 0.5)
self.assertEqual(s["accuracy"], 1.0)
self.assertEqual(s["errors"], 0)
def test_nothing_answered_has_no_accuracy(self):
self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])
class CoverageTests(unittest.TestCase):
def test_largest_prefix_at_target(self):
conf = [0.99, 0.98, 0.97, 0.5, 0.4]
correct = [True, True, True, False, True]
c = metrics.coverage_at_accuracy(conf, correct, 0.95)
self.assertEqual(c["answered"], 3)
self.assertEqual(c["threshold"], 0.97)
def test_a_later_prefix_can_recover(self):
conf = [0.9, 0.8, 0.7, 0.6, 0.5]
correct = [True, False, True, True, True]
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)
def test_ties_are_kept_or_dropped_together(self):
conf = [0.9, 0.9, 0.9, 0.5]
correct = [True, True, False, True]
self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)
def test_none_when_unreachable(self):
self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))
class McNemarTests(unittest.TestCase):
def test_no_disagreement(self):
self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)
def test_lopsided_disagreement(self):
self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)
def test_symmetric(self):
self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))
class CompareTests(unittest.TestCase):
def test_counts_only_the_disagreements(self):
from banking77.compare import compare
a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
r = compare(a, b)
self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))
def test_refuses_runs_on_different_tests(self):
from banking77.compare import compare
with self.assertRaises(ValueError):
compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])
if __name__ == "__main__":
unittest.main()
|