Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/tests/test_metrics.py from VibeHash/laya-banking77: direct link, hf CLI and curl.
- Browser
- Download file 3.96 kB
-
https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/tests/test_metrics.py
- Command line
-
hf download hf://VibeHash/laya-banking77/code/tests/test_metrics.py
-
curl -L -o test_metrics.py https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/tests/test_metrics.py
3.96 kB
| """The scores the README reports, checked against values worked out by hand.""" | |
| from __future__ import annotations | |
| import unittest | |
| from banking77 import metrics | |
| class WilsonTests(unittest.TestCase): | |
| def test_half_of_ten(self): | |
| lo, hi = metrics.wilson(5, 10) | |
| self.assertAlmostEqual(lo, 0.2366, places=4) | |
| self.assertAlmostEqual(hi, 0.7634, places=4) | |
| def test_stays_inside_zero_and_one(self): | |
| self.assertEqual(metrics.wilson(0, 20)[0], 0.0) | |
| self.assertEqual(metrics.wilson(20, 20)[1], 1.0) | |
| self.assertGreater(metrics.wilson(20, 20)[0], 0.8) | |
| def test_refuses_bad_counts(self): | |
| with self.assertRaises(ValueError): | |
| metrics.wilson(3, 0) | |
| with self.assertRaises(ValueError): | |
| metrics.wilson(5, 4) | |
| class EceTests(unittest.TestCase): | |
| def test_honest_confidence_scores_zero(self): | |
| conf = [0.75] * 4 | |
| correct = [True, True, True, False] | |
| self.assertAlmostEqual(metrics.ece(conf, correct), 0.0) | |
| def test_overconfidence_shows_as_the_gap(self): | |
| conf = [0.9] * 10 | |
| correct = [True] * 6 + [False] * 4 | |
| self.assertAlmostEqual(metrics.ece(conf, correct), 0.3) | |
| def test_confidence_one_falls_in_the_last_bin(self): | |
| self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0) | |
| class SelectiveTests(unittest.TestCase): | |
| def test_threshold_keeps_only_confident_answers(self): | |
| conf = [0.99, 0.95, 0.6, 0.4] | |
| correct = [True, True, False, True] | |
| s = metrics.selective(conf, correct, 0.9) | |
| self.assertEqual(s["answered"], 2) | |
| self.assertEqual(s["coverage"], 0.5) | |
| self.assertEqual(s["accuracy"], 1.0) | |
| self.assertEqual(s["errors"], 0) | |
| def test_nothing_answered_has_no_accuracy(self): | |
| self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"]) | |
| class CoverageTests(unittest.TestCase): | |
| def test_largest_prefix_at_target(self): | |
| conf = [0.99, 0.98, 0.97, 0.5, 0.4] | |
| correct = [True, True, True, False, True] | |
| c = metrics.coverage_at_accuracy(conf, correct, 0.95) | |
| self.assertEqual(c["answered"], 3) | |
| self.assertEqual(c["threshold"], 0.97) | |
| def test_a_later_prefix_can_recover(self): | |
| conf = [0.9, 0.8, 0.7, 0.6, 0.5] | |
| correct = [True, False, True, True, True] | |
| self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5) | |
| def test_ties_are_kept_or_dropped_together(self): | |
| conf = [0.9, 0.9, 0.9, 0.5] | |
| correct = [True, True, False, True] | |
| self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95)) | |
| self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4) | |
| def test_none_when_unreachable(self): | |
| self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5)) | |
| class McNemarTests(unittest.TestCase): | |
| def test_no_disagreement(self): | |
| self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0) | |
| def test_lopsided_disagreement(self): | |
| self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024) | |
| def test_symmetric(self): | |
| self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3)) | |
| class CompareTests(unittest.TestCase): | |
| def test_counts_only_the_disagreements(self): | |
| from banking77.compare import compare | |
| a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}] | |
| b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}] | |
| r = compare(a, b) | |
| self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1)) | |
| def test_refuses_runs_on_different_tests(self): | |
| from banking77.compare import compare | |
| with self.assertRaises(ValueError): | |
| compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}]) | |
| if __name__ == "__main__": | |
| unittest.main() | |