Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/banking77/metrics.py from VibeHash/laya-banking77: direct link, hf CLI and curl.
- Browser
- Download file 3.89 kB
-
https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/metrics.py
- Command line
-
hf download hf://VibeHash/laya-banking77/code/banking77/metrics.py
-
curl -L -o metrics.py https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/metrics.py
3.89 kB
| """Scores for a model that states how sure it is: accuracy with its uncertainty, calibration, and coverage. | |
| Plain Python on purpose. These functions decide what the README claims, so they should be easy to read and to | |
| check by hand, and they should not depend on the training stack. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import Dict, List, Optional, Sequence, Tuple | |
| Z95 = 1.959963984540054 | |
| def wilson(k: int, n: int, z: float = Z95) -> Tuple[float, float]: | |
| """The Wilson 95% interval for `k` right out of `n`. | |
| It stays inside [0, 1] and behaves near 0% and 100%, where the textbook normal interval does not. | |
| """ | |
| if n <= 0: | |
| raise ValueError("n must be positive") | |
| if not 0 <= k <= n: | |
| raise ValueError("k must be between 0 and n") | |
| p = k / n | |
| denom = 1 + z * z / n | |
| centre = (p + z * z / (2 * n)) / denom | |
| half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom | |
| return max(0.0, centre - half), min(1.0, centre + half) | |
| def ece(confidences: Sequence[float], correct: Sequence[bool], bins: int = 15) -> float: | |
| """Expected calibration error: the gap between stated confidence and accuracy, averaged over equal-width bins. | |
| A model that says 0.9 and is right 90% of the time scores 0. Confidence 1.0 falls in the last bin. | |
| """ | |
| if len(confidences) != len(correct): | |
| raise ValueError("confidences and correct differ in length") | |
| n = len(confidences) | |
| if n == 0: | |
| raise ValueError("no predictions") | |
| sums: List[List[float]] = [[0, 0.0, 0.0] for _ in range(bins)] | |
| for c, ok in zip(confidences, correct): | |
| b = min(int(c * bins), bins - 1) | |
| sums[b][0] += 1 | |
| sums[b][1] += c | |
| sums[b][2] += 1.0 if ok else 0.0 | |
| return sum(abs(s[1] - s[2]) for s in sums if s[0]) / n | |
| def selective(confidences: Sequence[float], correct: Sequence[bool], threshold: float) -> Dict[str, float]: | |
| """What happens when the model answers only at or above `threshold` and hands the rest to a person.""" | |
| kept = [ok for c, ok in zip(confidences, correct) if c >= threshold] | |
| n = len(confidences) | |
| return {"threshold": threshold, "answered": len(kept), "coverage": len(kept) / n, | |
| "accuracy": (sum(kept) / len(kept)) if kept else None, | |
| "errors": len(kept) - sum(kept)} | |
| def coverage_at_accuracy(confidences: Sequence[float], correct: Sequence[bool], | |
| target: float) -> Optional[Dict[str, float]]: | |
| """The most messages the model could answer, most confident first, while staying at `target` accuracy or above. | |
| The cut is chosen on the same predictions it is scored on, so it describes this data and is optimistic for new | |
| data. Use it to draw the coverage curve; use `selective` with a threshold fixed beforehand to make a claim. | |
| A cut may only fall between two different confidences: messages with equal confidence are kept or dropped | |
| together, as a real threshold would. Returns None when no cut reaches the target. | |
| """ | |
| order = sorted(range(len(confidences)), key=lambda i: -confidences[i]) | |
| best = None | |
| right = 0 | |
| for pos, i in enumerate(order): | |
| right += 1 if correct[i] else 0 | |
| last_of_tie = pos + 1 == len(order) or confidences[order[pos + 1]] < confidences[i] | |
| if last_of_tie and right / (pos + 1) >= target: | |
| best = {"target": target, "threshold": confidences[i], "answered": pos + 1, | |
| "coverage": (pos + 1) / len(order), "accuracy": right / (pos + 1)} | |
| return best | |
| def mcnemar_exact(only_a: int, only_b: int) -> float: | |
| """Two-sided exact McNemar p-value from the two kinds of disagreement between paired models.""" | |
| n = only_a + only_b | |
| if n == 0: | |
| return 1.0 | |
| k = min(only_a, only_b) | |
| tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n | |
| return min(1.0, 2 * tail) | |