Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 1,891 Bytes
4e09985 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """Compare two scored runs message by message, with the exact McNemar test.
Two models scored on the same test can be compared more sharply than their two accuracies suggest: only the
messages where exactly one of them is right carry information about which is better.
Run it with `python -m banking77.compare zero-shot seed0`.
"""
from __future__ import annotations
import argparse
import json
from typing import Dict, List
from banking77.data import home
from banking77.metrics import mcnemar_exact
def load(name: str) -> List[Dict]:
with open(home() / "runs" / name / "predictions.jsonl", encoding="utf-8") as f:
return [json.loads(line) for line in f if line.strip()]
def compare(a: List[Dict], b: List[Dict]) -> Dict[str, float]:
if len(a) != len(b) or any(x["gold"] != y["gold"] for x, y in zip(a, b)):
raise ValueError("the two runs were not scored on the same test")
ra = [x["pred"] == x["gold"] for x in a]
rb = [y["pred"] == y["gold"] for y in b]
only_a = sum(1 for x, y in zip(ra, rb) if x and not y)
only_b = sum(1 for x, y in zip(ra, rb) if y and not x)
return {"n": len(a), "a_right": sum(ra), "b_right": sum(rb), "only_a": only_a, "only_b": only_b,
"p": mcnemar_exact(only_a, only_b)}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("a")
parser.add_argument("b")
args = parser.parse_args()
r = compare(load(args.a), load(args.b))
# Past about 1,100 disagreements the exact p-value is smaller than a float can hold.
p = "p = %.3g" % r["p"] if r["p"] > 0 else "p < 1e-300"
print("%s %d right, %s %d right of %d; only %s right %d, only %s right %d; exact McNemar %s"
% (args.a, r["a_right"], args.b, r["b_right"], r["n"], args.a, r["only_a"], args.b, r["only_b"], p))
if __name__ == "__main__":
main()
|