Text Classification
Laya
Safetensors
English
banking77
intent-classification
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-banking77 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-banking77 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/banking77/offtopic.py from VibeHash/laya-banking77: direct link, hf CLI and curl.
- Browser
- Download file 4.46 kB
-
https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/offtopic.py
- Command line
-
hf download hf://VibeHash/laya-banking77/code/banking77/offtopic.py
-
curl -L -o offtopic.py https://huggingface.co/VibeHash/laya-banking77/resolve/main/code/banking77/offtopic.py
4.46 kB
| """Measure how often messages that belong to none of the 77 intents get past the confidence threshold. | |
| Banking77 has no "none of these" option, so the model always picks one of the 77 intents. The threshold is the only | |
| guard against a message about something else. This script asks the model about the 1,000 test queries that CLINC150 | |
| labels out of scope (questions about the stock market, screen brightness, maths and so on) and counts how many it | |
| would have answered by itself. | |
| A few of those queries read like banking questions, so a high-confidence answer is not always wrong. The count is the | |
| number a deployer should plan for, not an exact error rate. | |
| Run it with `python -m banking77.offtopic --model <dir> --name <run>`. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from collections import Counter | |
| from pathlib import Path | |
| from typing import List | |
| from banking77.data import data_dir, home, read_jsonl, sha256_file | |
| from banking77.evaluate import gate_threshold | |
| from banking77.train import HEAD_MAX_LEN, base_dir | |
| REPO = "clinc/clinc_oos" | |
| REVISION = "155b9c710419136e17307b80d0a13e68cd46b4ec" | |
| FILE = "plus/test-00000-of-00001.parquet" | |
| SHA256 = "3e60e45b25bf86543aa5df8ba4fcc674114164e6184f0197690648c2908d0102" | |
| LICENSE = "CC BY 3.0" | |
| ATTRIBUTION = ("Larson et al., An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction, " | |
| "EMNLP-IJCNLP 2019. Dataset: clinc/clinc_oos on Hugging Face, CC BY 3.0.") | |
| def out_of_scope_texts() -> List[str]: | |
| """The out-of-scope queries of CLINC150's test split, in file order, from the pinned and hashed file.""" | |
| import pyarrow.parquet as pq | |
| from huggingface_hub import hf_hub_download | |
| path = Path(hf_hub_download(REPO, FILE, repo_type="dataset", revision=REVISION)) | |
| got = sha256_file(path) | |
| if got != SHA256: | |
| raise RuntimeError("%s: SHA-256 is %s, expected %s" % (FILE, got, SHA256)) | |
| table = pq.read_table(path) | |
| names = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"]["intent"]["names"] | |
| oos = names.index("oos") | |
| return [r["text"] for r in table.to_pylist() if r["intent"] == oos] | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya") | |
| parser.add_argument("--name", required=True, help="run name; results go to $LAYA_B77_HOME/runs/<name>/") | |
| parser.add_argument("--batch-size", type=int, default=32) | |
| parser.add_argument("--device", default="cuda") | |
| args = parser.parse_args() | |
| import laya | |
| question = read_jsonl(data_dir() / "test.jsonl")[0]["questions"] | |
| model_dir = base_dir() if args.model == "base" else args.model | |
| gate = gate_threshold(model_dir) | |
| if gate is None: | |
| raise SystemExit("%s has no fitted threshold, so there is nothing to test" % model_dir) | |
| agent = laya.load(model_dir, device=args.device) | |
| texts = out_of_scope_texts() | |
| out = agent.predict_batch(texts, question, batch_size=args.batch_size, head_max_len=HEAD_MAX_LEN) | |
| conf = [max(r["answers"]["intent"]["probabilities"].values()) for r in out] | |
| picks = [r["answers"]["intent"]["choice"] for r in out] | |
| passed = [(t, p, c) for t, p, c in zip(texts, picks, conf) if c >= gate] | |
| result = { | |
| "name": args.name, "source": {"repo": REPO, "revision": REVISION, "file": FILE, "sha256": SHA256, | |
| "license": LICENSE, "attribution": ATTRIBUTION}, | |
| "threshold": gate, "queries": len(texts), "passed": len(passed), "passed_share": len(passed) / len(texts), | |
| "mean_confidence": sum(conf) / len(conf), | |
| "passed_as": Counter(p for _, p, _ in passed).most_common(10), | |
| "passed_examples": [{"text": t, "answered_as": p, "confidence": c} | |
| for t, p, c in sorted(passed, key=lambda x: -x[2])[:15]], | |
| } | |
| dest = home() / "runs" / args.name | |
| dest.mkdir(parents=True, exist_ok=True) | |
| with open(dest / "offtopic.json", "w", encoding="utf-8") as f: | |
| json.dump(result, f, indent=2) | |
| print("%s: %d of %d out-of-scope queries pass the %.4f threshold (%.1f%%); mean confidence %.3f" | |
| % (args.name, len(passed), len(texts), gate, 100 * result["passed_share"], result["mean_confidence"])) | |
| for p, n in result["passed_as"][:5]: | |
| print(" answered as %s: %d" % (p, n)) | |
| if __name__ == "__main__": | |
| main() | |