"""Measure how often messages that belong to none of the 77 intents get past the confidence threshold. Banking77 has no "none of these" option, so the model always picks one of the 77 intents. The threshold is the only guard against a message about something else. This script asks the model about the 1,000 test queries that CLINC150 labels out of scope (questions about the stock market, screen brightness, maths and so on) and counts how many it would have answered by itself. A few of those queries read like banking questions, so a high-confidence answer is not always wrong. The count is the number a deployer should plan for, not an exact error rate. Run it with `python -m banking77.offtopic --model --name `. """ from __future__ import annotations import argparse import json from collections import Counter from pathlib import Path from typing import List from banking77.data import data_dir, home, read_jsonl, sha256_file from banking77.evaluate import gate_threshold from banking77.train import HEAD_MAX_LEN, base_dir REPO = "clinc/clinc_oos" REVISION = "155b9c710419136e17307b80d0a13e68cd46b4ec" FILE = "plus/test-00000-of-00001.parquet" SHA256 = "3e60e45b25bf86543aa5df8ba4fcc674114164e6184f0197690648c2908d0102" LICENSE = "CC BY 3.0" ATTRIBUTION = ("Larson et al., An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction, " "EMNLP-IJCNLP 2019. Dataset: clinc/clinc_oos on Hugging Face, CC BY 3.0.") def out_of_scope_texts() -> List[str]: """The out-of-scope queries of CLINC150's test split, in file order, from the pinned and hashed file.""" import pyarrow.parquet as pq from huggingface_hub import hf_hub_download path = Path(hf_hub_download(REPO, FILE, repo_type="dataset", revision=REVISION)) got = sha256_file(path) if got != SHA256: raise RuntimeError("%s: SHA-256 is %s, expected %s" % (FILE, got, SHA256)) table = pq.read_table(path) names = json.loads(table.schema.metadata[b"huggingface"])["info"]["features"]["intent"]["names"] oos = names.index("oos") return [r["text"] for r in table.to_pylist() if r["intent"] == oos] def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--model", required=True, help="a checkpoint folder, or 'base' for untuned Laya") parser.add_argument("--name", required=True, help="run name; results go to $LAYA_B77_HOME/runs//") parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--device", default="cuda") args = parser.parse_args() import laya question = read_jsonl(data_dir() / "test.jsonl")[0]["questions"] model_dir = base_dir() if args.model == "base" else args.model gate = gate_threshold(model_dir) if gate is None: raise SystemExit("%s has no fitted threshold, so there is nothing to test" % model_dir) agent = laya.load(model_dir, device=args.device) texts = out_of_scope_texts() out = agent.predict_batch(texts, question, batch_size=args.batch_size, head_max_len=HEAD_MAX_LEN) conf = [max(r["answers"]["intent"]["probabilities"].values()) for r in out] picks = [r["answers"]["intent"]["choice"] for r in out] passed = [(t, p, c) for t, p, c in zip(texts, picks, conf) if c >= gate] result = { "name": args.name, "source": {"repo": REPO, "revision": REVISION, "file": FILE, "sha256": SHA256, "license": LICENSE, "attribution": ATTRIBUTION}, "threshold": gate, "queries": len(texts), "passed": len(passed), "passed_share": len(passed) / len(texts), "mean_confidence": sum(conf) / len(conf), "passed_as": Counter(p for _, p, _ in passed).most_common(10), "passed_examples": [{"text": t, "answered_as": p, "confidence": c} for t, p, c in sorted(passed, key=lambda x: -x[2])[:15]], } dest = home() / "runs" / args.name dest.mkdir(parents=True, exist_ok=True) with open(dest / "offtopic.json", "w", encoding="utf-8") as f: json.dump(result, f, indent=2) print("%s: %d of %d out-of-scope queries pass the %.4f threshold (%.1f%%); mean confidence %.3f" % (args.name, len(passed), len(texts), gate, 100 * result["passed_share"], result["mean_confidence"])) for p, n in result["passed_as"][:5]: print(" answered as %s: %d" % (p, n)) if __name__ == "__main__": main()