v0.1: first release, 3-class span tagger (lenient / strict), trained on dataset v0.4
5a1c5f9 verified | """Check the install: find unsupported spans in four sample answers (all text invented for this example). | |
| python example.py | |
| """ | |
| import time | |
| from cbx_hallucination import HallucinationDetector | |
| PASSAGE = ("The Larkfield Community Library opened in 1962 on Mill Road. It moved to the former post office on " | |
| "Station Street in 1998. The library lends books, audiobooks and board games, and runs a homework club " | |
| "on Tuesday and Thursday afternoons.") | |
| RECORD = {"name": "Copper Kettle Cafe", "city": "Larkfield", "stars": 4.0, "attributes": {"WiFi": "free", | |
| "OutdoorSeating": None}, "hours": {"Monday": "8:0-15:0", "Tuesday": "8:0-15:0", "Wednesday": "8:0-15:0", | |
| "Thursday": "8:0-15:0", "Friday": "8:0-15:0", "Saturday": "9:0-14:0"}} | |
| SAMPLES = [ | |
| ("qa, faithful", PASSAGE, "When does the homework club meet?", | |
| "The homework club meets on Tuesday and Thursday afternoons."), | |
| ("qa, two errors", PASSAGE, "Tell me about the library's history.", | |
| "The library opened in 1972 on Mill Road and moved to the old post office in 1998, after a fire destroyed " | |
| "the original building."), | |
| ("record, faithful", RECORD, "Write a short description of this cafe.", | |
| "Copper Kettle Cafe in Larkfield has a 4-star rating and free Wi-Fi. It is open 8 AM to 3 PM on weekdays and " | |
| "9 AM to 2 PM on Saturday."), | |
| ("record, errors", RECORD, "Write a short description of this cafe.", | |
| "Copper Kettle Cafe in Larkfield is open every day from 8 AM to 6 PM and has a sunny outdoor terrace."), | |
| ] | |
| det = HallucinationDetector() | |
| print(f"model loaded on {det.device}, threshold {det.threshold}\n") | |
| for name, context, question, answer in SAMPLES: | |
| for mode in ("lenient", "strict"): | |
| t = time.perf_counter() | |
| res = det.detect(context, answer, question=question, mode=mode) | |
| ms = (time.perf_counter() - t) * 1000 | |
| found = "; ".join(f"[{s['type']} {s['score']:.2f}] {s['text']}" for s in res["spans"]) or "no unsupported spans" | |
| print(f"{name:17} {mode:7} -> {found} ({ms:.0f} ms)") | |
| print() | |