File size: 2,090 Bytes
c6f57cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | """Check the install: find unsupported spans in four sample answers (all text invented for this example).
python example.py
"""
import time
from cbx_hallucination import HallucinationDetector
PASSAGE = ("The Larkfield Community Library opened in 1962 on Mill Road. It moved to the former post office on "
"Station Street in 1998. The library lends books, audiobooks and board games, and runs a homework club "
"on Tuesday and Thursday afternoons.")
RECORD = {"name": "Copper Kettle Cafe", "city": "Larkfield", "stars": 4.0, "attributes": {"WiFi": "free",
"OutdoorSeating": None}, "hours": {"Monday": "8:0-15:0", "Tuesday": "8:0-15:0", "Wednesday": "8:0-15:0",
"Thursday": "8:0-15:0", "Friday": "8:0-15:0", "Saturday": "9:0-14:0"}}
SAMPLES = [
("qa, faithful", PASSAGE, "When does the homework club meet?",
"The homework club meets on Tuesday and Thursday afternoons."),
("qa, two errors", PASSAGE, "Tell me about the library's history.",
"The library opened in 1972 on Mill Road and moved to the old post office in 1998, after a fire destroyed "
"the original building."),
("record, faithful", RECORD, "Write a short description of this cafe.",
"Copper Kettle Cafe in Larkfield has a 4-star rating and free Wi-Fi. It is open 8 AM to 3 PM on weekdays and "
"9 AM to 2 PM on Saturday."),
("record, errors", RECORD, "Write a short description of this cafe.",
"Copper Kettle Cafe in Larkfield is open every day from 8 AM to 6 PM and has a sunny outdoor terrace."),
]
det = HallucinationDetector()
print(f"model loaded on {det.device}, threshold {det.threshold}\n")
for name, context, question, answer in SAMPLES:
for mode in ("lenient", "strict"):
t = time.perf_counter()
res = det.detect(context, answer, question=question, mode=mode)
ms = (time.perf_counter() - t) * 1000
found = "; ".join(f"[{s['type']} {s['score']:.2f}] {s['text']}" for s in res["spans"]) or "no unsupported spans"
print(f"{name:17} {mode:7} -> {found} ({ms:.0f} ms)")
print()
|