acpcchatbot_testing_first / app /guardrails /hallucination_checker.py
acpcchatbotadmin's picture
Upload 40 files
481dfe6 verified
Raw History Blame Contribute Delete
1.48 kB
import json
import logging
from langchain_core.documents import Document
from langchain_core.messages import HumanMessage
from langchain_openai import ChatOpenAI
from app.agent.prompts import HALLUCINATION_CHECK_PROMPT
from app.config import settings
logger = logging.getLogger(__name__)
def is_grounded(generation: str, documents: list[Document]) -> bool:
"""
Check if the generated answer is grounded in the provided documents.
Returns True if grounded (safe to show), False if hallucination detected.
"""
if not documents:
return False
context = "\n\n".join(doc.page_content[:800] for doc in documents)
prompt = HALLUCINATION_CHECK_PROMPT.format(documents=context, answer=generation)
llm = ChatOpenAI(
model=settings.llm_model,
api_key=settings.openai_api_key,
base_url=settings.openai_api_base,
temperature=0.0,
)
try:
response = llm.invoke([HumanMessage(content=prompt)])
result = json.loads(response.content.strip())
grounded = result.get("grounded", True)
if not grounded:
logger.warning("Hallucination detected: %s", result.get("reason", "no reason given"))
return bool(grounded)
except (json.JSONDecodeError, Exception) as e:
logger.warning("Hallucination check error: %s — defaulting grounded=True", e)
return True # Fail open (don't block valid answers on check errors)