acpcchatbot_testing_first / app /guardrails /relevance_grader.py
acpcchatbotadmin's picture
Upload 40 files
481dfe6 verified
Raw History Blame Contribute Delete
1.49 kB
import json
import logging
from langchain_core.documents import Document
from langchain_core.messages import HumanMessage
from langchain_openai import ChatOpenAI
from app.agent.prompts import RELEVANCE_GRADING_PROMPT
from app.config import settings
logger = logging.getLogger(__name__)
def grade_document(query: str, document: Document, llm: ChatOpenAI) -> bool:
"""Grade a single document. Returns True if relevant."""
prompt = RELEVANCE_GRADING_PROMPT.format(
document=document.page_content[:1000],
question=query,
)
try:
response = llm.invoke([HumanMessage(content=prompt)])
result = json.loads(response.content.strip())
return result.get("relevant") == "yes"
except (json.JSONDecodeError, Exception) as e:
logger.warning("Grade error: %s — including document", e)
return True # Fail open
def grade_documents(query: str, documents: list[Document]) -> tuple[list[Document], float]:
"""
Grade all documents for relevance.
Returns (relevant_documents, confidence_score).
"""
if not documents:
return [], 0.0
llm = ChatOpenAI(
model=settings.llm_model,
api_key=settings.openai_api_key,
base_url=settings.openai_api_base,
temperature=0.0,
)
relevant = [doc for doc in documents if grade_document(query, doc, llm)]
confidence = len(relevant) / len(documents)
return relevant, confidence