import json import logging from langchain_core.documents import Document from langchain_core.messages import HumanMessage from langchain_openai import ChatOpenAI from app.agent.prompts import RELEVANCE_GRADING_PROMPT from app.config import settings logger = logging.getLogger(__name__) def grade_document(query: str, document: Document, llm: ChatOpenAI) -> bool: """Grade a single document. Returns True if relevant.""" prompt = RELEVANCE_GRADING_PROMPT.format( document=document.page_content[:1000], question=query, ) try: response = llm.invoke([HumanMessage(content=prompt)]) result = json.loads(response.content.strip()) return result.get("relevant") == "yes" except (json.JSONDecodeError, Exception) as e: logger.warning("Grade error: %s — including document", e) return True # Fail open def grade_documents(query: str, documents: list[Document]) -> tuple[list[Document], float]: """ Grade all documents for relevance. Returns (relevant_documents, confidence_score). """ if not documents: return [], 0.0 llm = ChatOpenAI( model=settings.llm_model, api_key=settings.openai_api_key, base_url=settings.openai_api_base, temperature=0.0, ) relevant = [doc for doc in documents if grade_document(query, doc, llm)] confidence = len(relevant) / len(documents) return relevant, confidence