# evaluation.py import os import random import time import numpy as np random.seed(42) np.random.seed(42) from app import answer_question from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate # 1. Initialize the Judge LLM via OpenRouter Free Tier OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") judge_llm = ChatOpenAI( openai_api_base="https://openrouter.ai/api/v1", openai_api_key=OPENROUTER_API_KEY, model="openrouter/auto", temperature=0.0 # Strict deterministic judging ) # 2. Define 15-30 Test Questions covering your uploaded policies EVALUATION_SET = [ {"question": "What is the company policy on Paid Time Off (PTO) accumulation?", "category": "PTO"}, {"question": "How many PTO days does an employee with 4 years of service accrue per year?", "category": "PTO"}, {"question": "What happens to unused PTO at the end of the calendar year?", "category": "PTO"}, {"question": "Can I work remotely from another country permanently?", "category": "Remote Work"}, {"question": "How many days per week can employees work remotely?", "category": "Remote Work"}, {"question": "What is the maximum budget allowed for a client dinner expense?", "category": "Expenses"}, {"question": "Do I need pre-approval for expenses over $500?", "category": "Expenses"}, {"question": "How do I report a physical security breach?", "category": "Security"}, {"question": "What should I do if my company laptop is lost or stolen?", "category": "Security"}, {"question": "Are gazetted public holidays automatically paid days off?", "category": "Holidays"}, {"question": "How many floating holidays do employees receive per year?", "category": "Holidays"}, {"question": "How many weeks of paid parental leave does a primary caregiver receive?", "category": "Parental Leave"}, {"question": "How many sick days are employees entitled to per year?", "category": "Sick Leave"}, {"question": "What is the annual Learning and Development budget per employee?", "category": "L&D"}, {"question": "What performance rating is required to be eligible for a merit increase?", "category": "Performance"}, {"question": "What is the hotel nightly rate cap in high-cost cities?", "category": "Travel"}, {"question": "Are USB drives allowed to be connected to company machines?", "category": "Acceptable Use"}, {"question": "What mandatory training must be completed in the first 5 business days?", "category": "Onboarding"}, {"question": "How do I request an ergonomic assessment?", "category": "Health & Safety"}, {"question": "What should I do if I witness a code of conduct violation?", "category": "Code of Conduct"}, ] # 3. Define the Evaluation Prompts for the Judge LLM groundedness_prompt = ChatPromptTemplate.from_messages([ ("system", ( "You are an expert quality assurance judge auditing an AI system.\n" "Analyze the given Context and the AI-generated Answer. Determine if the Answer contains " "any information that is completely absent from or contradicted by the Context.\n" "Respond with exactly one word: 'PASS' if the answer is completely grounded, or 'FAIL' if there is an ungrounded claim." )), ("human", "Context:\n{context}\n\nAI Answer:\n{answer}") ]) citation_prompt = ChatPromptTemplate.from_messages([ ("system", ( "You are an expert QA judge verifying citations.\n" "Review the AI-generated Answer and the retrieved Context chunks.\n" "Verify if the source citations listed in the Answer accurately point to the specific chunk " "that actually supports that factual claim.\n" "Respond with exactly one word: 'PASS' if all citations are correct and honest, or 'FAIL' if any citation is missing or misleading." )), ("human", "Context:\n{context}\n\nAI Answer:\n{answer}") ]) def run_evaluation(): latencies = [] groundedness_scores = [] citation_scores = [] print(f"Starting RAG Evaluation across {len(EVALUATION_SET)} test queries...\n") for idx, item in enumerate(EVALUATION_SET): q = item["question"] print(f"[{idx+1}/{len(EVALUATION_SET)}] Testing: '{q}'") # Measure System Latency start_time = time.time() response = answer_question(q) end_time = time.time() latency = end_time - start_time latencies.append(latency) answer = response["answer"] # Flatten retrieved document contents for the judge context_str = "\n---\n".join([doc.page_content for doc in response["context"]]) # Evaluate Groundedness g_chain = groundedness_prompt | judge_llm g_res = g_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper() groundedness_scores.append(1 if "PASS" in g_res else 0) # Evaluate Citation Accuracy c_chain = citation_prompt | judge_llm c_res = c_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper() citation_scores.append(1 if "PASS" in c_res else 0) print(f" Latency: {latency:.2f}s | Groundedness: {g_res} | Citation Check: {c_res}") # 4. Compute Final Aggregated Metrics p50_latency = np.percentile(latencies, 50) p95_latency = np.percentile(latencies, 95) groundedness_pct = (sum(groundedness_scores) / len(groundedness_scores)) * 100 citation_pct = (sum(citation_scores) / len(citation_scores)) * 100 # Print a beautiful markdown summary ready for design-and-evaluation.md print("\n" + "="*40) print("FINAL EVALUATION REPORT SUMMARY") print("="*40) print(f"| Metric | Target Achieved |") print(f"| :--- | :--- |") print(f"| **Groundedness Rate** | {groundedness_pct:.1f}% |") print(f"| **Citation Accuracy** | {citation_pct:.1f}% |") print(f"| **p50 Latency (Median)** | {p50_latency:.2f} seconds |") print(f"| **p95 Latency (Tail)** | {p95_latency:.2f} seconds |") print("="*40) if __name__ == "__main__": if not OPENROUTER_API_KEY: print("Error: Please set your OPENROUTER_API_KEY environment variable.") else: run_evaluation()