ai-engineering-project / evaluation.py
Danleon56's picture
fixed syntax errors
0ea83cf
Raw History Blame Contribute Delete
6.26 kB
# evaluation.py
import os
import random
import time
import numpy as np
random.seed(42)
np.random.seed(42)
from app import answer_question
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
# 1. Initialize the Judge LLM via OpenRouter Free Tier
OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
judge_llm = ChatOpenAI(
openai_api_base="https://openrouter.ai/api/v1",
openai_api_key=OPENROUTER_API_KEY,
model="openrouter/auto",
temperature=0.0 # Strict deterministic judging
)
# 2. Define 15-30 Test Questions covering your uploaded policies
EVALUATION_SET = [
{"question": "What is the company policy on Paid Time Off (PTO) accumulation?", "category": "PTO"},
{"question": "How many PTO days does an employee with 4 years of service accrue per year?", "category": "PTO"},
{"question": "What happens to unused PTO at the end of the calendar year?", "category": "PTO"},
{"question": "Can I work remotely from another country permanently?", "category": "Remote Work"},
{"question": "How many days per week can employees work remotely?", "category": "Remote Work"},
{"question": "What is the maximum budget allowed for a client dinner expense?", "category": "Expenses"},
{"question": "Do I need pre-approval for expenses over $500?", "category": "Expenses"},
{"question": "How do I report a physical security breach?", "category": "Security"},
{"question": "What should I do if my company laptop is lost or stolen?", "category": "Security"},
{"question": "Are gazetted public holidays automatically paid days off?", "category": "Holidays"},
{"question": "How many floating holidays do employees receive per year?", "category": "Holidays"},
{"question": "How many weeks of paid parental leave does a primary caregiver receive?", "category": "Parental Leave"},
{"question": "How many sick days are employees entitled to per year?", "category": "Sick Leave"},
{"question": "What is the annual Learning and Development budget per employee?", "category": "L&D"},
{"question": "What performance rating is required to be eligible for a merit increase?", "category": "Performance"},
{"question": "What is the hotel nightly rate cap in high-cost cities?", "category": "Travel"},
{"question": "Are USB drives allowed to be connected to company machines?", "category": "Acceptable Use"},
{"question": "What mandatory training must be completed in the first 5 business days?", "category": "Onboarding"},
{"question": "How do I request an ergonomic assessment?", "category": "Health & Safety"},
{"question": "What should I do if I witness a code of conduct violation?", "category": "Code of Conduct"},
]
# 3. Define the Evaluation Prompts for the Judge LLM
groundedness_prompt = ChatPromptTemplate.from_messages([
("system", (
"You are an expert quality assurance judge auditing an AI system.\n"
"Analyze the given Context and the AI-generated Answer. Determine if the Answer contains "
"any information that is completely absent from or contradicted by the Context.\n"
"Respond with exactly one word: 'PASS' if the answer is completely grounded, or 'FAIL' if there is an ungrounded claim."
)),
("human", "Context:\n{context}\n\nAI Answer:\n{answer}")
])
citation_prompt = ChatPromptTemplate.from_messages([
("system", (
"You are an expert QA judge verifying citations.\n"
"Review the AI-generated Answer and the retrieved Context chunks.\n"
"Verify if the source citations listed in the Answer accurately point to the specific chunk "
"that actually supports that factual claim.\n"
"Respond with exactly one word: 'PASS' if all citations are correct and honest, or 'FAIL' if any citation is missing or misleading."
)),
("human", "Context:\n{context}\n\nAI Answer:\n{answer}")
])
def run_evaluation():
latencies = []
groundedness_scores = []
citation_scores = []
print(f"Starting RAG Evaluation across {len(EVALUATION_SET)} test queries...\n")
for idx, item in enumerate(EVALUATION_SET):
q = item["question"]
print(f"[{idx+1}/{len(EVALUATION_SET)}] Testing: '{q}'")
# Measure System Latency
start_time = time.time()
response = answer_question(q)
end_time = time.time()
latency = end_time - start_time
latencies.append(latency)
answer = response["answer"]
# Flatten retrieved document contents for the judge
context_str = "\n---\n".join([doc.page_content for doc in response["context"]])
# Evaluate Groundedness
g_chain = groundedness_prompt | judge_llm
g_res = g_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper()
groundedness_scores.append(1 if "PASS" in g_res else 0)
# Evaluate Citation Accuracy
c_chain = citation_prompt | judge_llm
c_res = c_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper()
citation_scores.append(1 if "PASS" in c_res else 0)
print(f" Latency: {latency:.2f}s | Groundedness: {g_res} | Citation Check: {c_res}")
# 4. Compute Final Aggregated Metrics
p50_latency = np.percentile(latencies, 50)
p95_latency = np.percentile(latencies, 95)
groundedness_pct = (sum(groundedness_scores) / len(groundedness_scores)) * 100
citation_pct = (sum(citation_scores) / len(citation_scores)) * 100
# Print a beautiful markdown summary ready for design-and-evaluation.md
print("\n" + "="*40)
print("FINAL EVALUATION REPORT SUMMARY")
print("="*40)
print(f"| Metric | Target Achieved |")
print(f"| :--- | :--- |")
print(f"| **Groundedness Rate** | {groundedness_pct:.1f}% |")
print(f"| **Citation Accuracy** | {citation_pct:.1f}% |")
print(f"| **p50 Latency (Median)** | {p50_latency:.2f} seconds |")
print(f"| **p95 Latency (Tail)** | {p95_latency:.2f} seconds |")
print("="*40)
if __name__ == "__main__":
if not OPENROUTER_API_KEY:
print("Error: Please set your OPENROUTER_API_KEY environment variable.")
else:
run_evaluation()