Download evaluation.py from Danleon56/ai-engineering-project: direct link, hf CLI and curl.
- Browser
- Download file 6.26 kB
-
https://huggingface.co/spaces/Danleon56/ai-engineering-project/resolve/main/evaluation.py
- Command line
-
hf download hf://spaces/Danleon56/ai-engineering-project/evaluation.py
-
curl -L -o evaluation.py https://huggingface.co/spaces/Danleon56/ai-engineering-project/resolve/main/evaluation.py
6.26 kB
| # evaluation.py | |
| import os | |
| import random | |
| import time | |
| import numpy as np | |
| random.seed(42) | |
| np.random.seed(42) | |
| from app import answer_question | |
| from langchain_openai import ChatOpenAI | |
| from langchain_core.prompts import ChatPromptTemplate | |
| # 1. Initialize the Judge LLM via OpenRouter Free Tier | |
| OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") | |
| judge_llm = ChatOpenAI( | |
| openai_api_base="https://openrouter.ai/api/v1", | |
| openai_api_key=OPENROUTER_API_KEY, | |
| model="openrouter/auto", | |
| temperature=0.0 # Strict deterministic judging | |
| ) | |
| # 2. Define 15-30 Test Questions covering your uploaded policies | |
| EVALUATION_SET = [ | |
| {"question": "What is the company policy on Paid Time Off (PTO) accumulation?", "category": "PTO"}, | |
| {"question": "How many PTO days does an employee with 4 years of service accrue per year?", "category": "PTO"}, | |
| {"question": "What happens to unused PTO at the end of the calendar year?", "category": "PTO"}, | |
| {"question": "Can I work remotely from another country permanently?", "category": "Remote Work"}, | |
| {"question": "How many days per week can employees work remotely?", "category": "Remote Work"}, | |
| {"question": "What is the maximum budget allowed for a client dinner expense?", "category": "Expenses"}, | |
| {"question": "Do I need pre-approval for expenses over $500?", "category": "Expenses"}, | |
| {"question": "How do I report a physical security breach?", "category": "Security"}, | |
| {"question": "What should I do if my company laptop is lost or stolen?", "category": "Security"}, | |
| {"question": "Are gazetted public holidays automatically paid days off?", "category": "Holidays"}, | |
| {"question": "How many floating holidays do employees receive per year?", "category": "Holidays"}, | |
| {"question": "How many weeks of paid parental leave does a primary caregiver receive?", "category": "Parental Leave"}, | |
| {"question": "How many sick days are employees entitled to per year?", "category": "Sick Leave"}, | |
| {"question": "What is the annual Learning and Development budget per employee?", "category": "L&D"}, | |
| {"question": "What performance rating is required to be eligible for a merit increase?", "category": "Performance"}, | |
| {"question": "What is the hotel nightly rate cap in high-cost cities?", "category": "Travel"}, | |
| {"question": "Are USB drives allowed to be connected to company machines?", "category": "Acceptable Use"}, | |
| {"question": "What mandatory training must be completed in the first 5 business days?", "category": "Onboarding"}, | |
| {"question": "How do I request an ergonomic assessment?", "category": "Health & Safety"}, | |
| {"question": "What should I do if I witness a code of conduct violation?", "category": "Code of Conduct"}, | |
| ] | |
| # 3. Define the Evaluation Prompts for the Judge LLM | |
| groundedness_prompt = ChatPromptTemplate.from_messages([ | |
| ("system", ( | |
| "You are an expert quality assurance judge auditing an AI system.\n" | |
| "Analyze the given Context and the AI-generated Answer. Determine if the Answer contains " | |
| "any information that is completely absent from or contradicted by the Context.\n" | |
| "Respond with exactly one word: 'PASS' if the answer is completely grounded, or 'FAIL' if there is an ungrounded claim." | |
| )), | |
| ("human", "Context:\n{context}\n\nAI Answer:\n{answer}") | |
| ]) | |
| citation_prompt = ChatPromptTemplate.from_messages([ | |
| ("system", ( | |
| "You are an expert QA judge verifying citations.\n" | |
| "Review the AI-generated Answer and the retrieved Context chunks.\n" | |
| "Verify if the source citations listed in the Answer accurately point to the specific chunk " | |
| "that actually supports that factual claim.\n" | |
| "Respond with exactly one word: 'PASS' if all citations are correct and honest, or 'FAIL' if any citation is missing or misleading." | |
| )), | |
| ("human", "Context:\n{context}\n\nAI Answer:\n{answer}") | |
| ]) | |
| def run_evaluation(): | |
| latencies = [] | |
| groundedness_scores = [] | |
| citation_scores = [] | |
| print(f"Starting RAG Evaluation across {len(EVALUATION_SET)} test queries...\n") | |
| for idx, item in enumerate(EVALUATION_SET): | |
| q = item["question"] | |
| print(f"[{idx+1}/{len(EVALUATION_SET)}] Testing: '{q}'") | |
| # Measure System Latency | |
| start_time = time.time() | |
| response = answer_question(q) | |
| end_time = time.time() | |
| latency = end_time - start_time | |
| latencies.append(latency) | |
| answer = response["answer"] | |
| # Flatten retrieved document contents for the judge | |
| context_str = "\n---\n".join([doc.page_content for doc in response["context"]]) | |
| # Evaluate Groundedness | |
| g_chain = groundedness_prompt | judge_llm | |
| g_res = g_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper() | |
| groundedness_scores.append(1 if "PASS" in g_res else 0) | |
| # Evaluate Citation Accuracy | |
| c_chain = citation_prompt | judge_llm | |
| c_res = c_chain.invoke({"context": context_str, "answer": answer}).content.strip().upper() | |
| citation_scores.append(1 if "PASS" in c_res else 0) | |
| print(f" Latency: {latency:.2f}s | Groundedness: {g_res} | Citation Check: {c_res}") | |
| # 4. Compute Final Aggregated Metrics | |
| p50_latency = np.percentile(latencies, 50) | |
| p95_latency = np.percentile(latencies, 95) | |
| groundedness_pct = (sum(groundedness_scores) / len(groundedness_scores)) * 100 | |
| citation_pct = (sum(citation_scores) / len(citation_scores)) * 100 | |
| # Print a beautiful markdown summary ready for design-and-evaluation.md | |
| print("\n" + "="*40) | |
| print("FINAL EVALUATION REPORT SUMMARY") | |
| print("="*40) | |
| print(f"| Metric | Target Achieved |") | |
| print(f"| :--- | :--- |") | |
| print(f"| **Groundedness Rate** | {groundedness_pct:.1f}% |") | |
| print(f"| **Citation Accuracy** | {citation_pct:.1f}% |") | |
| print(f"| **p50 Latency (Median)** | {p50_latency:.2f} seconds |") | |
| print(f"| **p95 Latency (Tail)** | {p95_latency:.2f} seconds |") | |
| print("="*40) | |
| if __name__ == "__main__": | |
| if not OPENROUTER_API_KEY: | |
| print("Error: Please set your OPENROUTER_API_KEY environment variable.") | |
| else: | |
| run_evaluation() | |