Spaces:
Running
Running
Download backend/src/evaluate.py from tomeese/support-ticket-analyzer: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/spaces/tomeese/support-ticket-analyzer/resolve/main/backend/src/evaluate.py
- Command line
-
hf download hf://spaces/tomeese/support-ticket-analyzer/backend/src/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/tomeese/support-ticket-analyzer/resolve/main/backend/src/evaluate.py
15.6 kB
| """ | |
| Cloud Docs RAG β W&B Weave Evaluation Harness | |
| ---------------------------------------------- | |
| Runs a suite of test questions against the RAG pipeline and scores each | |
| response on three dimensions: | |
| - Correctness : does the answer match the reference answer? | |
| - Relevance : are the retrieved chunks relevant to the question? | |
| - Groundedness : is the answer supported by the retrieved context? | |
| Each scorer uses Claude as an LLM judge (0.0 - 1.0 scale). | |
| Usage: | |
| cd backend | |
| export ANTHROPIC_API_KEY=sk-ant-... | |
| export WANDB_API_KEY=... | |
| export WANDB_PROJECT=aerospike-chatbot | |
| make eval # runs: python -m src.evaluate | |
| Results appear in your W&B Weave dashboard. | |
| """ | |
| import json | |
| import os | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| import weave | |
| from dotenv import load_dotenv | |
| from pydantic import BaseModel | |
| load_dotenv(".env.local") | |
| load_dotenv() | |
| from src.rag import ( # noqa: E402 β env must be loaded first | |
| MODEL, | |
| answer_text, | |
| build_context, | |
| generate_answer, | |
| get_claude, | |
| retrieve_chunks, | |
| ) | |
| # ββ Evaluation dataset ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Questions grounded in the scraped cloud documentation | |
| # "expected" is a concise reference answer used by the correctness scorer. | |
| # Questions marked not_in_docs=True test groundedness β the model should | |
| # say it doesn't know rather than hallucinate. | |
| EVAL_DATASET = [ | |
| { | |
| "question": "What is the default CIDR block for the database cluster?", | |
| "expected": "10.128.0.0/19", | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "What is the CIDR block used for Halyard Cloud internal services?", | |
| "expected": "10.129.0.0/24", | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "What TCP ports are used by Halyard Cloud?", | |
| "expected": "TCP port 4000 for TLS connections, TCP port 3000 for insecure (non-TLS) connections, and optionally TCP port 9145 for Prometheus metrics.", | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "Can customers manage the cloud security settings?", | |
| "expected": ( | |
| "Partially. Customers can manage network security (VPC peering, security groups, " | |
| "firewall rules for their own VPCs), access and identity management (database users, " | |
| "roles, credentials, account members), and application-level TLS configuration. " | |
| "Customers cannot manage infrastructure-level security including OS patching, " | |
| "encryption settings, certificate rotation, key lifecycle, or Halyard's own " | |
| "network configuration. These are managed by Halyard under the shared " | |
| "responsibility model." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "Does Halyard Cloud encrypt data in transit?", | |
| "expected": ( | |
| "Yes. All data in transit is encrypted with TLS 1.2 using strong cipher suites. " | |
| "TLS is required for all control plane access to the API and UI, and is enabled " | |
| "by default for database connections on port 4000. Certificate rotation and key " | |
| "lifecycle management are handled by Halyard." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "What is the retention period for a full backup?", | |
| "expected": "Full backups are retained for 90 days.", | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "How many namespaces can I create with Halyard Cloud?", | |
| "expected": ( | |
| "Each cluster supports only one namespace. " | |
| "To isolate workloads or data, use sets within the single namespace for logical " | |
| "separation, or provision separate clusters for complete isolation." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "What level of encryption does Halyard Cloud use for Encryption at Rest?", | |
| "expected": ( | |
| "All data at rest is encrypted using AES-256 with platform-managed " | |
| "encryption keys. Data in transit is encrypted with TLS 1.2 using strong cipher " | |
| "suites. Certificate rotation and key lifecycle are managed by Halyard." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "How fast can Halyard Cloud scale up the database to add capacity?", | |
| "expected": ( | |
| "The exact duration of scaling is not documented. Scaling involves a migration process " | |
| "and is not instantaneous. The documentation recommends proactive scaling β monitor " | |
| "storage and memory gauges and scale before approaching limits to avoid write failures. " | |
| "Contact support for specific timing details." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "Can a customer set the TTL for records?", | |
| "expected": ( | |
| "Yes. TTL can be set client-side when writing records, but only if record expiration " | |
| "is enabled on the namespace. At the namespace level, default-ttl is configured in " | |
| "seconds (e.g., 604800 for 7 days). At the set level, individual sets can override " | |
| "with their own default-ttl. TTL requires nsup-period to be set to a non-zero value " | |
| "in advanced configuration β new clusters default to 0, which causes TTL writes to " | |
| "fail with FAIL_FORBIDDEN. Fix by either setting nsup-period to a non-zero value or " | |
| "removing TTL from client writes." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "How can I prevent stop writes from occurring?", | |
| "expected": ( | |
| "Stop-writes is triggered by thresholds set in the namespace's advanced configuration: " | |
| "stop-writes-sys-memory-pct, stop-writes-avail-pct and stop-writes-used-pct. Monitor storage " | |
| "and memory usage and scale the cluster proactively before reaching them, allowing time for " | |
| "migrations to complete." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| { | |
| "question": "What replication granularity does XDR support offer?", | |
| "expected": ( | |
| "XDR supports three levels of replication granularity: " | |
| "Namespace (replicate all data in a namespace, the default), " | |
| "Set (replicate only specific sets within a namespace using ship-only-specified-sets and ship-set parameters), " | |
| "and Record deletes (optionally replicate client deletes; NSUP deletes are not shipped by default)." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| # Groundedness test cases β answer should NOT be in the Cloud docs | |
| { | |
| "question": "What storage engines support Strong Consistency?", | |
| "expected": "NOT_IN_DOCS", | |
| "not_in_docs": True, | |
| }, | |
| { | |
| "question": "Does Halyard Cloud support the use of Shadow Devices?", | |
| "expected": "NOT_IN_DOCS", | |
| "not_in_docs": True, | |
| }, | |
| { | |
| "question": "What instance types does Halyard Cloud support?", | |
| "expected": ( | |
| "Storage optimized: i3en, i4i, i7i, i7ie, i8g, i8ge, is4gen. " | |
| "Memory optimized: r7g, r7gd, r8g, r8gd, x8g. " | |
| "General purpose: m6gd, m7gd, m8gd." | |
| ), | |
| "not_in_docs": False, | |
| }, | |
| ] | |
| # ββ RAG pipeline (shared with the API via rag.py) ββββββββββββββββββββββββββββββ | |
| def rag_pipeline(question: str) -> dict: | |
| docs, metas, confidence = retrieve_chunks(question) | |
| context = build_context(docs, metas) | |
| answer = answer_text(generate_answer(question, context)) | |
| return { | |
| "answer": answer, | |
| "context": context, | |
| "sources": [m["title"] for m in metas], | |
| "confidence": confidence, | |
| "docs": docs, | |
| } | |
| # ββ Scorers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| JUDGE_SYSTEM = """You are an objective evaluator of RAG (retrieval-augmented generation) | |
| system outputs. Score responses on a 0.0 to 1.0 scale based on the specific criterion | |
| asked, and give a one-sentence reason. | |
| Example: score 0.8, reason: Answer covers the main point but misses the port number.""" | |
| class JudgeResult(BaseModel): | |
| score: float | |
| reason: str | |
| def llm_judge(prompt: str) -> dict: | |
| """Call Claude as an LLM judge. Returns {score, reason}.""" | |
| response = get_claude().messages.parse( | |
| model=MODEL, | |
| max_tokens=300, | |
| system=JUDGE_SYSTEM, | |
| messages=[{"role": "user", "content": prompt}], | |
| output_format=JudgeResult, | |
| ) | |
| if response.parsed_output is None: | |
| return {"score": 0.5, "reason": f"Judge returned no result (stop_reason={response.stop_reason})"} | |
| return response.parsed_output.model_dump() | |
| def score_correctness(question: str, answer: str, expected: str, not_in_docs: bool) -> dict: | |
| """ | |
| Does the answer correctly address the question? | |
| For not_in_docs questions, a correct answer acknowledges the info isn't available. | |
| """ | |
| if not_in_docs: | |
| prompt = f"""Question: {question} | |
| Expected behavior: The system should clearly state that this information is not available | |
| in the documentation, rather than guessing or hallucinating an answer. | |
| Actual answer: {answer} | |
| Score 1.0 if the system clearly says it doesn't know / info not in docs. | |
| Score 0.0 if the system provides a fabricated or confident answer without basis.""" | |
| else: | |
| prompt = f"""Question: {question} | |
| Reference answer: {expected} | |
| Actual answer: {answer} | |
| Score how well the actual answer matches the reference answer in terms of | |
| factual correctness and completeness. 1.0 = perfect match, 0.0 = completely wrong.""" | |
| result = llm_judge(prompt) | |
| return {"correctness": result["score"], "correctness_reason": result["reason"]} | |
| def score_relevance(question: str, sources: list, confidence: float, docs: list) -> dict: | |
| """ | |
| Are the retrieved chunks relevant to the question? | |
| Passes actual chunk content to the judge, not just titles. | |
| """ | |
| # The full text of each retrieved chunk β the same content the answering model saw. | |
| # (Truncated excerpts made the judge miss facts that were in the chunk.) | |
| excerpts = "\n\n".join( | |
| f"[Source {i+1}: {title}]\n{doc}" | |
| for i, (title, doc) in enumerate(zip(sources, docs)) | |
| ) | |
| prompt = f"""Question: {question} | |
| Retrieved chunks (actual content): | |
| {excerpts} | |
| Score 1.0 if the retrieved chunks contain information directly useful for answering | |
| the question. Score 0.0 if the chunks are unrelated to the question.""" | |
| result = llm_judge(prompt) | |
| return { | |
| "relevance": result["score"], | |
| "relevance_reason": result["reason"], | |
| "retrieval_confidence": confidence, | |
| } | |
| def score_groundedness(answer: str, context: str, not_in_docs: bool) -> dict: | |
| """ | |
| Is the answer grounded in the retrieved context, or did the model hallucinate? | |
| """ | |
| if not_in_docs: | |
| # For not_in_docs questions, groundedness = model didn't make things up | |
| prompt = f"""Context provided to the model: | |
| {context} | |
| Model's answer: {answer} | |
| Score 1.0 if the model's answer is consistent with (or correctly acknowledges gaps in) | |
| the provided context. Score 0.0 if the model claims information not present in the context.""" | |
| else: | |
| prompt = f"""Context provided to the model: | |
| {context} | |
| Model's answer: {answer} | |
| Score how well the answer is supported by the provided context. | |
| 1.0 = fully grounded in context, 0.0 = answer contains claims not in context (hallucination).""" | |
| result = llm_judge(prompt) | |
| return {"groundedness": result["score"], "groundedness_reason": result["reason"]} | |
| # ββ Evaluation runner βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def evaluate_single(item: dict) -> dict: | |
| question = item["question"] | |
| expected = item["expected"] | |
| not_in_docs = item["not_in_docs"] | |
| # Run RAG pipeline | |
| result = rag_pipeline(question) | |
| # Score | |
| correctness = score_correctness(question, result["answer"], expected, not_in_docs) | |
| relevance = score_relevance(question, result["sources"], result["confidence"], result["docs"]) | |
| groundedness = score_groundedness(result["answer"], result["context"], not_in_docs) | |
| return { | |
| "question": question, | |
| "answer": result["answer"], | |
| "expected": expected, | |
| "not_in_docs": not_in_docs, | |
| "sources": result["sources"], | |
| **correctness, | |
| **relevance, | |
| **groundedness, | |
| "composite_score": round( | |
| (correctness["correctness"] + relevance["relevance"] + groundedness["groundedness"]) / 3, | |
| 4, | |
| ), | |
| } | |
| def run_evaluation(): | |
| weave.init(os.getenv("WANDB_PROJECT", "aerospike-chatbot")) | |
| print(f"Running evaluation on {len(EVAL_DATASET)} questions...\n") | |
| results = [] | |
| for i, item in enumerate(EVAL_DATASET, 1): | |
| print(f"[{i}/{len(EVAL_DATASET)}] {item['question'][:70]}...") | |
| result = evaluate_single(item) | |
| results.append(result) | |
| print(f" correctness={result['correctness']:.2f} " | |
| f"relevance={result['relevance']:.2f} " | |
| f"groundedness={result['groundedness']:.2f} " | |
| f"composite={result['composite_score']:.2f}") | |
| # Summary | |
| avg_correctness = sum(r["correctness"] for r in results) / len(results) | |
| avg_relevance = sum(r["relevance"] for r in results) / len(results) | |
| avg_groundedness = sum(r["groundedness"] for r in results) / len(results) | |
| avg_composite = sum(r["composite_score"] for r in results) / len(results) | |
| print(f"\n{'='*60}") | |
| print(f"EVALUATION SUMMARY ({len(results)} questions)") | |
| print(f"{'='*60}") | |
| print(f" Avg Correctness : {avg_correctness:.3f}") | |
| print(f" Avg Relevance : {avg_relevance:.3f}") | |
| print(f" Avg Groundedness : {avg_groundedness:.3f}") | |
| print(f" Avg Composite : {avg_composite:.3f}") | |
| print("\nFull results available in W&B Weave dashboard.") | |
| write_results(results) | |
| return results | |
| RESULTS_FILE = Path("results/eval_results.json") | |
| def write_results(results: list[dict]): | |
| """Save a summary for the app's scorecard (GET /eval-summary) plus per-question detail.""" | |
| def avg(rows, key): | |
| return round(sum(r[key] for r in rows) / len(rows), 3) if rows else None | |
| in_scope = [r for r in results if not r["not_in_docs"]] | |
| out_of_scope = [r for r in results if r["not_in_docs"]] | |
| summary = { | |
| "run_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), | |
| "model": MODEL, | |
| "questions": len(results), | |
| "correctness": avg(in_scope, "correctness"), | |
| "groundedness": avg(results, "groundedness"), | |
| "retrieval_relevance": avg(in_scope, "relevance"), | |
| "out_of_scope_handling": avg(out_of_scope, "correctness"), | |
| "out_of_scope_questions": len(out_of_scope), | |
| } | |
| RESULTS_FILE.parent.mkdir(exist_ok=True) | |
| RESULTS_FILE.write_text(json.dumps({"summary": summary, "results": results}, indent=2)) | |
| print(f"Results written to {RESULTS_FILE}") | |
| if __name__ == "__main__": | |
| run_evaluation() |