""" Cloud Docs RAG — W&B Weave Evaluation Harness ---------------------------------------------- Runs a suite of test questions against the RAG pipeline and scores each response on three dimensions: - Correctness : does the answer match the reference answer? - Relevance : are the retrieved chunks relevant to the question? - Groundedness : is the answer supported by the retrieved context? Each scorer uses Claude as an LLM judge (0.0 - 1.0 scale). Usage: cd backend export ANTHROPIC_API_KEY=sk-ant-... export WANDB_API_KEY=... export WANDB_PROJECT=aerospike-chatbot make eval # runs: python -m src.evaluate Results appear in your W&B Weave dashboard. """ import json import os from datetime import datetime, timezone from pathlib import Path import weave from dotenv import load_dotenv from pydantic import BaseModel load_dotenv(".env.local") load_dotenv() from src.rag import ( # noqa: E402 — env must be loaded first MODEL, answer_text, build_context, generate_answer, get_claude, retrieve_chunks, ) # ── Evaluation dataset ──────────────────────────────────────────────────────── # Questions grounded in the scraped cloud documentation # "expected" is a concise reference answer used by the correctness scorer. # Questions marked not_in_docs=True test groundedness — the model should # say it doesn't know rather than hallucinate. EVAL_DATASET = [ { "question": "What is the default CIDR block for the database cluster?", "expected": "10.128.0.0/19", "not_in_docs": False, }, { "question": "What is the CIDR block used for Halyard Cloud internal services?", "expected": "10.129.0.0/24", "not_in_docs": False, }, { "question": "What TCP ports are used by Halyard Cloud?", "expected": "TCP port 4000 for TLS connections, TCP port 3000 for insecure (non-TLS) connections, and optionally TCP port 9145 for Prometheus metrics.", "not_in_docs": False, }, { "question": "Can customers manage the cloud security settings?", "expected": ( "Partially. Customers can manage network security (VPC peering, security groups, " "firewall rules for their own VPCs), access and identity management (database users, " "roles, credentials, account members), and application-level TLS configuration. " "Customers cannot manage infrastructure-level security including OS patching, " "encryption settings, certificate rotation, key lifecycle, or Halyard's own " "network configuration. These are managed by Halyard under the shared " "responsibility model." ), "not_in_docs": False, }, { "question": "Does Halyard Cloud encrypt data in transit?", "expected": ( "Yes. All data in transit is encrypted with TLS 1.2 using strong cipher suites. " "TLS is required for all control plane access to the API and UI, and is enabled " "by default for database connections on port 4000. Certificate rotation and key " "lifecycle management are handled by Halyard." ), "not_in_docs": False, }, { "question": "What is the retention period for a full backup?", "expected": "Full backups are retained for 90 days.", "not_in_docs": False, }, { "question": "How many namespaces can I create with Halyard Cloud?", "expected": ( "Each cluster supports only one namespace. " "To isolate workloads or data, use sets within the single namespace for logical " "separation, or provision separate clusters for complete isolation." ), "not_in_docs": False, }, { "question": "What level of encryption does Halyard Cloud use for Encryption at Rest?", "expected": ( "All data at rest is encrypted using AES-256 with platform-managed " "encryption keys. Data in transit is encrypted with TLS 1.2 using strong cipher " "suites. Certificate rotation and key lifecycle are managed by Halyard." ), "not_in_docs": False, }, { "question": "How fast can Halyard Cloud scale up the database to add capacity?", "expected": ( "The exact duration of scaling is not documented. Scaling involves a migration process " "and is not instantaneous. The documentation recommends proactive scaling — monitor " "storage and memory gauges and scale before approaching limits to avoid write failures. " "Contact support for specific timing details." ), "not_in_docs": False, }, { "question": "Can a customer set the TTL for records?", "expected": ( "Yes. TTL can be set client-side when writing records, but only if record expiration " "is enabled on the namespace. At the namespace level, default-ttl is configured in " "seconds (e.g., 604800 for 7 days). At the set level, individual sets can override " "with their own default-ttl. TTL requires nsup-period to be set to a non-zero value " "in advanced configuration — new clusters default to 0, which causes TTL writes to " "fail with FAIL_FORBIDDEN. Fix by either setting nsup-period to a non-zero value or " "removing TTL from client writes." ), "not_in_docs": False, }, { "question": "How can I prevent stop writes from occurring?", "expected": ( "Stop-writes is triggered by thresholds set in the namespace's advanced configuration: " "stop-writes-sys-memory-pct, stop-writes-avail-pct and stop-writes-used-pct. Monitor storage " "and memory usage and scale the cluster proactively before reaching them, allowing time for " "migrations to complete." ), "not_in_docs": False, }, { "question": "What replication granularity does XDR support offer?", "expected": ( "XDR supports three levels of replication granularity: " "Namespace (replicate all data in a namespace, the default), " "Set (replicate only specific sets within a namespace using ship-only-specified-sets and ship-set parameters), " "and Record deletes (optionally replicate client deletes; NSUP deletes are not shipped by default)." ), "not_in_docs": False, }, # Groundedness test cases — answer should NOT be in the Cloud docs { "question": "What storage engines support Strong Consistency?", "expected": "NOT_IN_DOCS", "not_in_docs": True, }, { "question": "Does Halyard Cloud support the use of Shadow Devices?", "expected": "NOT_IN_DOCS", "not_in_docs": True, }, { "question": "What instance types does Halyard Cloud support?", "expected": ( "Storage optimized: i3en, i4i, i7i, i7ie, i8g, i8ge, is4gen. " "Memory optimized: r7g, r7gd, r8g, r8gd, x8g. " "General purpose: m6gd, m7gd, m8gd." ), "not_in_docs": False, }, ] # ── RAG pipeline (shared with the API via rag.py) ────────────────────────────── @weave.op() def rag_pipeline(question: str) -> dict: docs, metas, confidence = retrieve_chunks(question) context = build_context(docs, metas) answer = answer_text(generate_answer(question, context)) return { "answer": answer, "context": context, "sources": [m["title"] for m in metas], "confidence": confidence, "docs": docs, } # ── Scorers ─────────────────────────────────────────────────────────────────── JUDGE_SYSTEM = """You are an objective evaluator of RAG (retrieval-augmented generation) system outputs. Score responses on a 0.0 to 1.0 scale based on the specific criterion asked, and give a one-sentence reason. Example: score 0.8, reason: Answer covers the main point but misses the port number.""" class JudgeResult(BaseModel): score: float reason: str def llm_judge(prompt: str) -> dict: """Call Claude as an LLM judge. Returns {score, reason}.""" response = get_claude().messages.parse( model=MODEL, max_tokens=300, system=JUDGE_SYSTEM, messages=[{"role": "user", "content": prompt}], output_format=JudgeResult, ) if response.parsed_output is None: return {"score": 0.5, "reason": f"Judge returned no result (stop_reason={response.stop_reason})"} return response.parsed_output.model_dump() @weave.op() def score_correctness(question: str, answer: str, expected: str, not_in_docs: bool) -> dict: """ Does the answer correctly address the question? For not_in_docs questions, a correct answer acknowledges the info isn't available. """ if not_in_docs: prompt = f"""Question: {question} Expected behavior: The system should clearly state that this information is not available in the documentation, rather than guessing or hallucinating an answer. Actual answer: {answer} Score 1.0 if the system clearly says it doesn't know / info not in docs. Score 0.0 if the system provides a fabricated or confident answer without basis.""" else: prompt = f"""Question: {question} Reference answer: {expected} Actual answer: {answer} Score how well the actual answer matches the reference answer in terms of factual correctness and completeness. 1.0 = perfect match, 0.0 = completely wrong.""" result = llm_judge(prompt) return {"correctness": result["score"], "correctness_reason": result["reason"]} @weave.op() def score_relevance(question: str, sources: list, confidence: float, docs: list) -> dict: """ Are the retrieved chunks relevant to the question? Passes actual chunk content to the judge, not just titles. """ # The full text of each retrieved chunk — the same content the answering model saw. # (Truncated excerpts made the judge miss facts that were in the chunk.) excerpts = "\n\n".join( f"[Source {i+1}: {title}]\n{doc}" for i, (title, doc) in enumerate(zip(sources, docs)) ) prompt = f"""Question: {question} Retrieved chunks (actual content): {excerpts} Score 1.0 if the retrieved chunks contain information directly useful for answering the question. Score 0.0 if the chunks are unrelated to the question.""" result = llm_judge(prompt) return { "relevance": result["score"], "relevance_reason": result["reason"], "retrieval_confidence": confidence, } @weave.op() def score_groundedness(answer: str, context: str, not_in_docs: bool) -> dict: """ Is the answer grounded in the retrieved context, or did the model hallucinate? """ if not_in_docs: # For not_in_docs questions, groundedness = model didn't make things up prompt = f"""Context provided to the model: {context} Model's answer: {answer} Score 1.0 if the model's answer is consistent with (or correctly acknowledges gaps in) the provided context. Score 0.0 if the model claims information not present in the context.""" else: prompt = f"""Context provided to the model: {context} Model's answer: {answer} Score how well the answer is supported by the provided context. 1.0 = fully grounded in context, 0.0 = answer contains claims not in context (hallucination).""" result = llm_judge(prompt) return {"groundedness": result["score"], "groundedness_reason": result["reason"]} # ── Evaluation runner ───────────────────────────────────────────────────────── @weave.op() def evaluate_single(item: dict) -> dict: question = item["question"] expected = item["expected"] not_in_docs = item["not_in_docs"] # Run RAG pipeline result = rag_pipeline(question) # Score correctness = score_correctness(question, result["answer"], expected, not_in_docs) relevance = score_relevance(question, result["sources"], result["confidence"], result["docs"]) groundedness = score_groundedness(result["answer"], result["context"], not_in_docs) return { "question": question, "answer": result["answer"], "expected": expected, "not_in_docs": not_in_docs, "sources": result["sources"], **correctness, **relevance, **groundedness, "composite_score": round( (correctness["correctness"] + relevance["relevance"] + groundedness["groundedness"]) / 3, 4, ), } def run_evaluation(): weave.init(os.getenv("WANDB_PROJECT", "aerospike-chatbot")) print(f"Running evaluation on {len(EVAL_DATASET)} questions...\n") results = [] for i, item in enumerate(EVAL_DATASET, 1): print(f"[{i}/{len(EVAL_DATASET)}] {item['question'][:70]}...") result = evaluate_single(item) results.append(result) print(f" correctness={result['correctness']:.2f} " f"relevance={result['relevance']:.2f} " f"groundedness={result['groundedness']:.2f} " f"composite={result['composite_score']:.2f}") # Summary avg_correctness = sum(r["correctness"] for r in results) / len(results) avg_relevance = sum(r["relevance"] for r in results) / len(results) avg_groundedness = sum(r["groundedness"] for r in results) / len(results) avg_composite = sum(r["composite_score"] for r in results) / len(results) print(f"\n{'='*60}") print(f"EVALUATION SUMMARY ({len(results)} questions)") print(f"{'='*60}") print(f" Avg Correctness : {avg_correctness:.3f}") print(f" Avg Relevance : {avg_relevance:.3f}") print(f" Avg Groundedness : {avg_groundedness:.3f}") print(f" Avg Composite : {avg_composite:.3f}") print("\nFull results available in W&B Weave dashboard.") write_results(results) return results RESULTS_FILE = Path("results/eval_results.json") def write_results(results: list[dict]): """Save a summary for the app's scorecard (GET /eval-summary) plus per-question detail.""" def avg(rows, key): return round(sum(r[key] for r in rows) / len(rows), 3) if rows else None in_scope = [r for r in results if not r["not_in_docs"]] out_of_scope = [r for r in results if r["not_in_docs"]] summary = { "run_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "model": MODEL, "questions": len(results), "correctness": avg(in_scope, "correctness"), "groundedness": avg(results, "groundedness"), "retrieval_relevance": avg(in_scope, "relevance"), "out_of_scope_handling": avg(out_of_scope, "correctness"), "out_of_scope_questions": len(out_of_scope), } RESULTS_FILE.parent.mkdir(exist_ok=True) RESULTS_FILE.write_text(json.dumps({"summary": summary, "results": results}, indent=2)) print(f"Results written to {RESULTS_FILE}") if __name__ == "__main__": run_evaluation()