tomeese's picture
Claude Opus 5.5
Fix eval judges and publish first scorecard results
bb393d8
Raw History Blame Contribute Delete
15.6 kB
"""
Cloud Docs RAG β€” W&B Weave Evaluation Harness
----------------------------------------------
Runs a suite of test questions against the RAG pipeline and scores each
response on three dimensions:
- Correctness : does the answer match the reference answer?
- Relevance : are the retrieved chunks relevant to the question?
- Groundedness : is the answer supported by the retrieved context?
Each scorer uses Claude as an LLM judge (0.0 - 1.0 scale).
Usage:
cd backend
export ANTHROPIC_API_KEY=sk-ant-...
export WANDB_API_KEY=...
export WANDB_PROJECT=aerospike-chatbot
make eval # runs: python -m src.evaluate
Results appear in your W&B Weave dashboard.
"""
import json
import os
from datetime import datetime, timezone
from pathlib import Path
import weave
from dotenv import load_dotenv
from pydantic import BaseModel
load_dotenv(".env.local")
load_dotenv()
from src.rag import ( # noqa: E402 β€” env must be loaded first
MODEL,
answer_text,
build_context,
generate_answer,
get_claude,
retrieve_chunks,
)
# ── Evaluation dataset ────────────────────────────────────────────────────────
# Questions grounded in the scraped cloud documentation
# "expected" is a concise reference answer used by the correctness scorer.
# Questions marked not_in_docs=True test groundedness β€” the model should
# say it doesn't know rather than hallucinate.
EVAL_DATASET = [
{
"question": "What is the default CIDR block for the database cluster?",
"expected": "10.128.0.0/19",
"not_in_docs": False,
},
{
"question": "What is the CIDR block used for Halyard Cloud internal services?",
"expected": "10.129.0.0/24",
"not_in_docs": False,
},
{
"question": "What TCP ports are used by Halyard Cloud?",
"expected": "TCP port 4000 for TLS connections, TCP port 3000 for insecure (non-TLS) connections, and optionally TCP port 9145 for Prometheus metrics.",
"not_in_docs": False,
},
{
"question": "Can customers manage the cloud security settings?",
"expected": (
"Partially. Customers can manage network security (VPC peering, security groups, "
"firewall rules for their own VPCs), access and identity management (database users, "
"roles, credentials, account members), and application-level TLS configuration. "
"Customers cannot manage infrastructure-level security including OS patching, "
"encryption settings, certificate rotation, key lifecycle, or Halyard's own "
"network configuration. These are managed by Halyard under the shared "
"responsibility model."
),
"not_in_docs": False,
},
{
"question": "Does Halyard Cloud encrypt data in transit?",
"expected": (
"Yes. All data in transit is encrypted with TLS 1.2 using strong cipher suites. "
"TLS is required for all control plane access to the API and UI, and is enabled "
"by default for database connections on port 4000. Certificate rotation and key "
"lifecycle management are handled by Halyard."
),
"not_in_docs": False,
},
{
"question": "What is the retention period for a full backup?",
"expected": "Full backups are retained for 90 days.",
"not_in_docs": False,
},
{
"question": "How many namespaces can I create with Halyard Cloud?",
"expected": (
"Each cluster supports only one namespace. "
"To isolate workloads or data, use sets within the single namespace for logical "
"separation, or provision separate clusters for complete isolation."
),
"not_in_docs": False,
},
{
"question": "What level of encryption does Halyard Cloud use for Encryption at Rest?",
"expected": (
"All data at rest is encrypted using AES-256 with platform-managed "
"encryption keys. Data in transit is encrypted with TLS 1.2 using strong cipher "
"suites. Certificate rotation and key lifecycle are managed by Halyard."
),
"not_in_docs": False,
},
{
"question": "How fast can Halyard Cloud scale up the database to add capacity?",
"expected": (
"The exact duration of scaling is not documented. Scaling involves a migration process "
"and is not instantaneous. The documentation recommends proactive scaling β€” monitor "
"storage and memory gauges and scale before approaching limits to avoid write failures. "
"Contact support for specific timing details."
),
"not_in_docs": False,
},
{
"question": "Can a customer set the TTL for records?",
"expected": (
"Yes. TTL can be set client-side when writing records, but only if record expiration "
"is enabled on the namespace. At the namespace level, default-ttl is configured in "
"seconds (e.g., 604800 for 7 days). At the set level, individual sets can override "
"with their own default-ttl. TTL requires nsup-period to be set to a non-zero value "
"in advanced configuration β€” new clusters default to 0, which causes TTL writes to "
"fail with FAIL_FORBIDDEN. Fix by either setting nsup-period to a non-zero value or "
"removing TTL from client writes."
),
"not_in_docs": False,
},
{
"question": "How can I prevent stop writes from occurring?",
"expected": (
"Stop-writes is triggered by thresholds set in the namespace's advanced configuration: "
"stop-writes-sys-memory-pct, stop-writes-avail-pct and stop-writes-used-pct. Monitor storage "
"and memory usage and scale the cluster proactively before reaching them, allowing time for "
"migrations to complete."
),
"not_in_docs": False,
},
{
"question": "What replication granularity does XDR support offer?",
"expected": (
"XDR supports three levels of replication granularity: "
"Namespace (replicate all data in a namespace, the default), "
"Set (replicate only specific sets within a namespace using ship-only-specified-sets and ship-set parameters), "
"and Record deletes (optionally replicate client deletes; NSUP deletes are not shipped by default)."
),
"not_in_docs": False,
},
# Groundedness test cases β€” answer should NOT be in the Cloud docs
{
"question": "What storage engines support Strong Consistency?",
"expected": "NOT_IN_DOCS",
"not_in_docs": True,
},
{
"question": "Does Halyard Cloud support the use of Shadow Devices?",
"expected": "NOT_IN_DOCS",
"not_in_docs": True,
},
{
"question": "What instance types does Halyard Cloud support?",
"expected": (
"Storage optimized: i3en, i4i, i7i, i7ie, i8g, i8ge, is4gen. "
"Memory optimized: r7g, r7gd, r8g, r8gd, x8g. "
"General purpose: m6gd, m7gd, m8gd."
),
"not_in_docs": False,
},
]
# ── RAG pipeline (shared with the API via rag.py) ──────────────────────────────
@weave.op()
def rag_pipeline(question: str) -> dict:
docs, metas, confidence = retrieve_chunks(question)
context = build_context(docs, metas)
answer = answer_text(generate_answer(question, context))
return {
"answer": answer,
"context": context,
"sources": [m["title"] for m in metas],
"confidence": confidence,
"docs": docs,
}
# ── Scorers ───────────────────────────────────────────────────────────────────
JUDGE_SYSTEM = """You are an objective evaluator of RAG (retrieval-augmented generation)
system outputs. Score responses on a 0.0 to 1.0 scale based on the specific criterion
asked, and give a one-sentence reason.
Example: score 0.8, reason: Answer covers the main point but misses the port number."""
class JudgeResult(BaseModel):
score: float
reason: str
def llm_judge(prompt: str) -> dict:
"""Call Claude as an LLM judge. Returns {score, reason}."""
response = get_claude().messages.parse(
model=MODEL,
max_tokens=300,
system=JUDGE_SYSTEM,
messages=[{"role": "user", "content": prompt}],
output_format=JudgeResult,
)
if response.parsed_output is None:
return {"score": 0.5, "reason": f"Judge returned no result (stop_reason={response.stop_reason})"}
return response.parsed_output.model_dump()
@weave.op()
def score_correctness(question: str, answer: str, expected: str, not_in_docs: bool) -> dict:
"""
Does the answer correctly address the question?
For not_in_docs questions, a correct answer acknowledges the info isn't available.
"""
if not_in_docs:
prompt = f"""Question: {question}
Expected behavior: The system should clearly state that this information is not available
in the documentation, rather than guessing or hallucinating an answer.
Actual answer: {answer}
Score 1.0 if the system clearly says it doesn't know / info not in docs.
Score 0.0 if the system provides a fabricated or confident answer without basis."""
else:
prompt = f"""Question: {question}
Reference answer: {expected}
Actual answer: {answer}
Score how well the actual answer matches the reference answer in terms of
factual correctness and completeness. 1.0 = perfect match, 0.0 = completely wrong."""
result = llm_judge(prompt)
return {"correctness": result["score"], "correctness_reason": result["reason"]}
@weave.op()
def score_relevance(question: str, sources: list, confidence: float, docs: list) -> dict:
"""
Are the retrieved chunks relevant to the question?
Passes actual chunk content to the judge, not just titles.
"""
# The full text of each retrieved chunk β€” the same content the answering model saw.
# (Truncated excerpts made the judge miss facts that were in the chunk.)
excerpts = "\n\n".join(
f"[Source {i+1}: {title}]\n{doc}"
for i, (title, doc) in enumerate(zip(sources, docs))
)
prompt = f"""Question: {question}
Retrieved chunks (actual content):
{excerpts}
Score 1.0 if the retrieved chunks contain information directly useful for answering
the question. Score 0.0 if the chunks are unrelated to the question."""
result = llm_judge(prompt)
return {
"relevance": result["score"],
"relevance_reason": result["reason"],
"retrieval_confidence": confidence,
}
@weave.op()
def score_groundedness(answer: str, context: str, not_in_docs: bool) -> dict:
"""
Is the answer grounded in the retrieved context, or did the model hallucinate?
"""
if not_in_docs:
# For not_in_docs questions, groundedness = model didn't make things up
prompt = f"""Context provided to the model:
{context}
Model's answer: {answer}
Score 1.0 if the model's answer is consistent with (or correctly acknowledges gaps in)
the provided context. Score 0.0 if the model claims information not present in the context."""
else:
prompt = f"""Context provided to the model:
{context}
Model's answer: {answer}
Score how well the answer is supported by the provided context.
1.0 = fully grounded in context, 0.0 = answer contains claims not in context (hallucination)."""
result = llm_judge(prompt)
return {"groundedness": result["score"], "groundedness_reason": result["reason"]}
# ── Evaluation runner ─────────────────────────────────────────────────────────
@weave.op()
def evaluate_single(item: dict) -> dict:
question = item["question"]
expected = item["expected"]
not_in_docs = item["not_in_docs"]
# Run RAG pipeline
result = rag_pipeline(question)
# Score
correctness = score_correctness(question, result["answer"], expected, not_in_docs)
relevance = score_relevance(question, result["sources"], result["confidence"], result["docs"])
groundedness = score_groundedness(result["answer"], result["context"], not_in_docs)
return {
"question": question,
"answer": result["answer"],
"expected": expected,
"not_in_docs": not_in_docs,
"sources": result["sources"],
**correctness,
**relevance,
**groundedness,
"composite_score": round(
(correctness["correctness"] + relevance["relevance"] + groundedness["groundedness"]) / 3,
4,
),
}
def run_evaluation():
weave.init(os.getenv("WANDB_PROJECT", "aerospike-chatbot"))
print(f"Running evaluation on {len(EVAL_DATASET)} questions...\n")
results = []
for i, item in enumerate(EVAL_DATASET, 1):
print(f"[{i}/{len(EVAL_DATASET)}] {item['question'][:70]}...")
result = evaluate_single(item)
results.append(result)
print(f" correctness={result['correctness']:.2f} "
f"relevance={result['relevance']:.2f} "
f"groundedness={result['groundedness']:.2f} "
f"composite={result['composite_score']:.2f}")
# Summary
avg_correctness = sum(r["correctness"] for r in results) / len(results)
avg_relevance = sum(r["relevance"] for r in results) / len(results)
avg_groundedness = sum(r["groundedness"] for r in results) / len(results)
avg_composite = sum(r["composite_score"] for r in results) / len(results)
print(f"\n{'='*60}")
print(f"EVALUATION SUMMARY ({len(results)} questions)")
print(f"{'='*60}")
print(f" Avg Correctness : {avg_correctness:.3f}")
print(f" Avg Relevance : {avg_relevance:.3f}")
print(f" Avg Groundedness : {avg_groundedness:.3f}")
print(f" Avg Composite : {avg_composite:.3f}")
print("\nFull results available in W&B Weave dashboard.")
write_results(results)
return results
RESULTS_FILE = Path("results/eval_results.json")
def write_results(results: list[dict]):
"""Save a summary for the app's scorecard (GET /eval-summary) plus per-question detail."""
def avg(rows, key):
return round(sum(r[key] for r in rows) / len(rows), 3) if rows else None
in_scope = [r for r in results if not r["not_in_docs"]]
out_of_scope = [r for r in results if r["not_in_docs"]]
summary = {
"run_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"model": MODEL,
"questions": len(results),
"correctness": avg(in_scope, "correctness"),
"groundedness": avg(results, "groundedness"),
"retrieval_relevance": avg(in_scope, "relevance"),
"out_of_scope_handling": avg(out_of_scope, "correctness"),
"out_of_scope_questions": len(out_of_scope),
}
RESULTS_FILE.parent.mkdir(exist_ok=True)
RESULTS_FILE.write_text(json.dumps({"summary": summary, "results": results}, indent=2))
print(f"Results written to {RESULTS_FILE}")
if __name__ == "__main__":
run_evaluation()