Sync from GitHub via hub-sync

evals/run_eval.py

@@ -1,23 +1,11 @@
-"""Evaluation harness for Code Compass RAG system.
-
-Computes 4 core metrics:
-- Hit rate @ top-5 (retrieval quality)
-- Grounded answer rate (citation accuracy)
-- LLM-as-judge faithfulness (Claude 3.5 Sonnet via RAGAS)
-- Query latency P95 (responsiveness)
-"""
-
-import asyncio
 import json
 import os
-import sys
 import re
+import sys
 import time
 from pathlib import Path
-from collections import Counter, defaultdict
 from statistics import mean
 
-import requests
 from dotenv import load_dotenv
 
 SERVER_ROOT = Path(__file__).resolve().parents[1]
@@ -26,770 +14,241 @@ if str(SERVER_ROOT) not in sys.path:
 
 load_dotenv(SERVER_ROOT / ".env")
 
-from src.bedrock_claude import create_bedrock_runtime_client, generate_bedrock_claude_text
-from src.embeddings import EmbeddingGenerator
-
+from src.rag_system import CodebaseRAGSystem
 
-API_URL = os.getenv("CODEBASE_RAG_API_URL", "http://localhost:8000")
-REPO_ID = int(os.getenv("CODEBASE_RAG_REPO_ID", "1"))
-SESSION_ID = os.getenv("CODEBASE_RAG_SESSION_ID", "eval-session")
+EVAL_SESSION_KEY = "eval-session"
 TOP_K = int(os.getenv("CODEBASE_RAG_TOP_K", "8"))
-QUERY_TIMEOUT_SECONDS = int(os.getenv("CODEBASE_RAG_QUERY_TIMEOUT_SECONDS", "180"))
-QUERY_MAX_RETRIES = int(os.getenv("CODEBASE_RAG_QUERY_MAX_RETRIES", "5"))
-QUERY_RETRY_BASE_SECONDS = float(os.getenv("CODEBASE_RAG_QUERY_RETRY_BASE_SECONDS", "2"))
 EVAL_SET_PATH = Path(
-    os.getenv(
-        "CODEBASE_RAG_EVAL_SET",
-        Path(__file__).with_name("sample_eval_set.json"),
-    )
+    os.getenv("CODEBASE_RAG_EVAL_SET", Path(__file__).with_name("sample_eval_set.json"))
 )
+EVAL_OUTPUT_PATH = os.getenv("CODEBASE_RAG_EVAL_OUTPUT")
+ENABLE_FAITHFULNESS = os.getenv("CODEBASE_RAG_ENABLE_FAITHFULNESS", "1") == "1"
 
 
 def log(message: str):
-    """Log message to stderr with [eval] prefix."""
     print(f"[eval] {message}", file=sys.stderr, flush=True)
 
 
-def get_app_model_config():
-    llm_provider = os.getenv("LLM_PROVIDER", "bedrock").lower()
-    if llm_provider == "groq":
-        llm_model = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
-    elif llm_provider == "bedrock":
-        llm_model = os.getenv(
-            "BEDROCK_LLM_MODEL",
-            "anthropic.claude-3-5-sonnet-20240620-v1:0",
-        )
-    elif llm_provider == "vertex_ai":
-        llm_model = os.getenv("VERTEX_LLM_MODEL", "claude-3-5-sonnet@20240620")
-    else:
-        llm_model = "unknown"
-
-    embedding_provider = os.getenv("EMBEDDING_PROVIDER", "auto").lower()
-    if embedding_provider == "bedrock":
-        embedding_model = os.getenv("BEDROCK_EMBEDDING_MODEL", "cohere.embed-v3:0")
-    elif embedding_provider == "vertex_ai":
-        embedding_model = os.getenv("VERTEX_EMBEDDING_MODEL", "gemini-embedding-001")
-    elif embedding_provider == "openai":
-        embedding_model = os.getenv("EMBEDDING_MODEL", "text-embedding-3-small")
-    elif embedding_provider == "local":
-        embedding_model = os.getenv("EMBEDDING_MODEL") or os.getenv(
-            "LOCAL_EMBEDDING_MODEL", "nomic-ai/CodeRankEmbed"
-        )
-    else:
-        embedding_model = os.getenv("EMBEDDING_MODEL") or "auto"
-
-    eval_model = os.getenv(
-        "EVAL_MODEL",
-        os.getenv("BEDROCK_EVAL_MODEL", "anthropic.claude-3-5-sonnet-20240620-v1:0"),
-    )
-    return {
-        "llm_provider": llm_provider,
-        "llm_model": llm_model,
-        "embedding_provider": embedding_provider,
-        "embedding_model": embedding_model,
-        "eval_model": eval_model,
-    }
-
-
-def load_eval_rows():
+def load_eval_set():
     return json.loads(EVAL_SET_PATH.read_text())
 
 
-def post_query(row):
-    payload = {
-        "repo_id": REPO_ID,
-        "question": row["question"],
-        "top_k": TOP_K,
-        "history": row.get("turns", []),
-    }
-    case_id = row.get("id", row["question"])
-
-    for attempt in range(1, QUERY_MAX_RETRIES + 1):
-        response = requests.post(
-            f"{API_URL}/api/query",
-            json=payload,
-            headers={"X-Session-Id": SESSION_ID},
-            timeout=QUERY_TIMEOUT_SECONDS,
-        )
-        if response.ok:
-            return response.json()
-
-        detail = response.text
-        try:
-            parsed = response.json()
-            detail = parsed.get("detail") or parsed
-        except Exception:
-            pass
-
-        detail_text = str(detail)
-        is_retryable = response.status_code in {429, 500, 502, 503, 504} and any(
-            marker in detail_text
-            for marker in [
-                "ThrottlingException",
-                "throttled",
-                "Too many requests",
-                "timed out",
-                "timeout",
-                "ServiceUnavailable",
-                "temporarily unavailable",
-            ]
-        )
-        if is_retryable and attempt < QUERY_MAX_RETRIES:
-            retry_after = response.headers.get("Retry-After")
-            try:
-                wait_seconds = (
-                    float(retry_after)
-                    if retry_after
-                    else QUERY_RETRY_BASE_SECONDS * (2 ** (attempt - 1))
-                )
-            except ValueError:
-                wait_seconds = QUERY_RETRY_BASE_SECONDS * (2 ** (attempt - 1))
-            log(
-                f"Retrying case {case_id} after transient query failure "
-                f"(attempt {attempt}/{QUERY_MAX_RETRIES}, wait={wait_seconds:.1f}s): {detail_text}"
-            )
-            time.sleep(wait_seconds)
-            continue
-
-        raise RuntimeError(
-            f"Query failed for eval case {case_id!r} "
-            f"with status {response.status_code}: {detail}"
-        )
-
-    raise RuntimeError(f"Query failed for eval case {case_id!r}: exhausted retries")
+def validate_eval_set(repositories):
+    errors = []
+    seen_ids = set()
+    for repo in repositories:
+        if not repo.get("github_url"):
+            errors.append(f"repo {repo.get('id', '?')}: missing github_url")
+        if not repo.get("cases"):
+            errors.append(f"repo {repo.get('id', '?')}: has no cases")
+        for case in repo.get("cases", []):
+            case_id = case.get("id") or case.get("question", "?")
+            if case_id in seen_ids:
+                errors.append(f"duplicate case id: {case_id}")
+            seen_ids.add(case_id)
+            if not case.get("question", "").strip():
+                errors.append(f"{case_id}: missing question")
+            if not case.get("ground_truth", "").strip():
+                errors.append(f"{case_id}: missing ground_truth")
+            if not case.get("expected_sources"):
+                errors.append(f"{case_id}: expected_sources must be a non-empty list")
+    return errors
 
 
 def normalize_path(path: str) -> str:
     return path.strip().lstrip("./").lower()
 
 
-STOPWORDS = {
-    "a",
-    "an",
-    "and",
-    "are",
-    "as",
-    "at",
-    "be",
-    "by",
-    "for",
-    "from",
-    "how",
-    "in",
-    "into",
-    "is",
-    "it",
-    "its",
-    "of",
-    "on",
-    "or",
-    "that",
-    "the",
-    "their",
-    "this",
-    "to",
-    "what",
-    "when",
-    "where",
-    "which",
-    "with",
-}
-
-
 def tokenize_text(text: str):
-    tokens = []
-    for raw_token in re.findall(r"[A-Za-z0-9_./+-]+", text or ""):
-        token = raw_token.lower()
-        tokens.append(token)
+    return re.findall(r"[a-z0-9_]+", (text or "").lower())
 
-        camel_parts = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", raw_token).split()
-        split_parts = re.split(r"[._/+-]+", token)
-        for part in [*camel_parts, *split_parts]:
-            normalized = part.strip().lower()
-            if normalized and normalized != token:
-                tokens.append(normalized)
 
-    return tokens
-
-
-def normalize_keywords(keywords):
-    normalized = []
-    seen = set()
-    for keyword in keywords or []:
-        phrase = " ".join(tokenize_text(str(keyword)))
-        if not phrase or phrase in seen:
-            continue
-        seen.add(phrase)
-        normalized.append(phrase)
-    return normalized
+def matches_expected(actual_path: str, expected_sources) -> bool:
+    actual = normalize_path(actual_path)
+    for expected in expected_sources:
+        expected_norm = normalize_path(expected).rstrip("/")
+        is_dir = "." not in expected_norm.rsplit("/", 1)[-1]
+        if actual == expected_norm:
+            return True
+        if is_dir and actual.startswith(expected_norm + "/"):
+            return True
+    return False
 
 
 def compute_retrieval_metrics(expected_sources, actual_sources):
-    """Compute retrieval metrics: hit rate and top-1 hit for given rank k."""
-    expected = {normalize_path(path) for path in expected_sources}
-    actual = [normalize_path(path) for path in actual_sources]
-
-    def matches_expected(actual_path: str) -> bool:
-        for expected_path in expected:
-            expected_is_directory = (
-                expected_path.endswith("/")
-                or "." not in expected_path.rsplit("/", 1)[-1]
-            )
-            normalized_expected = expected_path.rstrip("/")
-            if actual_path == expected_path:
-                return True
-            if expected_is_directory and actual_path.startswith(normalized_expected + "/"):
-                return True
-        return False
-
-    # Hit rate: was any retrieved source relevant?
-    hit = 1 if any(matches_expected(path) for path in actual) else 0
-
-    # Top-1 hit: was the first retrieved source relevant?
-    top1_hit = 1 if actual and matches_expected(actual[0]) else 0
+    hit = any(matches_expected(path, expected_sources) for path in actual_sources)
+    top1 = bool(actual_sources) and matches_expected(actual_sources[0], expected_sources)
+    return {"retrieval_hit": int(hit), "top1_hit": int(top1)}
 
-    return {
-        "retrieval_hit": hit,
-        "top1_hit": top1_hit,
-    }
 
-
-def keyword_match_details(row, answer: str):
-    keywords = normalize_keywords(row.get("must_include_any", []))
+def keyword_hits(answer: str, keywords):
     if not keywords:
-        return None
-
-    answer_tokens = tokenize_text(answer)
-    if not answer_tokens:
-        return {
-            "coverage": 0.0,
-            "matched_count": 0,
-            "total_keywords": len(keywords),
-            "matched_keywords": [],
-            "missing_keywords": keywords,
-        }
-
-    matched_keywords = []
+        return 0, 0
+    tokens = set(tokenize_text(answer))
+    matched = 0
     for keyword in keywords:
-        keyword_tokens = keyword.split()
-        window = len(keyword_tokens)
-        if window == 1:
-            if keyword_tokens[0] in answer_tokens:
-                matched_keywords.append(keyword)
-            continue
-
-def answer_length_metrics(answer: str):
-    """Check if answer has substantive content."""
-    tokens = tokenize_text(answer)
-    return {
-        "answer_word_count": len(tokens),
-        "has_substantive_answer": 1 if len(tokens) >= 40 else 0,
-    }
-
-
-def validate_eval_rows(rows):
-    errors = []
-    warnings = []
-    category_counts = Counter()
-    id_counts = Counter()
-    id_prefix_counts = Counter()
-    expected_source_counts = []
-    keyword_counts = []
-    conversation_cases = 0
-    benchmark_scope = {
-        "type": "mixed_or_unknown",
-        "dominant_prefix": None,
-        "dominant_prefix_fraction": 0.0,
-    }
+        keyword_tokens = tokenize_text(keyword)
+        if keyword_tokens and all(token in tokens for token in keyword_tokens):
+            matched += 1
+    return matched, len(keywords)
 
-    for index, row in enumerate(rows, start=1):
-        row_id = row.get("id") or f"row-{index}"
-        id_counts[row_id] += 1
-        prefix = row_id.split("-", 1)[0].lower()
-        if prefix:
-            id_prefix_counts[prefix] += 1
-        category_counts[row.get("category", "general")] += 1
-
-        question = str(row.get("question", "")).strip()
-        ground_truth = str(row.get("ground_truth", "")).strip()
-        expected_sources = row.get("expected_sources", [])
-        must_include_any = row.get("must_include_any", [])
-
-        if not question:
-            errors.append(f"{row_id}: missing question")
-        if not ground_truth:
-            errors.append(f"{row_id}: missing ground_truth")
-        if not isinstance(expected_sources, list) or not expected_sources:
-            errors.append(f"{row_id}: expected_sources must be a non-empty list")
-        if must_include_any and not isinstance(must_include_any, list):
-            errors.append(f"{row_id}: must_include_any must be a list when present")
-        if isinstance(must_include_any, list):
-            normalized_keywords = normalize_keywords(must_include_any)
-            if len(normalized_keywords) != len([keyword for keyword in must_include_any if str(keyword).strip()]):
-                warnings.append(
-                    f"{row_id}: duplicate or case-variant keywords were normalized; "
-                    "resume metrics are stricter than the raw checklist wording."
-                )
-        if row.get("turns"):
-            conversation_cases += 1
-        expected_source_counts.append(len(expected_sources) if isinstance(expected_sources, list) else 0)
-        keyword_counts.append(len(must_include_any) if isinstance(must_include_any, list) else 0)
-
-    duplicate_ids = sorted(row_id for row_id, count in id_counts.items() if count > 1)
-    if duplicate_ids:
-        errors.append(f"duplicate ids found: {', '.join(duplicate_ids)}")
-
-    if len(rows) < 25:
-        warnings.append(
-            "Eval set has fewer than 25 cases. Good for iteration, but light for resume-grade benchmarking."
-        )
-    if len(category_counts) < 4:
-        warnings.append("Eval set covers fewer than 4 categories, so breadth is limited.")
-    if conversation_cases < 2:
-        warnings.append("Eval set has very little multi-turn coverage.")
-    if category_counts and min(category_counts.values()) < 2:
-        sparse = sorted(category for category, count in category_counts.items() if count < 2)
-        warnings.append(f"Some categories are underrepresented: {', '.join(sparse)}.")
-
-    if id_prefix_counts:
-        dominant_prefix, dominant_count = id_prefix_counts.most_common(1)[0]
-        dominant_prefix_fraction = dominant_count / len(rows)
-        if dominant_prefix_fraction >= 0.8:
-            benchmark_scope = {
-                "type": "single_repository",
-                "dominant_prefix": dominant_prefix,
-                "dominant_prefix_fraction": round(dominant_prefix_fraction, 4),
-            }
 
-    return {
-        "case_count": len(rows),
-        "category_counts": dict(sorted(category_counts.items())),
-        "conversation_case_count": conversation_cases,
-        "average_expected_sources": round(mean(expected_source_counts), 2) if expected_source_counts else 0.0,
-        "average_keywords_per_case": round(mean(keyword_counts), 2) if keyword_counts else 0.0,
-        "benchmark_scope": benchmark_scope,
-        "errors": errors,
-        "warnings": warnings,
-        "is_valid": not errors,
-    }
-
-
-def summarize_custom_metrics(details, latency_p95=None):
-    """Compute only the 4 core metrics: hit rate @ top-5, grounded answer rate, faithfulness, latency P95."""
-    # Grounded answer: retrieval hit AND has substantive answer AND no failed keyword checks
-    grounded_answer_passes = [
-        1
-        for item in details
-        if item["retrieval_hit"] == 1
-        and item["has_substantive_answer"] == 1
-    ]
-    return {
-        "retrieval_hit_rate": round(mean(item["retrieval_hit"] for item in details), 4),
-        "top1_hit_rate": round(mean(item["top1_hit"] for item in details), 4),
-        "grounded_answer_rate": round(sum(grounded_answer_passes) / len(details), 4) if details else 0.0,
-        "latency_p95_ms": round(latency_p95, 2) if latency_p95 is not None else None,
-    }
+def judge_faithfulness(rag_system, question: str, answer: str, sources: list):
+    if not ENABLE_FAITHFULNESS or not answer.strip() or not sources:
+        return None
+    context = "\n\n".join(
+        f"[{i}] {source['file_path']}\n{source['snippet'][:800]}"
+        for i, source in enumerate(sources, start=1)
+    )
+    system_prompt = (
+        "You are a strict grading assistant. Given a question, retrieved code context, and a "
+        "generated answer, output ONLY a single number between 0 and 1 for how faithful the "
+        "answer is to the context (1.0 = every claim is supported, 0.0 = the answer invents or "
+        "contradicts facts not in the context). Output just the number."
+    )
+    user_prompt = f"Question: {question}\n\nContext:\n{context}\n\nAnswer:\n{answer}\n\nFaithfulness score:"
+    try:
+        text, _ = rag_system._generate_markdown_response(system_prompt, user_prompt)
+        match = re.search(r"(\d(?:\.\d+)?)", text)
+        if not match:
+            return None
+        return max(0.0, min(1.0, float(match.group(1))))
+    except Exception as exc:
+        log(f"Faithfulness judge failed: {exc}")
+        return None
 
 
-def summarize_by_category(details):
-    """Summarize metrics by category using only the 4 core metrics."""
-    grouped = defaultdict(list)
-    for item in details:
-        grouped[item["category"]].append(item)
-
-    summary = {}
-    for category, items in sorted(grouped.items()):
-        summary[category] = {
-            "case_count": len(items),
-            "retrieval_hit_rate": round(mean(item["retrieval_hit"] for item in items), 4),
-            "top1_hit_rate": round(mean(item["top1_hit"] for item in items), 4),
-            "grounded_answer_rate": round(
-                mean(
-                    1
-                    if item["retrieval_hit"] == 1 and item["has_substantive_answer"] == 1
-                    else 0
-                    for item in items
-                ),
-                4,
-            ),
-        }
-    return summary
-
-
-def build_headline_metrics(custom_metrics, audit):
-    """Build headline metrics section with only the 4 core metrics."""
-    return {
-        "sample_size": audit["case_count"],
-        "category_count": len(audit["category_counts"]),
-        "retrieval_hit_rate": custom_metrics["retrieval_hit_rate"],
-        "top1_hit_rate": custom_metrics["top1_hit_rate"],
-        "grounded_answer_rate": custom_metrics["grounded_answer_rate"],
-        "latency_p95_ms": custom_metrics["latency_p95_ms"],
-    }
+def index_repo(rag_system, github_url: str, name: str):
+    repo = rag_system.create_or_reset_repository(github_url, EVAL_SESSION_KEY)
+    log(f"Indexing {name} ({github_url}), repo_id={repo.id}")
+    rag_system.index_repository(repo.id)
+    repo_state = rag_system.get_repository_for_session(repo.id, EVAL_SESSION_KEY)
+    if not repo_state or repo_state["status"] != "indexed":
+        detail = repo_state.get("error_message") if repo_state else "repository disappeared"
+        raise RuntimeError(f"Failed to index {name}: {detail}")
+    log(
+        f"Indexed {name}: {repo_state['file_count']} files, "
+        f"{repo_state['chunk_count']} chunks"
+    )
+    return repo.id
 
 
-def build_metric_guidance(custom_metrics, ragas_report):
-    """Build guidance using only the 4 core metrics."""
-    # Primary gate: retrieval hit rate >= 80%
-    retrieval_gate_pass = custom_metrics["retrieval_hit_rate"] >= 0.8
+def run_case(rag_system, repo_id: int, repo_name: str, case: dict):
+    start = time.time()
+    result = rag_system.answer_question(
+        repo_id=repo_id,
+        session_key=EVAL_SESSION_KEY,
+        question=case["question"],
+        top_k=TOP_K,
+        history=case.get("turns", []),
+    )
+    elapsed_ms = (time.time() - start) * 1000
 
-    next_focus = []
-    if custom_metrics["grounded_answer_rate"] < 0.75:
-        next_focus.append("Tighten answer grounding to ensure answers cite sources.")
-    if custom_metrics["latency_p95_ms"] and custom_metrics["latency_p95_ms"] > 5000:
-        next_focus.append("Optimize query latency for better responsiveness.")
+    sources = result.get("sources", [])
+    cited_paths = [source["file_path"] for source in sources]
+    retrieval = compute_retrieval_metrics(case.get("expected_sources", []), cited_paths)
+    matched, total_keywords = keyword_hits(result.get("answer", ""), case.get("must_include_any", []))
+    has_citations = bool(result.get("citations"))
+    grounded = retrieval["retrieval_hit"] == 1 and has_citations and (total_keywords == 0 or matched > 0)
 
     return {
-        "primary_gate": "pass" if retrieval_gate_pass else "needs_work",
-        "primary_gate_basis": "retrieval_hit_rate",
-        "next_focus": next_focus,
+        "id": case.get("id", case["question"]),
+        "repo": repo_name,
+        "category": case.get("category", "general"),
+        "question": case["question"],
+        "answer": result.get("answer", ""),
+        "citations": result.get("citations", []),
+        "expected_sources": case.get("expected_sources", []),
+        "retrieved_sources": cited_paths,
+        "retrieval_hit": retrieval["retrieval_hit"],
+        "top1_hit": retrieval["top1_hit"],
+        "grounded": int(grounded),
+        "faithfulness": judge_faithfulness(rag_system, case["question"], result.get("answer", ""), sources),
+        "latency_ms": round(elapsed_ms, 1),
     }
 
 
-def build_resume_summary(custom_metrics, audit, ragas_report, ragas_error):
-    """Build resume summary using only the 4 core metrics: hit rate top-5, grounded answer rate, faithfulness, latency."""
-    lines = [
-        (
-            f"Evaluated on {audit['case_count']} repo-QA cases across "
-            f"{len(audit['category_counts'])} categories."
-        ),
-        (
-            f"Retrieval hit rate @ top-5: {custom_metrics['retrieval_hit_rate']:.1%}, "
-            f"top-1 hit rate: {custom_metrics['top1_hit_rate']:.1%}."
-        ),
-        (
-            f"Grounded answer rate: {custom_metrics['grounded_answer_rate']:.1%}."
-        ),
-    ]
-
-    if ragas_report and not ragas_error:
-        lines.append(
-            f"Faithfulness (Claude 3.5 Sonnet judge): {ragas_report.get('faithfulness', 0.0):.3f}."
-        )
-    else:
-        lines.append("Faithfulness metrics skipped or unavailable.")
-
-    if custom_metrics["latency_p95_ms"] is not None:
-        lines.append(f"Query latency P95: {custom_metrics['latency_p95_ms']:.0f}ms.")
-
-    scope = audit.get("benchmark_scope", {})
-    if scope.get("type") == "single_repository":
-        lines.append(
-            "Benchmark scope: single-repository benchmark "
-            f"({scope.get('dominant_prefix')}); use it to judge this target repo, not cross-repo generalization."
-        )
-
-    if audit["warnings"]:
-        lines.append(
-            "Benchmark caveat: "
-            + " ".join(audit["warnings"][:2])
-        )
-
-    return " ".join(lines)
-
-
-def benchmark_readiness(audit, ragas_error, metric_guidance=None):
-    reasons = []
-    if audit["case_count"] < 25:
-        reasons.append("small_sample")
-    if len(audit["category_counts"]) < 4:
-        reasons.append("limited_category_coverage")
-    if audit["conversation_case_count"] < 2:
-        reasons.append("limited_multi_turn_coverage")
-    if audit["warnings"]:
-        reasons.append("eval_set_warnings")
-    if ragas_error not in {None, "disabled"}:
-        reasons.append("ragas_instability")
-    if metric_guidance and metric_guidance.get("primary_gate") != "pass":
-        reasons.append("primary_gate_failed")
-
-    if reasons:
-        status = "single_repo_benchmark_needs_work"
-        if audit.get("benchmark_scope", {}).get("type") != "single_repository":
-            status = "internal_or_demo_benchmark"
-        return {
-            "status": status,
-            "reasons": reasons,
-        }
-    if audit.get("benchmark_scope", {}).get("type") == "single_repository":
-        return {
-            "status": "single_repo_benchmark_ready",
-            "reasons": [],
-        }
+def summarize(details):
+    if not details:
+        return {}
+    latencies = sorted(item["latency_ms"] for item in details)
+    p95_index = min(len(latencies) - 1, int(len(latencies) * 0.95))
+    faith_scores = [item["faithfulness"] for item in details if item["faithfulness"] is not None]
     return {
-        "status": "presentation_ready",
-        "reasons": [],
+        "case_count": len(details),
+        "retrieval_hit_rate": round(mean(item["retrieval_hit"] for item in details), 4),
+        "top1_hit_rate": round(mean(item["top1_hit"] for item in details), 4),
+        "grounded_answer_rate": round(mean(item["grounded"] for item in details), 4),
+        "faithfulness": round(mean(faith_scores), 4) if faith_scores else None,
+        "latency_p95_ms": round(latencies[p95_index], 1),
     }
 
 
-def maybe_write_report(report):
-    output_path = os.getenv("CODEBASE_RAG_EVAL_OUTPUT")
-    if not output_path:
-        return None
-    target = Path(output_path)
-    target.parent.mkdir(parents=True, exist_ok=True)
-    target.write_text(json.dumps(report, indent=2))
-    return str(target)
-
-
-def build_bedrock_ragas_llm(run_config):
-    from langchain_core.outputs import Generation, LLMResult
-    from ragas.llms.base import BaseRagasLLM
-
-    class BedrockRagasLLM(BaseRagasLLM):
-        def __init__(self, model: str, run_config):
-            self.client = create_bedrock_runtime_client()
-            self.model = model
-            self.set_run_config(run_config)
-
-        def _prompt_to_text(self, prompt):
-            prefix = (
-                "Return only valid JSON or the exact structured output requested by the prompt. "
-                "Do not add markdown fences, explanations, or extra prose.\n\n"
-            )
-            if hasattr(prompt, "to_string"):
-                return prefix + prompt.to_string()
-            return prefix + str(prompt)
-
-        def _generate_once(self, prompt, n=1, temperature=1e-8, stop=None, callbacks=None):
-            prompt_text = self._prompt_to_text(prompt)
-            text, _ = generate_bedrock_claude_text(
-                self.client,
-                self.model,
-                "Return only valid JSON or the exact structured output requested.",
-                prompt_text,
-                max_tokens=int(os.getenv("EVAL_MAX_OUTPUT_TOKENS", "2048")),
-                temperature=0.0,
-            )
-
-            generations = [Generation(text=text)] if text else []
-
-            if not generations:
-                raise RuntimeError("Bedrock Claude judge returned an empty response.")
-
-            return LLMResult(generations=[generations])
-
-        def generate_text(self, prompt, n=1, temperature=1e-8, stop=None, callbacks=None):
-            return self._generate_once(
-                prompt=prompt,
-                n=n,
-                temperature=temperature,
-                stop=stop,
-                callbacks=callbacks,
-            )
-
-        async def agenerate_text(self, prompt, n=1, temperature=1e-8, stop=None, callbacks=None):
-            return await asyncio.to_thread(
-                self._generate_once,
-                prompt,
-                n,
-                temperature,
-                stop,
-                callbacks,
-            )
-
-    model = os.getenv(
-        "EVAL_MODEL",
-        os.getenv("BEDROCK_EVAL_MODEL", "anthropic.claude-3-5-sonnet-20240620-v1:0"),
-    )
-    return BedrockRagasLLM(model=model, run_config=run_config)
-
-
-def build_ragas_embeddings(run_config):
-    from ragas.embeddings.base import BaseRagasEmbeddings
-
-    class AppEmbeddingWrapper(BaseRagasEmbeddings):
-        def __init__(self, generator, run_config):
-            self.generator = generator
-            self.set_run_config(run_config)
-
-        def embed_query(self, text):
-            return self.generator.embed_text(text).tolist()
-
-        def embed_documents(self, texts):
-            vectors = self.generator.embed_batch(list(texts))
-            return vectors.tolist()
+def summarize_by_repo(details):
+    grouped = {}
+    for item in details:
+        grouped.setdefault(item["repo"], []).append(item)
+    return {repo: summarize(items) for repo, items in grouped.items()}
 
-        async def aembed_query(self, text):
-            return await asyncio.to_thread(self.embed_query, text)
 
-        async def aembed_documents(self, texts):
-            return await asyncio.to_thread(self.embed_documents, texts)
+def summarize_by_category(details):
+    grouped = {}
+    for item in details:
+        grouped.setdefault(item["category"], []).append(item)
+    return {category: summarize(items) for category, items in sorted(grouped.items())}
 
-    return AppEmbeddingWrapper(EmbeddingGenerator(), run_config=run_config)
 
+def run():
+    eval_set = load_eval_set()
+    repositories = eval_set["repositories"]
 
-def run_ragas(rows, outputs):
-    if not ENABLE_RAGAS:
-        log("RAGAS disabled via CODEBASE_RAG_ENABLE_RAGAS=0. Reporting custom metrics only.")
-        return None, "disabled"
+    errors = validate_eval_set(repositories)
+    if errors:
+        raise RuntimeError("Eval set validation failed: " + "; ".join(errors))
 
-    try:
-        from datasets import Dataset
-        from ragas import evaluate
-        from ragas.metrics import faithfulness
-        from ragas.run_config import RunConfig
-    except Exception as exc:
-        log(f"Skipping RAGAS because the evaluation dependencies could not be loaded: {exc}")
-        return None, f"import_error: {exc}"
-
-    def build_ragas_dataset():
-        samples = []
-        for row, result in zip(rows, outputs):
-            samples.append(
-                {
-                    "question": row["question"],
-                    "answer": result["answer"],
-                    "contexts": [source["snippet"] for source in result.get("sources", [])],
-                    "ground_truth": row["ground_truth"],
-                }
-            )
-        return Dataset.from_list(samples)
-
-    log("Running RAGAS metrics. This can take a while.")
-    try:
-        timeout_seconds = int(os.getenv("EVAL_TIMEOUT_SECONDS", "180"))
-        thread_timeout_seconds = float(os.getenv("EVAL_THREAD_TIMEOUT_SECONDS", str(max(timeout_seconds, 240))))
-        max_workers = int(os.getenv("EVAL_MAX_WORKERS", "2"))
-        run_config = RunConfig(
-            timeout=timeout_seconds,
-            thread_timeout=thread_timeout_seconds,
-            max_workers=max_workers,
-            max_retries=int(os.getenv("EVAL_MAX_RETRIES", "3")),
-            max_wait=int(os.getenv("EVAL_MAX_WAIT_SECONDS", "60")),
-        )
-        log(
-            "Using Bedrock for RAGAS judge model "
-            f"({os.getenv('EVAL_MODEL', os.getenv('BEDROCK_EVAL_MODEL', 'anthropic.claude-3-5-sonnet-20240620-v1:0'))})"
-        )
-        log(
-            f"RAGAS runtime: async={RAGAS_ASYNC}, raise_exceptions={RAGAS_RAISE_EXCEPTIONS}, "
-            f"timeout={timeout_seconds}s, thread_timeout={thread_timeout_seconds}s, max_workers={max_workers}"
-        )
-        llm = build_bedrock_ragas_llm(run_config)
-        embeddings = build_ragas_embeddings(run_config)
-        # Only use faithfulness as the RAGAS metric (simplified to 4-core metrics)
-        ragas_report = evaluate(
-            build_ragas_dataset(),
-            metrics=[faithfulness],
-            llm=llm,
-            embeddings=embeddings,
-            run_config=run_config,
-            is_async=RAGAS_ASYNC,
-            raise_exceptions=RAGAS_RAISE_EXCEPTIONS,
-        )
-        return {key: float(value) for key, value in ragas_report.items()}, None
-    except Exception as exc:
-        log(f"RAGAS evaluation failed: {exc}")
-        return None, str(exc)
+    total_cases = sum(len(repo["cases"]) for repo in repositories)
+    log(f"Loaded eval set: {len(repositories)} repositories, {total_cases} cases")
 
+    rag_system = CodebaseRAGSystem()
+    log(f"LLM provider={rag_system.llm_provider} model={rag_system.llm_model}")
 
-def run():
-    log(f"Loading eval set from {EVAL_SET_PATH}")
-    rows = load_eval_rows()
-    audit = validate_eval_rows(rows)
-    model_config = get_app_model_config()
-    if audit["errors"]:
-        raise RuntimeError("Eval set validation failed: " + "; ".join(audit["errors"]))
-    for warning in audit["warnings"]:
-        log(f"Eval set warning: {warning}")
-    log(
-        "Eval model config: "
-        f"qna_provider={model_config['llm_provider']}, "
-        f"qna_model={model_config['llm_model']}, "
-        f"embedding_provider={model_config['embedding_provider']}, "
-        f"embedding_model={model_config['embedding_model']}, "
-        f"judge_model={model_config['eval_model']}"
-    )
-    log(
-        f"Starting eval with api_url={API_URL}, repo_id={REPO_ID}, "
-        f"session_id={SESSION_ID}, top_k={TOP_K}, cases={len(rows)}"
-    )
-    outputs = []
     details = []
-    latencies = []
-
-    for index, row in enumerate(rows, start=1):
-        case_id = row.get("id", row["question"])
-        log(f"[{index}/{len(rows)}] Querying case {case_id}")
-        start_time = time.time()
-        result = post_query(row)
-        elapsed_ms = (time.time() - start_time) * 1000
-        latencies.append(elapsed_ms)
-        outputs.append(result)
-        log(
-            f"[{index}/{len(rows)}] Received answer for {case_id} "
-            f"with {len(result.get('sources', []))} sources in {elapsed_ms:.0f}ms"
-        )
-
-        cited_paths = [source["file_path"] for source in result.get("sources", [])]
-        metrics = compute_retrieval_metrics(row.get("expected_sources", []), cited_paths)
-        length_metrics = answer_length_metrics(result.get("answer", ""))
-
-        details.append(
-            {
-                "id": row.get("id", row["question"]),
-                "category": row.get("category", "general"),
-                "question": row["question"],
-                "answer": result.get("answer", ""),
-                "expected_sources": row.get("expected_sources", []),
-                "retrieved_sources": cited_paths,
-                "retrieval_hit": metrics["retrieval_hit"],
-                "top1_hit": metrics["top1_hit"],
-                **length_metrics,
-            }
-        )
-
-    # Compute P95 latency
-    latencies.sort()
-    p95_index = int(len(latencies) * 0.95)
-    latency_p95 = latencies[p95_index] if latencies else None
-
-    log("Finished query loop. Computing aggregate metrics.")
-    custom_metrics = summarize_custom_metrics(details, latency_p95)
-    category_breakdown = summarize_by_category(details)
-    ragas_report, ragas_error = run_ragas(rows, outputs)
-    headline_metrics = build_headline_metrics(custom_metrics, audit)
-    metric_guidance = build_metric_guidance(custom_metrics, ragas_report)
-    resume_summary = build_resume_summary(custom_metrics, audit, ragas_report, ragas_error)
-    readiness = benchmark_readiness(audit, ragas_error, metric_guidance)
+    try:
+        for repo_config in repositories:
+            repo_id = index_repo(rag_system, repo_config["github_url"], repo_config["name"])
+            cases = repo_config["cases"]
+            for index, case in enumerate(cases, start=1):
+                log(f"[{repo_config['id']} {index}/{len(cases)}] {case['id']}")
+                details.append(run_case(rag_system, repo_id, repo_config["name"], case))
+    finally:
+        rag_system.end_session(EVAL_SESSION_KEY)
 
     report = {
         "config": {
-            "api_url": API_URL,
-            "repo_id": REPO_ID,
-            "session_id": SESSION_ID,
+            "llm_provider": rag_system.llm_provider,
+            "llm_model": rag_system.llm_model,
             "top_k": TOP_K,
-            "qna_provider": model_config["llm_provider"],
-            "qna_model": model_config["llm_model"],
-            "embedding_provider": model_config["embedding_provider"],
-            "embedding_model": model_config["embedding_model"],
-            "eval_model": model_config["eval_model"],
-            "query_timeout_seconds": QUERY_TIMEOUT_SECONDS,
-            "query_max_retries": QUERY_MAX_RETRIES,
-            "query_retry_base_seconds": QUERY_RETRY_BASE_SECONDS,
             "eval_set": str(EVAL_SET_PATH),
-            "min_reference_overlap": MIN_REFERENCE_OVERLAP,
-            "min_reference_term_matches": MIN_REFERENCE_TERM_MATCHES,
+            "repositories": [
+                {"id": repo["id"], "name": repo["name"], "github_url": repo["github_url"]}
+                for repo in repositories
+            ],
         },
-        "eval_set_audit": audit,
-        "headline_metrics": headline_metrics,
-        "benchmark_readiness": readiness,
-        "metric_guidance": metric_guidance,
-        "ragas": ragas_report,
-        "ragas_error": ragas_error,
-        "custom_metrics": custom_metrics,
-        "category_breakdown": category_breakdown,
-        "resume_summary": resume_summary,
+        "headline_metrics": summarize(details),
+        "repo_breakdown": summarize_by_repo(details),
+        "category_breakdown": summarize_by_category(details),
         "cases": details,
     }
-    output_path = maybe_write_report(report)
-    if output_path:
-        log(f"Wrote JSON report to {output_path}")
+
+    if EVAL_OUTPUT_PATH:
+        target = Path(EVAL_OUTPUT_PATH)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(report, indent=2))
+        log(f"Wrote JSON report to {target}")
 
     log("Eval complete. Printing JSON report.")
     print(json.dumps(report, indent=2))
 
 
 if __name__ == "__main__":
-    run()
+    run()

evals/sample_eval_set.json

@@ -1,889 +1,229 @@
-[
-  {
-    "id": "documenso-purpose",
-    "category": "architecture",
-    "question": "What is Documenso and what product problem is it trying to solve?",
-    "ground_truth": "Documenso is an open-source document signing platform and DocuSign alternative. It lets users create, send, and sign documents electronically while emphasizing self-hosting, trust, and the ability to inspect how the signing system works under the hood.",
-    "expected_sources": [
-      "README.md",
-      "MANIFEST.md",
-      "ARCHITECTURE.md"
-    ],
-    "must_include_any": [
-      "document signing",
-      "self-host",
-      "DocuSign",
-      "open trust"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-monorepo-shape",
-    "category": "architecture",
-    "question": "How is the Documenso monorepo organized at a high level?",
-    "ground_truth": "The repo is organized as a Turborepo/npm-workspaces monorepo. Applications live under apps, including the main Remix/React Router application, documentation site, and openpage-api public analytics API, while shared domain and infrastructure packages live under packages, including lib, trpc, api, prisma, ui, email, auth, and signing.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "package.json",
-      "turbo.json",
-      "apps",
-      "packages"
-    ],
-    "must_include_any": [
-      "monorepo",
-      "Turborepo",
-      "apps",
-      "packages"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-remix-hono-app",
-    "category": "architecture",
-    "question": "What role does apps/remix play in Documenso?",
-    "ground_truth": "apps/remix is the main application. The architecture describes it as a React Router/Remix app served by a Hono server, exposing UI routes alongside API mounts such as /api/v1, /api/v2, /api/trpc, and /api/jobs.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "apps/remix",
-      "apps/remix/server",
-      "apps/remix/app"
-    ],
-    "must_include_any": [
-      "React Router",
-      "Remix",
-      "Hono",
-      "apps/remix"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-package-responsibilities",
-    "category": "architecture",
-    "question": "What are the main responsibilities of the core packages in Documenso?",
-    "ground_truth": "The core packages split responsibilities by layer: @documenso/lib holds server-only, client-only, and universal business logic; @documenso/trpc provides the current API V2 layer; @documenso/api maintains the older REST API V1; @documenso/prisma owns database access; @documenso/email owns React Email templates and mail delivery; @documenso/auth handles authentication; and @documenso/signing handles PDF signing.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib",
-      "packages/trpc",
-      "packages/api",
-      "packages/prisma",
-      "packages/email",
-      "packages/auth",
-      "packages/signing"
-    ],
-    "must_include_any": [
-      "lib",
-      "trpc",
-      "prisma",
-      "email",
-      "signing"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-api-architecture-overview",
-    "category": "architecture",
-    "question": "How does Documenso separate API V1, API V2, and internal tRPC APIs?",
-    "ground_truth": "Documenso keeps API V1 in packages/api/v1 using ts-rest, marks it as deprecated but maintained, and mounts it under /api/v1. API V2 lives under packages/trpc/server, uses tRPC plus trpc-to-openapi, and is mounted under /api/v2 and /api/v2-beta. Internal frontend-to-backend tRPC is mounted under /api/trpc and uses session-based auth.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/api",
-      "packages/trpc/server",
-      "apps/remix/server"
-    ],
-    "must_include_any": [
-      "API V1",
-      "API V2",
-      "tRPC",
-      "ts-rest",
-      "OpenAPI"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-readme-positioning",
-    "category": "docs",
-    "question": "How does the README position Documenso to someone evaluating the project?",
-    "ground_truth": "The README presents Documenso as the open-source DocuSign alternative and frames its mission around making digital document signing fast, easy, trustworthy, self-hostable, and inspectable under the hood.",
-    "expected_sources": [
-      "README.md"
-    ],
-    "must_include_any": [
-      "DocuSign",
-      "open-source",
-      "self-host",
-      "trust"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-readme-tech-stack",
-    "category": "docs",
-    "question": "What technology stack does the README advertise for Documenso?",
-    "ground_truth": "The README lists a TypeScript application using React Router, Prisma, Tailwind, shadcn/ui, React Email, tRPC, PDF signing tooling, React-PDF, PDF-Lib, and Stripe.",
-    "expected_sources": [
-      "README.md",
-      "ARCHITECTURE.md",
-      "package.json"
-    ],
-    "must_include_any": [
-      "TypeScript",
-      "Prisma",
-      "tRPC",
-      "React Email",
-      "Stripe"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-local-dev-quickstart",
-    "category": "docs",
-    "question": "What local development workflow does the README recommend for getting Documenso running quickly?",
-    "ground_truth": "The README and developer docs recommend using Node.js, Postgres, and optionally Docker. The quickstart has users copy or create an env file, run npm run dx to start supporting services such as Postgres and Inbucket, then run npm run dev for the app.",
-    "expected_sources": [
-      "README.md",
-      ".env.example",
-      "apps/docs/content/docs/developers/local-development/quickstart.mdx",
-      "docker"
-    ],
-    "must_include_any": [
-      "Node.js",
-      "Postgres",
-      "Docker",
-      "npm run dx",
-      "npm run dev"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-self-hosting-docs",
-    "category": "docs",
-    "question": "What does the project documentation say about self-hosting Documenso?",
-    "ground_truth": "The README and docs describe self-hosting by cloning the repository, copying .env.example to .env, setting required web, database, encryption, signing, and SMTP-related variables, then deploying with Docker, Docker Compose, or a manual flow. The docs also cover configuration and maintenance topics.",
-    "expected_sources": [
-      "README.md",
-      ".env.example",
-      "docker",
-      "apps/docs/content/docs/self-hosting"
-    ],
-    "must_include_any": [
-      "self-host",
-      ".env.example",
-      "database",
-      "SMTP",
-      "Docker"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-docs-app-purpose",
-    "category": "docs",
-    "question": "What is the purpose of the apps/docs application in the repository?",
-    "ground_truth": "apps/docs is the documentation site application. ARCHITECTURE.md identifies it as the docs site, while the app's own README, package.json, and source config show the current implementation as a Next.js/Fumadocs MDX app with content loading, docs layout, and a search route.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "apps/docs/README.md",
-      "apps/docs/package.json",
-      "apps/docs/source.config.ts",
-      "apps/docs/src/lib/source.ts",
-      "apps/docs/src/app/api/search/route.ts"
-    ],
-    "must_include_any": [
-      "documentation",
-      "Next.js",
-      "Fumadocs",
-      "search"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-required-env",
-    "category": "config-setup",
-    "question": "Which environment variables are central to running a self-hosted Documenso instance?",
-    "ground_truth": "The setup expects values such as NEXTAUTH_SECRET, NEXT_PRIVATE_ENCRYPTION_KEY, NEXT_PRIVATE_ENCRYPTION_SECONDARY_KEY, NEXT_PUBLIC_WEBAPP_URL, NEXT_PRIVATE_DATABASE_URL, NEXT_PRIVATE_DIRECT_DATABASE_URL, NEXT_PRIVATE_SMTP_FROM_NAME, and NEXT_PRIVATE_SMTP_FROM_ADDRESS. The env example and self-hosting docs also cover internal URLs, signing transport, storage transport, and optional OAuth and webhook configuration.",
-    "expected_sources": [
-      "README.md",
-      ".env.example",
-      "apps/docs/content/docs/self-hosting/configuration/environment.mdx"
-    ],
-    "must_include_any": [
-      "NEXTAUTH_SECRET",
-      "NEXT_PUBLIC_WEBAPP_URL",
-      "DATABASE_URL",
-      "SMTP",
-      "encryption"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-database-setup",
-    "category": "config-setup",
-    "question": "How is the database layer configured and managed in Documenso?",
-    "ground_truth": "Documenso uses PostgreSQL with Prisma and Kysely. Database connection URLs are configured through NEXT_PRIVATE_DATABASE_URL and NEXT_PRIVATE_DIRECT_DATABASE_URL, the self-hosting docs describe database setup, and schema or migration work is handled from the packages/prisma package with commands such as prisma migrations.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      ".env.example",
-      "apps/docs/content/docs/self-hosting/configuration/database.mdx",
-      "packages/prisma",
-      "packages/prisma/schema.prisma"
-    ],
-    "must_include_any": [
-      "PostgreSQL",
-      "Prisma",
-      "Kysely",
-      "DATABASE_URL",
-      "migration"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-signing-config",
-    "category": "config-setup",
-    "question": "How is document signing configured for local and cloud-backed signing?",
-    "ground_truth": "Signing is configured with NEXT_PRIVATE_SIGNING_TRANSPORT. The env example and self-hosting docs document local signing as the default and also support gcloud-hsm with related key, certificate, credential, certificate chain, and timestamp authority configuration.",
-    "expected_sources": [
-      ".env.example",
-      "apps/docs/content/docs/self-hosting/configuration/signing-certificate",
-      "packages/signing",
-      "packages/signing/transports"
-    ],
-    "must_include_any": [
-      "NEXT_PRIVATE_SIGNING_TRANSPORT",
-      "local",
-      "gcloud-hsm",
-      "certificate",
-      "timestamp"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-workspace-build-config",
-    "category": "config-setup",
-    "question": "Where should you look to understand package workspaces and build orchestration in Documenso?",
-    "ground_truth": "The root package.json defines npm workspaces and scripts, while turbo.json defines Turborepo task orchestration. ARCHITECTURE.md explains the monorepo layout and how apps and packages fit together.",
-    "expected_sources": [
-      "package.json",
-      "turbo.json",
-      "ARCHITECTURE.md"
-    ],
-    "must_include_any": [
-      "package.json",
-      "workspaces",
-      "turbo",
-      "Turborepo"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-api-v2-document-router",
-    "category": "api",
-    "question": "Where is the current document API implemented and how is it exposed?",
-    "ground_truth": "The current API V2 is implemented under packages/trpc/server, with document operations organized under a document router. It is exposed through /api/v2 and /api/v2-beta with tRPC and OpenAPI support, and it accepts API-token or session-cookie authentication depending on the route.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/trpc/server",
-      "packages/trpc/server/document-router",
-      "apps/remix/server"
-    ],
-    "must_include_any": [
-      "packages/trpc/server",
-      "document-router",
-      "API V2",
-      "OpenAPI",
-      "tRPC"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-api-v1-deprecated",
-    "category": "api",
-    "question": "What is the older API V1 layer in Documenso and what is its status?",
-    "ground_truth": "API V1 lives under packages/api/v1, uses ts-rest for contract-based REST, is mounted under /api/v1, and is described as deprecated but maintained.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/api",
-      "packages/api/v1"
-    ],
-    "must_include_any": [
-      "API V1",
-      "packages/api/v1",
-      "ts-rest",
-      "deprecated"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-internal-trpc",
-    "category": "api",
-    "question": "How does the frontend talk to the backend internally in Documenso?",
-    "ground_truth": "The frontend uses an internal tRPC API mounted under /api/trpc for frontend-to-backend communication. This internal API is separate from the public API V1 and V2 routes and uses session-based authentication.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "apps/remix/server/trpc",
-      "apps/remix/server/router.ts",
-      "packages/trpc"
-    ],
-    "must_include_any": [
-      "/api/trpc",
-      "tRPC",
-      "frontend",
-      "session"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-public-api-auth",
-    "category": "api",
-    "question": "How are public API requests authenticated in Documenso?",
-    "ground_truth": "Public API routes use API-token style authentication, typically through bearer or API-key headers. API V2 can also use session cookies where appropriate, while internal tRPC uses session-based auth.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/server-only/public-api",
-      "packages/trpc/server",
-      "packages/api"
-    ],
-    "must_include_any": [
-      "API Token",
-      "Bearer",
-      "session",
-      "public API"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-openapi-support",
-    "category": "api",
-    "question": "Where does OpenAPI support fit into Documenso's API architecture?",
-    "ground_truth": "OpenAPI support appears in both API layers. API V2 generates an OpenAPI document from the tRPC app router in packages/trpc/server/open-api.ts and mounts it from the Remix/Hono server. API V1 is the older ts-rest layer under packages/api/v1 and also exposes an OpenAPI document through packages/api/hono.ts.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "apps/remix/server/router.ts",
-      "packages/trpc/server/open-api.ts",
-      "packages/api/hono.ts",
-      "packages/api/v1/openapi.ts"
-    ],
-    "must_include_any": [
-      "OpenAPI",
-      "trpc-to-openapi",
-      "tRPC",
-      "API V2"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-document-business-logic",
-    "category": "specific-function",
-    "question": "Where is document business logic likely implemented in Documenso?",
-    "ground_truth": "Document business logic belongs primarily in packages/lib/server-only/document, with API-facing operations routed through packages/trpc/server/document-router or the legacy packages/api layer. Database persistence is supported by packages/prisma.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/server-only/document",
-      "packages/trpc/server/document-router",
-      "packages/prisma"
-    ],
-    "must_include_any": [
-      "server-only",
-      "document",
-      "document-router",
-      "prisma"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-recipient-field-logic",
-    "category": "specific-function",
-    "question": "Where should you look for recipient and field behavior in Documenso?",
-    "ground_truth": "Recipient and field behavior is split between packages/lib/server-only/recipient and packages/lib/server-only/field for core logic, with packages/trpc/server/recipient-router and packages/trpc/server/field-router exposing API operations.",
-    "expected_sources": [
-      "packages/lib/server-only/recipient",
-      "packages/lib/server-only/field",
-      "packages/trpc/server/recipient-router",
-      "packages/trpc/server/field-router",
-      "ARCHITECTURE.md"
-    ],
-    "must_include_any": [
-      "recipient",
-      "field",
-      "server-only",
-      "router"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-template-envelope-logic",
-    "category": "specific-function",
-    "question": "How are templates and envelopes represented in the codebase?",
-    "ground_truth": "Templates and envelopes have their own server-only domain areas under packages/lib/server-only/template and packages/lib/server-only/envelope. API V2 also organizes routes into template-router and envelope-router directories under packages/trpc/server.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/server-only/template",
-      "packages/lib/server-only/envelope",
-      "packages/trpc/server/template-router",
-      "packages/trpc/server/envelope-router"
-    ],
-    "must_include_any": [
-      "template",
-      "envelope",
-      "server-only",
-      "router"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-email-templates",
-    "category": "specific-function",
-    "question": "How are lifecycle emails represented in Documenso?",
-    "ground_truth": "Lifecycle emails live in the @documenso/email package, with React Email templates under packages/email/templates. The package also includes mailer, transports, providers, render helpers, and reusable template components.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/email",
-      "packages/email/templates",
-      "packages/email/mailer.ts",
-      "packages/email/render.tsx"
-    ],
-    "must_include_any": [
-      "React Email",
-      "templates",
-      "mailer",
-      "transports"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-signing-package",
-    "category": "specific-function",
-    "question": "What does the signing package do in Documenso?",
-    "ground_truth": "The @documenso/signing package owns PDF signing behavior. Its signPdf entry point selects a signing transport, applies timestamp authority settings when configured, and supports local P12 signing and Google Cloud KMS/HSM-backed signing through transport implementations.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/signing/index.ts",
-      "packages/signing/helpers",
-      "packages/signing/transports",
-      ".env.example"
-    ],
-    "must_include_any": [
-      "PDF signing",
-      "transports",
-      "local",
-      "Google",
-      "KMS"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-job-system",
-    "category": "specific-function",
-    "question": "What role do jobs play in Documenso?",
-    "ground_truth": "Jobs handle asynchronous operations such as email sending, document sealing, reminders, and webhooks. The architecture describes Inngest or local providers, with job definitions and clients under packages/lib/jobs.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/jobs",
-      "packages/lib/server-only/email",
-      "packages/lib/server-only/webhooks"
-    ],
-    "must_include_any": [
-      "jobs",
-      "Inngest",
-      "local",
-      "email",
-      "webhooks"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-document-send-flow",
-    "category": "cross-file",
-    "question": "How does a document send operation flow across the Documenso codebase?",
-    "ground_truth": "A document send operation starts at an API or UI route, goes through the API layer such as packages/trpc/server/document-router, delegates core behavior to packages/lib/server-only/document and related recipient or field logic, persists through packages/prisma, and can trigger emails or jobs through packages/email and packages/lib/jobs.",
-    "expected_sources": [
-      "packages/trpc/server/document-router",
-      "packages/lib/server-only/document",
-      "packages/lib/server-only/recipient",
-      "packages/prisma",
-      "packages/email",
-      "packages/lib/jobs"
-    ],
-    "must_include_any": [
-      "document-router",
-      "server-only/document",
-      "prisma",
-      "email",
-      "jobs"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-signing-completion-flow",
-    "category": "cross-file",
-    "question": "How do recipient fields, PDF handling, signing, and completion emails connect?",
-    "ground_truth": "Signing completion crosses several packages: recipient and field logic model who signs and which fields are completed, the seal-document job prepares the completed PDF, server-only PDF helpers insert fields/certificates/audit logs, @documenso/signing applies cryptographic signing, Prisma persists state, and document-completed email templates notify participants.",
-    "expected_sources": [
-      "packages/lib/server-only/recipient",
-      "packages/lib/server-only/field",
-      "packages/lib/server-only/pdf",
-      "packages/lib/jobs/definitions/internal/seal-document.handler.ts",
-      "packages/signing",
-      "packages/prisma",
-      "packages/email/templates/document-completed.tsx"
-    ],
-    "must_include_any": [
-      "recipient",
-      "field",
-      "PDF",
-      "signing",
-      "document-completed"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-webhook-job-flow",
-    "category": "cross-file",
-    "question": "How do background jobs and webhooks complement each other in Documenso?",
-    "ground_truth": "Background jobs handle asynchronous work, and webhook logic lives under packages/lib/server-only/webhooks. Together they allow document lifecycle events to be processed outside the immediate request path and delivered to external integrations.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/jobs",
-      "packages/lib/server-only/webhooks",
-      ".env.example"
-    ],
-    "must_include_any": [
-      "jobs",
-      "webhooks",
-      "asynchronous",
-      "events"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-ui-to-api-flow",
-    "category": "cross-file",
-    "question": "How does the Remix UI connect to server routes and shared API packages?",
-    "ground_truth": "The Remix UI is under apps/remix/app with route definitions and root app wiring. The server side under apps/remix/server wires routers and context, then connects to the shared packages/trpc APIs and packages/lib business logic.",
-    "expected_sources": [
-      "apps/remix/app/routes.ts",
-      "apps/remix/app/root.tsx",
-      "apps/remix/server/router.ts",
-      "apps/remix/server/context.ts",
-      "packages/trpc",
-      "packages/lib"
-    ],
-    "must_include_any": [
-      "apps/remix/app",
-      "apps/remix/server",
-      "router",
-      "context",
-      "tRPC"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-storage-pdf-flow",
-    "category": "cross-file",
-    "question": "How do storage, PDF processing, and database state fit together?",
-    "ground_truth": "The architecture separates storage provider concerns, PDF handling, and database state. Server-side PDF behavior lives under packages/lib/server-only/pdf, upload and download storage logic lives under packages/lib/universal/upload, document data records are created through packages/lib/server-only/document-data, and packages/prisma persists metadata and workflow state.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/lib/server-only/pdf",
-      "packages/lib/server-only/document-data",
-      "packages/prisma",
-      "packages/lib/universal/upload"
-    ],
-    "must_include_any": [
-      "storage",
-      "PDF",
-      "database",
-      "prisma",
-      "metadata"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-auth-session-flow",
-    "category": "cross-file",
-    "question": "How does authentication show up across Documenso's app and packages?",
-    "ground_truth": "Authentication spans the @documenso/auth package, server-only auth logic under packages/lib/server-only/auth, app server context in apps/remix/server/context.ts, and API layers that choose between session cookies and API tokens.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/auth",
-      "packages/lib/server-only/auth",
-      "apps/remix/server/context.ts",
-      "packages/trpc/server"
-    ],
-    "must_include_any": [
-      "auth",
-      "session",
-      "API token",
-      "context"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-public-api-errors",
-    "category": "error-handling",
-    "question": "Where would Documenso validate or reject bad public API requests?",
-    "ground_truth": "Bad public API requests are validated in the API layer and supporting server-only public API logic. The relevant code is under packages/trpc/server for API V2, packages/api for API V1, and packages/lib/server-only/public-api for shared public API behavior such as authentication, permissions, and validation.",
-    "expected_sources": [
-      "packages/trpc/server",
-      "packages/api",
-      "packages/lib/server-only/public-api"
-    ],
-    "must_include_any": [
-      "validation",
-      "API",
-      "auth",
-      "public-api"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-webhook-security-errors",
-    "category": "error-handling",
-    "question": "Where should you look for webhook security or SSRF-related safeguards?",
-    "ground_truth": "Webhook safeguards belong in packages/lib/server-only/webhooks, with related configuration documented in .env.example such as NEXT_PRIVATE_WEBHOOK_SSRF_BYPASS_HOSTS. These sources explain where outbound webhook behavior and security exceptions are controlled.",
-    "expected_sources": [
-      ".env.example",
-      "packages/lib/server-only/webhooks"
-    ],
-    "must_include_any": [
-      "webhook",
-      "SSRF",
-      "bypass",
-      "hosts"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-signing-config-errors",
-    "category": "error-handling",
-    "question": "Where would invalid signing configuration most likely be enforced?",
-    "ground_truth": "Invalid signing configuration is handled by the signing package and the code that reads signing environment variables. The key sources are .env.example for supported values, packages/signing/index.ts for transport selection and unsupported transport errors, and packages/signing/transports for local or gcloud-hsm certificate/key validation.",
-    "expected_sources": [
-      ".env.example",
-      "packages/signing/index.ts",
-      "packages/signing/transports",
-      "packages/signing/helpers"
-    ],
-    "must_include_any": [
-      "signing",
-      "transport",
-      "local",
-      "gcloud-hsm"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-playwright-tests",
-    "category": "tests",
-    "question": "Where does Documenso keep end-to-end app tests?",
-    "ground_truth": "The architecture identifies @documenso/app-tests as the E2E test package, and the packages/app-tests directory is intended for Playwright coverage of app behavior.",
-    "expected_sources": [
-      "ARCHITECTURE.md",
-      "packages/app-tests",
-      "packages/app-tests/package.json"
-    ],
-    "must_include_any": [
-      "app-tests",
-      "E2E",
-      "Playwright"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-package-test-config",
-    "category": "tests",
-    "question": "How can you find package-level test configuration for shared logic?",
-    "ground_truth": "Package-level tests and configuration can be found near the package they cover, such as packages/lib/vitest.config.ts for shared library tests and package.json scripts in the relevant package directories.",
-    "expected_sources": [
-      "packages/lib/vitest.config.ts",
-      "packages/lib/package.json",
-      "packages"
-    ],
-    "must_include_any": [
-      "vitest",
-      "package.json",
-      "packages/lib"
-    ],
-    "min_keyword_matches": 2
-  },
-  {
-    "id": "documenso-email-template-tests",
-    "category": "tests",
-    "question": "What should be tested when changing document lifecycle email behavior?",
-    "ground_truth": "A good answer should point to packages/email templates and rendering/mailer code, and should mention checking template output, variables, delivery behavior, and any app or E2E tests that cover document lifecycle notifications.",
-    "expected_sources": [
-      "packages/email/templates",
-      "packages/email/render.tsx",
-      "packages/email/mailer.ts",
-      "packages/app-tests"
-    ],
-    "must_include_any": [
-      "email",
-      "templates",
-      "render",
-      "mailer",
-      "tests"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-followup-api-v2-code-path",
-    "category": "conversation",
-    "turns": [
-      {
-        "role": "user",
-        "content": "How does Documenso expose document operations in the current public API?"
-      },
-      {
-        "role": "assistant",
-        "content": "The current public API is API V2, implemented through the tRPC server package and document router, with OpenAPI support."
-      }
-    ],
-    "question": "show me the code path for that",
-    "ground_truth": "The follow-up should stay anchored to packages/trpc/server/document-router and the apps/remix server routing that mounts API V2. It should avoid drifting to README-only setup material.",
-    "expected_sources": [
-      "packages/trpc/server/document-router",
-      "packages/trpc/server",
-      "apps/remix/server/router.ts",
-      "ARCHITECTURE.md"
-    ],
-    "must_include_any": [
-      "document-router",
-      "packages/trpc/server",
-      "API V2",
-      "router"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-followup-email-flow",
-    "category": "conversation",
-    "turns": [
-      {
-        "role": "user",
-        "content": "What happens after a document is sent for signing?"
-      },
-      {
-        "role": "assistant",
-        "content": "The send flow updates document state, recipient state, jobs, and lifecycle emails."
-      }
-    ],
-    "question": "where does the email part live?",
-    "ground_truth": "The follow-up should retrieve packages/email templates, mailer/render code, and any server-only email or job code that triggers those templates.",
-    "expected_sources": [
-      "packages/email/templates",
-      "packages/email/mailer.ts",
-      "packages/email/render.tsx",
-      "packages/lib/server-only/email",
-      "packages/lib/jobs"
-    ],
-    "must_include_any": [
-      "packages/email",
-      "templates",
-      "mailer",
-      "jobs"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-followup-self-hosting-config",
-    "category": "conversation",
-    "turns": [
-      {
-        "role": "user",
-        "content": "How do I run Documenso myself?"
-      },
-      {
-        "role": "assistant",
-        "content": "Self-hosting uses the README flow: copy .env.example, set required URLs, database, SMTP, and secrets, run migrations, then start apps/remix."
-      }
-    ],
-    "question": "which config files should I inspect?",
-    "ground_truth": "The follow-up should point to .env.example, README self-hosting instructions, package.json or turbo config for scripts, and apps/remix for where the app starts.",
-    "expected_sources": [
-      ".env.example",
-      "README.md",
-      "package.json",
-      "turbo.json",
-      "apps/remix"
-    ],
-    "must_include_any": [
-      ".env.example",
-      "README",
-      "package.json",
-      "apps/remix"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-followup-signing-bridge",
-    "category": "conversation",
-    "turns": [
-      {
-        "role": "user",
-        "content": "How does Documenso seal or sign completed documents?"
-      },
-      {
-        "role": "assistant",
-        "content": "PDF completion crosses server-only PDF/document logic and the @documenso/signing package, which supports local and Google-backed signing transports."
-      }
-    ],
-    "question": "show me where signing plugs in",
-    "ground_truth": "The follow-up should retrieve packages/signing, especially the signPdf entry point, transports, and helpers, plus the seal-document job where completed PDFs are decorated and passed into signing, and signing-related env configuration.",
-    "expected_sources": [
-      "packages/signing/index.ts",
-      "packages/signing/transports",
-      "packages/signing/helpers",
-      "packages/lib/jobs/definitions/internal/seal-document.handler.ts",
-      ".env.example"
-    ],
-    "must_include_any": [
-      "packages/signing",
-      "transports",
-      "PDF",
-      "signing"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-codegen-email-template-checklist",
-    "category": "code-generation",
-    "question": "Write a short implementation checklist for adding a new document lifecycle email template in Documenso",
-    "ground_truth": "A good checklist should mention adding a React Email template under packages/email/templates, wiring rendering or mailer usage if needed, passing required variables from server-side document or job logic, and covering the change with focused tests or previews.",
-    "expected_sources": [
-      "packages/email/templates",
-      "packages/email/mailer.ts",
-      "packages/email/render.tsx",
-      "packages/lib/jobs",
-      "packages/lib/server-only/document"
-    ],
-    "must_include_any": [
-      "packages/email/templates",
-      "React Email",
-      "mailer",
-      "jobs",
-      "tests"
-    ],
-    "min_keyword_matches": 3
-  },
-  {
-    "id": "documenso-codegen-api-route-checklist",
-    "category": "code-generation",
-    "question": "Write a short implementation checklist for adding a new API V2 document operation in Documenso",
-    "ground_truth": "A good checklist should point to packages/trpc/server and the document-router, define request and response types or schemas, delegate business logic to packages/lib/server-only/document, update OpenAPI exposure if applicable, persist with Prisma when needed, and add tests.",
-    "expected_sources": [
-      "packages/trpc/server/document-router",
-      "packages/trpc/server",
-      "packages/lib/server-only/document",
-      "packages/prisma",
-      "ARCHITECTURE.md"
-    ],
-    "must_include_any": [
-      "document-router",
-      "tRPC",
-      "OpenAPI",
-      "server-only/document",
-      "Prisma"
-    ],
-    "min_keyword_matches": 3
-  }
-]
+{
+  "repositories": [
+    {
+      "id": "documenso",
+      "name": "Documenso",
+      "github_url": "https://github.com/documenso/documenso.git",
+      "cases": [
+        {
+          "id": "documenso-purpose",
+          "category": "architecture",
+          "question": "What is Documenso and what product problem is it trying to solve?",
+          "ground_truth": "Documenso is an open-source document signing platform and DocuSign alternative. It lets users create, send, and sign documents electronically while emphasizing self-hosting, trust, and the ability to inspect how the signing system works under the hood.",
+          "expected_sources": ["README.md", "MANIFEST.md", "ARCHITECTURE.md"],
+          "must_include_any": ["document signing", "self-host", "DocuSign", "open trust"]
+        },
+        {
+          "id": "documenso-api-v2-document-router",
+          "category": "api",
+          "question": "Where is the current document API implemented and how is it exposed?",
+          "ground_truth": "The current API V2 is implemented under packages/trpc/server, with document operations organized under a document router. It is exposed through /api/v2 and /api/v2-beta with tRPC and OpenAPI support, and it accepts API-token or session-cookie authentication depending on the route.",
+          "expected_sources": ["ARCHITECTURE.md", "packages/trpc/server", "packages/trpc/server/document-router", "apps/remix/server"],
+          "must_include_any": ["packages/trpc/server", "document-router", "API V2", "OpenAPI", "tRPC"]
+        },
+        {
+          "id": "documenso-signing-package",
+          "category": "specific-function",
+          "question": "What does the signing package do in Documenso?",
+          "ground_truth": "The @documenso/signing package owns PDF signing behavior. Its signPdf entry point selects a signing transport, applies timestamp authority settings when configured, and supports local P12 signing and Google Cloud KMS/HSM-backed signing through transport implementations.",
+          "expected_sources": ["ARCHITECTURE.md", "packages/signing/index.ts", "packages/signing/helpers", "packages/signing/transports", ".env.example"],
+          "must_include_any": ["PDF signing", "transports", "local", "Google", "KMS"]
+        },
+        {
+          "id": "documenso-document-send-flow",
+          "category": "cross-file",
+          "question": "How does a document send operation flow across the Documenso codebase?",
+          "ground_truth": "A document send operation starts at an API or UI route, goes through the API layer such as packages/trpc/server/document-router, delegates core behavior to packages/lib/server-only/document and related recipient or field logic, persists through packages/prisma, and can trigger emails or jobs through packages/email and packages/lib/jobs.",
+          "expected_sources": ["packages/trpc/server/document-router", "packages/lib/server-only/document", "packages/lib/server-only/recipient", "packages/prisma", "packages/email", "packages/lib/jobs"],
+          "must_include_any": ["document-router", "server-only/document", "prisma", "email", "jobs"]
+        },
+        {
+          "id": "documenso-required-env",
+          "category": "config-setup",
+          "question": "Which environment variables are central to running a self-hosted Documenso instance?",
+          "ground_truth": "The setup expects values such as NEXTAUTH_SECRET, NEXT_PRIVATE_ENCRYPTION_KEY, NEXT_PRIVATE_ENCRYPTION_SECONDARY_KEY, NEXT_PUBLIC_WEBAPP_URL, NEXT_PRIVATE_DATABASE_URL, NEXT_PRIVATE_DIRECT_DATABASE_URL, NEXT_PRIVATE_SMTP_FROM_NAME, and NEXT_PRIVATE_SMTP_FROM_ADDRESS. The env example and self-hosting docs also cover internal URLs, signing transport, storage transport, and optional OAuth and webhook configuration.",
+          "expected_sources": ["README.md", ".env.example", "apps/docs/content/docs/self-hosting/configuration/environment.mdx"],
+          "must_include_any": ["NEXTAUTH_SECRET", "NEXT_PUBLIC_WEBAPP_URL", "DATABASE_URL", "SMTP", "encryption"]
+        },
+        {
+          "id": "documenso-playwright-tests",
+          "category": "tests",
+          "question": "Where does Documenso keep end-to-end app tests?",
+          "ground_truth": "The architecture identifies @documenso/app-tests as the E2E test package, and the packages/app-tests directory is intended for Playwright coverage of app behavior.",
+          "expected_sources": ["ARCHITECTURE.md", "packages/app-tests", "packages/app-tests/package.json"],
+          "must_include_any": ["app-tests", "E2E", "Playwright"]
+        },
+        {
+          "id": "documenso-webhook-security-errors",
+          "category": "error-handling",
+          "question": "Where should you look for webhook security or SSRF-related safeguards?",
+          "ground_truth": "Webhook safeguards belong in packages/lib/server-only/webhooks, with related configuration documented in .env.example such as NEXT_PRIVATE_WEBHOOK_SSRF_BYPASS_HOSTS. These sources explain where outbound webhook behavior and security exceptions are controlled.",
+          "expected_sources": [".env.example", "packages/lib/server-only/webhooks"],
+          "must_include_any": ["webhook", "SSRF", "bypass", "hosts"]
+        },
+        {
+          "id": "documenso-followup-signing-bridge",
+          "category": "conversation",
+          "turns": [
+            {"role": "user", "content": "How does Documenso seal or sign completed documents?"},
+            {"role": "assistant", "content": "PDF completion crosses server-only PDF/document logic and the @documenso/signing package, which supports local and Google-backed signing transports."}
+          ],
+          "question": "show me where signing plugs in",
+          "ground_truth": "The follow-up should retrieve packages/signing, especially the signPdf entry point, transports, and helpers, plus the seal-document job where completed PDFs are decorated and passed into signing, and signing-related env configuration.",
+          "expected_sources": ["packages/signing/index.ts", "packages/signing/transports", "packages/signing/helpers", "packages/lib/jobs/definitions/internal/seal-document.handler.ts", ".env.example"],
+          "must_include_any": ["packages/signing", "transports", "PDF", "signing"]
+        }
+      ]
+    },
+    {
+      "id": "sqlite",
+      "name": "SQLite",
+      "github_url": "https://github.com/sqlite/sqlite.git",
+      "cases": [
+        {
+          "id": "sqlite-purpose",
+          "category": "architecture",
+          "question": "What is SQLite and how is the project organized at a high level?",
+          "ground_truth": "SQLite is a self-contained, serverless, zero-configuration embedded SQL database engine. Its core sources live under src/, tests are TCL scripts, the build produces both a library and a single-file amalgamation, and the project's own AGENTS.md documents the execution pipeline for contributors.",
+          "expected_sources": ["README.md", "src", "AGENTS.md"],
+          "must_include_any": ["embedded", "serverless", "amalgamation", "SQL database engine"]
+        },
+        {
+          "id": "sqlite-query-pipeline",
+          "category": "cross-file",
+          "question": "How does a SQL statement flow through SQLite's execution pipeline?",
+          "ground_truth": "SQL text is tokenized in tokenize.c, parsed by the Lemon-generated parser from parse.y, turned into VDBE bytecode by the code generator files (build.c, select.c, insert.c, update.c, delete.c, expr.c) after being optimized by the where*.c query optimizer, then executed by the VDBE in vdbe.c against the B-tree layer in btree.c, the pager in pager.c, the WAL in wal.c, and finally the OS-level VFS such as os_unix.c.",
+          "expected_sources": ["src/tokenize.c", "src/parse.y", "src/vdbe.c", "src/btree.c", "src/pager.c", "src/wal.c", "AGENTS.md"],
+          "must_include_any": ["tokenizer", "parser", "VDBE", "B-Tree", "pager", "WAL"]
+        },
+        {
+          "id": "sqlite-btree-role",
+          "category": "specific-function",
+          "question": "What role does btree.c play in SQLite?",
+          "ground_truth": "btree.c implements the B-tree storage engine SQLite uses to organize table and index data on disk. Its public interface is declared in btree.h, while btreeInt.h defines the data structures used only internally by the module.",
+          "expected_sources": ["src/btree.c", "src/btree.h", "src/btreeInt.h"],
+          "must_include_any": ["B-Tree", "storage engine", "btree.h"]
+        },
+        {
+          "id": "sqlite-vdbe-opcodes",
+          "category": "specific-function",
+          "question": "How are VDBE opcodes generated and where do they come from?",
+          "ground_truth": "VDBE opcode numbers and names are extracted automatically by scanning src/vdbe.c with the mkopcodeh.tcl script, which generates opcodes.h; a second script, mkopcodec.tcl, then generates opcodes.c, which provides the reverse opcode-to-name mapping used for EXPLAIN output.",
+          "expected_sources": ["src/vdbe.c", "mkopcodeh.tcl", "mkopcodec.tcl"],
+          "must_include_any": ["opcodes.h", "mkopcodeh.tcl", "VDBE", "EXPLAIN"]
+        },
+        {
+          "id": "sqlite-parser-generation",
+          "category": "config-setup",
+          "question": "How is the SQL grammar parser built for SQLite?",
+          "ground_truth": "The grammar is defined in src/parse.y and compiled into parse.c by the Lemon LALR(1) parser generator in tool/lemon.c, which uses tool/lempar.c as a template and also emits the parse.h header as a side effect.",
+          "expected_sources": ["src/parse.y", "tool/lemon.c", "tool/lempar.c"],
+          "must_include_any": ["parse.y", "Lemon", "LALR", "parse.c"]
+        },
+        {
+          "id": "sqlite-testing",
+          "category": "tests",
+          "question": "How does SQLite run its test suite and what kind of tests does it use?",
+          "ground_truth": "SQLite's tests are TCL scripts executed through the testfixture binary, which is built with make testfixture. AGENTS.md instructs contributors to run at least make devtest after any change under src/, and make sqlite3.c builds the amalgamation used for distribution.",
+          "expected_sources": ["test", "AGENTS.md", "Makefile.in"],
+          "must_include_any": ["testfixture", "TCL", "devtest", "test suite"]
+        },
+        {
+          "id": "sqlite-build-amalgamation",
+          "category": "config-setup",
+          "question": "What is the SQLite amalgamation and how is it produced?",
+          "ground_truth": "The amalgamation is the single-file distribution form of SQLite, sqlite3.c, assembled from the individual sources under src/ during the build. The public C API is declared in the src/sqlite.h.in template, which is expanded into the sqlite3.h header shipped with the amalgamation.",
+          "expected_sources": ["src/sqlite.h.in", "Makefile.in", "AGENTS.md"],
+          "must_include_any": ["amalgamation", "sqlite3.c", "sqlite.h.in"]
+        },
+        {
+          "id": "sqlite-followup-wal",
+          "category": "conversation",
+          "turns": [
+            {"role": "user", "content": "How does SQLite guarantee that transactions survive a crash?"},
+            {"role": "assistant", "content": "Durability is enforced through the pager and, depending on journal mode, the write-ahead log."}
+          ],
+          "question": "which files implement that WAL behavior?",
+          "ground_truth": "WAL-mode durability is implemented mainly in src/wal.c, working together with src/pager.c, which coordinates pager and journal behavior, and the OS-level VFS layer such as src/os_unix.c, which performs the actual fsync/durability calls.",
+          "expected_sources": ["src/wal.c", "src/pager.c", "src/os_unix.c"],
+          "must_include_any": ["wal.c", "pager.c", "fsync", "VFS"]
+        }
+      ]
+    },
+    {
+      "id": "fastapi",
+      "name": "FastAPI",
+      "github_url": "https://github.com/fastapi/fastapi.git",
+      "cases": [
+        {
+          "id": "fastapi-purpose",
+          "category": "architecture",
+          "question": "What is FastAPI and what is it built on top of?",
+          "ground_truth": "FastAPI is a Python web framework for building APIs. It is built on top of Starlette for the web-facing parts and Pydantic for data validation and serialization, and it automatically generates an OpenAPI schema along with interactive Swagger UI and ReDoc documentation.",
+          "expected_sources": ["README.md", "fastapi/applications.py", "pyproject.toml"],
+          "must_include_any": ["Starlette", "Pydantic", "OpenAPI"]
+        },
+        {
+          "id": "fastapi-app-class",
+          "category": "specific-function",
+          "question": "What does the FastAPI application class do and where is it defined?",
+          "ground_truth": "The central FastAPI class is defined in fastapi/applications.py. It ties together routing, dependency injection, middleware, exception handling, and OpenAPI schema generation for the whole application.",
+          "expected_sources": ["fastapi/applications.py"],
+          "must_include_any": ["applications.py", "routing", "OpenAPI"]
+        },
+        {
+          "id": "fastapi-routing",
+          "category": "implementation",
+          "question": "Where is path operation routing implemented in FastAPI?",
+          "ground_truth": "Routing is implemented in fastapi/routing.py, which defines APIRoute and APIRouter. It handles path matching, resolves dependencies for each incoming request, and serializes the response for every registered endpoint.",
+          "expected_sources": ["fastapi/routing.py"],
+          "must_include_any": ["APIRoute", "APIRouter", "routing.py"]
+        },
+        {
+          "id": "fastapi-dependency-injection",
+          "category": "cross-file",
+          "question": "How does FastAPI resolve dependencies declared with Depends()?",
+          "ground_truth": "Depends() and related parameter markers are defined in fastapi/params.py and fastapi/param_functions.py. The actual dependency tree resolution for each request happens in fastapi/dependencies/utils.py, which fastapi/routing.py calls while handling a request.",
+          "expected_sources": ["fastapi/dependencies/utils.py", "fastapi/params.py", "fastapi/param_functions.py", "fastapi/routing.py"],
+          "must_include_any": ["Depends", "dependencies/utils.py", "dependency"]
+        },
+        {
+          "id": "fastapi-openapi-generation",
+          "category": "api",
+          "question": "How does FastAPI generate the OpenAPI schema and interactive docs?",
+          "ground_truth": "The OpenAPI JSON schema is generated in fastapi/openapi/utils.py from the app's routes and Pydantic models. The Swagger UI and ReDoc HTML pages are served through helper functions such as get_swagger_ui_html and get_redoc_html in fastapi/openapi/docs.py.",
+          "expected_sources": ["fastapi/openapi/utils.py", "fastapi/openapi/docs.py"],
+          "must_include_any": ["OpenAPI", "Swagger", "ReDoc", "openapi/utils.py"]
+        },
+        {
+          "id": "fastapi-error-handling",
+          "category": "error-handling",
+          "question": "How does FastAPI turn validation failures and raised exceptions into HTTP responses?",
+          "ground_truth": "FastAPI defines HTTPException and RequestValidationError in fastapi/exceptions.py. The default handlers that convert those exceptions into JSON error responses are registered in fastapi/exception_handlers.py.",
+          "expected_sources": ["fastapi/exceptions.py", "fastapi/exception_handlers.py"],
+          "must_include_any": ["HTTPException", "RequestValidationError", "exception_handlers.py"]
+        },
+        {
+          "id": "fastapi-security",
+          "category": "specific-function",
+          "question": "How does FastAPI support authentication schemes like OAuth2 and HTTP Bearer tokens?",
+          "ground_truth": "Authentication helpers live under fastapi/security, which provides classes such as OAuth2PasswordBearer and HTTPBearer. They integrate with the dependency injection system and are automatically reflected in the generated OpenAPI security schema.",
+          "expected_sources": ["fastapi/security", "fastapi/openapi/utils.py"],
+          "must_include_any": ["OAuth2", "HTTPBearer", "security", "dependency"]
+        },
+        {
+          "id": "fastapi-followup-encoder",
+          "category": "conversation",
+          "turns": [
+            {"role": "user", "content": "How does FastAPI encode response data before it goes back to the client?"},
+            {"role": "assistant", "content": "Response bodies are converted with jsonable_encoder before being serialized, factoring in the declared response_model."}
+          ],
+          "question": "where is that encoder implemented and how is it tested?",
+          "ground_truth": "jsonable_encoder is implemented in fastapi/encoders.py, converting Pydantic models and other Python objects into JSON-compatible structures. Its behavior is covered by the pytest suite under the tests directory.",
+          "expected_sources": ["fastapi/encoders.py", "tests"],
+          "must_include_any": ["jsonable_encoder", "encoders.py", "tests"]
+        }
+      ]
+    }
+  ]
+}

src/rag_system.py

@@ -497,7 +497,7 @@ Rules:
 7. Keep the answer complete. Do not stop mid-sentence.
 8. Use short sections or bullets only when they genuinely help readability.
 9. Do not leave unfinished headings, dangling bullets, or trailing markdown markers like #, ##, or ###.
-10. Do not include inline citation markers like [Source 1] in the prose. The UI already shows sources separately.
+10. Back up factual claims with inline citations. After a sentence or clause that relies on a specific source, add its number in brackets, e.g. [1] or [2][3] if multiple sources support it. Use only the source numbers given above (Source 1, Source 2, ...) and never invent a number.
 11. If you cannot answer the question using the provided context, say: "I cannot find sufficient evidence in the codebase to answer this question."
 12. Prefer the most canonical source files for API and implementation questions, such as package exports, core modules, and session/query code, over tutorial prose when they disagree in specificity.
 13. Keep the answer tight. Lead with the direct answer, then add only the most important supporting detail.
@@ -560,15 +560,9 @@ Do not leave the answer unfinished.
                 )
 
         answer_text = self._finalize_answer(answer_text)
+        answer_text, citations = self._attach_citations(answer_text, sources)
         confidence = self._estimate_confidence(sources)
         summary = " ".join(answer_text.split())[:160] if answer_text else ""
-        citations = [
-            {
-                "source": index,
-                "reason": f"Relevant context from {source['file_path']}",
-            }
-            for index, source in enumerate(sources[: min(len(sources), 4)], start=1)
-        ]
 
         return {
             "answer": answer_text,
@@ -699,7 +693,9 @@ Do not leave the answer unfinished.
     def _normalize_markdown_answer(raw_text: str) -> str:
         cleaned = (raw_text or "").strip()
         cleaned = re.sub(r"^```(?:markdown|md)?\s*|\s*```$", "", cleaned, flags=re.IGNORECASE)
-        cleaned = re.sub(r"\s*\[(?:Source\s+\d+(?:\s*,\s*Source\s+\d+)*)\]", "", cleaned, flags=re.IGNORECASE)
+        cleaned = re.sub(
+            r"\[Source\s+(\d+)\]", r"[\1]", cleaned, flags=re.IGNORECASE
+        )
         cleaned = re.sub(
             r"^(?:based on the provided context[,:\s-]*|from the provided context[,:\s-]*)",
             "",
@@ -768,6 +764,44 @@ Do not leave the answer unfinished.
             return True
         return False
 
+    @staticmethod
+    def _attach_citations(answer_text: str, sources: List[dict]) -> tuple[str, List[dict]]:
+        max_source = len(sources)
+        if max_source == 0 or not answer_text:
+            return answer_text, []
+
+        cited_numbers = set()
+
+        def _keep_or_drop(match: "re.Match") -> str:
+            number = int(match.group(1))
+            if 1 <= number <= max_source:
+                cited_numbers.add(number)
+                return match.group(0)
+            # Drop citation markers that don't correspond to a real source.
+            return ""
+
+        cleaned_text = re.sub(r"\[(\d+)\]", _keep_or_drop, answer_text)
+        cleaned_text = re.sub(r"[ \t]+([.,;:!?])", r"\1", cleaned_text).strip()
+
+        # If the model didn't cite anything inline, fall back to listing every
+        # retrieved source so the response is still grounded in a traceable way.
+        numbers_to_cite = cited_numbers if cited_numbers else set(range(1, max_source + 1))
+
+        citations = []
+        for index in sorted(numbers_to_cite):
+            source = sources[index - 1]
+            citations.append(
+                {
+                    "source": index,
+                    "file_path": source["file_path"],
+                    "symbol_name": source.get("symbol_name"),
+                    "line_start": source.get("line_start"),
+                    "line_end": source.get("line_end"),
+                    "location": f"{source['file_path']}:{source.get('line_start')}-{source.get('line_end')}",
+                }
+            )
+        return cleaned_text, citations
+
     @staticmethod
     def _estimate_confidence(sources: List[dict]) -> str:
         if not sources:
@@ -1693,4 +1727,4 @@ Do not leave the answer unfinished.
             "content": chunk["content"],
             "searchable_text": chunk["searchable_text"],
             "metadata_json": chunk.get("metadata_json") or {},
-        }
+        }