evals/sample_eval_set.json

{
  "repositories": [
    {
      "id": "documenso",
      "name": "Documenso",
      "github_url": "https://github.com/documenso/documenso.git",
      "cases": [
        {
          "id": "documenso-purpose",
          "category": "architecture",
          "question": "What is Documenso and what product problem is it trying to solve?",
          "ground_truth": "Documenso is an open-source document signing platform and DocuSign alternative. It lets users create, send, and sign documents electronically while emphasizing self-hosting, trust, and the ability to inspect how the signing system works under the hood.",
          "expected_sources": ["README.md", "MANIFEST.md", "ARCHITECTURE.md"],
          "must_include_any": ["document signing", "self-host", "DocuSign", "open trust"]
        },
        {
          "id": "documenso-api-v2-document-router",
          "category": "api",
          "question": "Where is the current document API implemented and how is it exposed?",
          "ground_truth": "The current API V2 is implemented under packages/trpc/server, with document operations organized under a document router. It is exposed through API V2 routes with tRPC and OpenAPI support.",
          "expected_sources": ["ARCHITECTURE.md", "packages/trpc/server", "packages/trpc/server/document-router", "apps/remix/server"],
          "must_include_any": ["packages/trpc/server", "document-router", "API V2", "OpenAPI", "tRPC"]
        },
        {
          "id": "documenso-signing-package",
          "category": "specific-function",
          "question": "What does the @documenso/signing package do, and where is it implemented?",
          "ground_truth": "The @documenso/signing package owns PDF signing behavior. Its signPdf entry point selects a signing transport, applies timestamp authority settings when configured, and supports local P12 signing and Google Cloud KMS/HSM-backed signing through transport implementations.",
          "expected_sources": ["ARCHITECTURE.md", "packages/signing/index.ts", "packages/signing/helpers", "packages/signing/transports", ".env.example"],
          "must_include_any": ["PDF signing", "packages/signing", "transports", "local", "Google", "KMS"]
        },
        {
          "id": "documenso-document-send-flow",
          "category": "cross-file",
          "question": "How does a document send operation flow across the Documenso codebase?",
          "ground_truth": "A document send operation starts at an API or UI route, delegates core behavior to server-only document logic and related recipient or field logic, persists state through the data layer, and can trigger emails or jobs.",
          "expected_sources": ["packages/trpc/server/document-router", "packages/lib/server-only/document", "packages/lib/server-only/recipient", "packages/prisma", "packages/email", "packages/lib/jobs"],
          "must_include_any": ["document-router", "server-only/document", "recipient", "prisma", "email", "jobs"]
        },
        {
          "id": "documenso-required-env",
          "category": "config-setup",
          "question": "Which environment variables are central to running a self-hosted Documenso instance?",
          "ground_truth": "Self-hosting requires configuration for authentication, encryption, the public web URL, database connectivity, and SMTP/email delivery. The environment example and self-hosting documentation are the primary sources.",
          "expected_sources": ["README.md", ".env.example", "apps/docs/content/docs/self-hosting/configuration/environment.mdx"],
          "must_include_any": ["NEXTAUTH_SECRET", "NEXT_PUBLIC_WEBAPP_URL", "DATABASE_URL", "SMTP", "encryption"]
        },
        {
          "id": "documenso-playwright-tests",
          "category": "tests",
          "question": "Where does Documenso keep end-to-end app tests?",
          "ground_truth": "Documenso keeps end-to-end application tests in the @documenso/app-tests package under packages/app-tests, using Playwright for app-level coverage.",
          "expected_sources": ["ARCHITECTURE.md", "packages/app-tests", "packages/app-tests/package.json"],
          "must_include_any": ["app-tests", "E2E", "Playwright"]
        },
        {
          "id": "documenso-webhook-security-errors",
          "category": "error-handling",
          "question": "Where should you look for webhook security or SSRF-related safeguards?",
          "ground_truth": "Webhook security safeguards are implemented in packages/lib/server-only/webhooks, with related configuration such as SSRF bypass hosts documented in environment configuration.",
          "expected_sources": [".env.example", "packages/lib/server-only/webhooks"],
          "must_include_any": ["webhook", "SSRF", "bypass", "hosts"]
        },
        {
          "id": "documenso-followup-signing-bridge",
          "category": "conversation",
          "turns": [
            {"role": "user", "content": "How does Documenso seal or sign completed documents?"},
            {"role": "assistant", "content": "Completed PDFs cross server-only document/job logic and the @documenso/signing package, whose signPdf entry point supports multiple signing transports."}
          ],
          "question": "show me where that signing package plugs in",
          "ground_truth": "The follow-up should connect the @documenso/signing package, especially its entry point, transports, and helpers, with the seal-document job that passes completed PDFs into signing and with relevant signing configuration.",
          "expected_sources": ["packages/signing/index.ts", "packages/signing/transports", "packages/signing/helpers", "packages/lib/jobs/definitions/internal/seal-document.handler.ts", ".env.example"],
          "must_include_any": ["packages/signing", "transports", "seal-document", "PDF", "signing"]
        }
      ]
    },
    {
      "id": "fastapi",
      "name": "FastAPI",
      "github_url": "https://github.com/fastapi/fastapi.git",
      "cases": [
        {
          "id": "fastapi-purpose",
          "category": "architecture",
          "question": "What is FastAPI and what is it built on top of?",
          "ground_truth": "FastAPI is a Python web framework for building APIs. It builds on Starlette for web functionality and Pydantic for data validation and serialization, and it generates OpenAPI-based documentation.",
          "expected_sources": ["README.md", "fastapi/applications.py", "pyproject.toml"],
          "must_include_any": ["Starlette", "Pydantic", "OpenAPI"]
        },
        {
          "id": "fastapi-app-class",
          "category": "specific-function",
          "question": "What does the FastAPI application class do and where is it defined?",
          "ground_truth": "The central FastAPI class is defined in fastapi/applications.py and coordinates application routing, middleware, exception handling, dependency-related behavior, and OpenAPI generation.",
          "expected_sources": ["fastapi/applications.py"],
          "must_include_any": ["applications.py", "routing", "OpenAPI"]
        },
        {
          "id": "fastapi-routing",
          "category": "implementation",
          "question": "Where is path operation routing implemented in FastAPI?",
          "ground_truth": "Path operation routing is implemented in fastapi/routing.py, including APIRoute and APIRouter and request handling that integrates dependency resolution and response serialization.",
          "expected_sources": ["fastapi/routing.py"],
          "must_include_any": ["APIRoute", "APIRouter", "routing.py"]
        },
        {
          "id": "fastapi-dependency-injection",
          "category": "cross-file",
          "question": "How does FastAPI resolve dependencies declared with Depends()?",
          "ground_truth": "Depends-related parameter declarations are defined in FastAPI's parameter modules, while dependency tree construction and request-time resolution live under fastapi/dependencies and are integrated into routing.",
          "expected_sources": ["fastapi/dependencies/utils.py", "fastapi/params.py", "fastapi/param_functions.py", "fastapi/routing.py"],
          "must_include_any": ["Depends", "dependencies/utils.py", "dependency", "routing.py"]
        },
        {
          "id": "fastapi-openapi-generation",
          "category": "api",
          "question": "How does FastAPI generate the OpenAPI schema and interactive docs?",
          "ground_truth": "FastAPI generates its OpenAPI schema from routes and models using its OpenAPI utilities, while Swagger UI and ReDoc HTML are produced by helpers in the OpenAPI docs module.",
          "expected_sources": ["fastapi/openapi/utils.py", "fastapi/openapi/docs.py"],
          "must_include_any": ["OpenAPI", "Swagger", "ReDoc", "openapi/utils.py"]
        },
        {
          "id": "fastapi-error-handling",
          "category": "error-handling",
          "question": "How does FastAPI turn validation failures and raised exceptions into HTTP responses?",
          "ground_truth": "FastAPI defines framework exceptions such as HTTPException and RequestValidationError and provides default exception handlers that convert them into HTTP responses.",
          "expected_sources": ["fastapi/exceptions.py", "fastapi/exception_handlers.py"],
          "must_include_any": ["HTTPException", "RequestValidationError", "exception_handlers.py"]
        },
        {
          "id": "fastapi-security",
          "category": "specific-function",
          "question": "How does FastAPI support authentication schemes like OAuth2 and HTTP Bearer tokens?",
          "ground_truth": "FastAPI provides authentication helpers under fastapi/security, including OAuth2 and HTTP bearer classes. They integrate with dependency injection and contribute security information to OpenAPI generation.",
          "expected_sources": ["fastapi/security", "fastapi/openapi/utils.py"],
          "must_include_any": ["OAuth2", "HTTPBearer", "security", "dependency"]
        },
        {
          "id": "fastapi-followup-encoder",
          "category": "conversation",
          "turns": [
            {"role": "user", "content": "How does FastAPI encode response data before it goes back to the client?"},
            {"role": "assistant", "content": "Response data can be converted with jsonable_encoder before serialization, taking declared response behavior into account."}
          ],
          "question": "where is jsonable_encoder implemented and where is that behavior tested?",
          "ground_truth": "jsonable_encoder is implemented in fastapi/encoders.py and converts supported Python and Pydantic values into JSON-compatible structures. Its behavior is exercised by the test suite.",
          "expected_sources": ["fastapi/encoders.py", "tests"],
          "must_include_any": ["jsonable_encoder", "encoders.py", "tests"]
        }
      ]
    },
    {
      "id": "django",
      "name": "Django",
      "github_url": "https://github.com/django/django.git",
      "cases": [
        {
          "id": "django-purpose",
          "category": "architecture",
          "question": "What is Django and how is the repository organized at a high level?",
          "ground_truth": "Django is a high-level Python web framework. Framework implementation code lives under django/, documentation under docs/, and the main test suite under tests/.",
          "expected_sources": ["README.rst", "django", "docs", "tests"],
          "must_include_any": ["web framework", "django", "docs", "tests"]
        },
        {
          "id": "django-url-routing",
          "category": "specific-function",
          "question": "Where is Django's URL routing machinery implemented and what does it do?",
          "ground_truth": "Django's URL routing machinery is implemented under django/urls. It provides URL configuration and resolution that maps incoming request paths to views.",
          "expected_sources": ["django/urls", "django/urls/resolvers.py"],
          "must_include_any": ["django/urls", "resolver", "URL", "view"]
        },
        {
          "id": "django-request-response",
          "category": "cross-file",
          "question": "How does an incoming HTTP request reach a Django view and become a response?",
          "ground_truth": "Django request handling crosses the core handler layer, middleware processing, URL resolution, the selected view, and HTTP response classes before the response is returned.",
          "expected_sources": ["django/core/handlers", "django/urls", "django/http"],
          "must_include_any": ["handler", "middleware", "URL", "view", "response"]
        },
        {
          "id": "django-queryset",
          "category": "specific-function",
          "question": "Where is Django's QuerySet implemented and what role does it play?",
          "ground_truth": "QuerySet is implemented in django/db/models/query.py. It is a central ORM abstraction representing database queries and collections of model instances and supports operations such as filtering and ordering.",
          "expected_sources": ["django/db/models/query.py"],
          "must_include_any": ["QuerySet", "query.py", "ORM", "database"]
        },
        {
          "id": "django-settings",
          "category": "config-setup",
          "question": "How does Django load and expose project settings?",
          "ground_truth": "Django's settings machinery lives under django/conf. The settings proxy loads configuration from the configured project settings module and exposes it to framework and application code.",
          "expected_sources": ["django/conf", "django/conf/__init__.py"],
          "must_include_any": ["settings", "django/conf", "SETTINGS_MODULE", "configuration"]
        },
        {
          "id": "django-tests",
          "category": "tests",
          "question": "Where is Django's main test suite and where are contributors told how to run it?",
          "ground_truth": "Django keeps its main test suite under tests/. The repository README points contributors to the unit-test instructions under docs/internals/contributing/writing-code/unit-tests.txt.",
          "expected_sources": ["tests", "README.rst", "docs/internals/contributing/writing-code/unit-tests.txt"],
          "must_include_any": ["tests", "unit-tests.txt", "test suite"]
        },
        {
          "id": "django-security-csrf",
          "category": "error-handling",
          "question": "Where is Django's CSRF protection implemented and what does it validate?",
          "ground_truth": "Django's CSRF protection is implemented in django/middleware/csrf.py. The middleware validates CSRF-related request state and tokens and rejects requests that fail its checks.",
          "expected_sources": ["django/middleware/csrf.py"],
          "must_include_any": ["CSRF", "middleware", "token", "csrf.py"]
        },
        {
          "id": "django-followup-template",
          "category": "conversation",
          "turns": [
            {"role": "user", "content": "How does Django render HTML templates?"},
            {"role": "assistant", "content": "Django's built-in template system loads templates through configured engines and renders them with a context."}
          ],
          "question": "where is that built-in template engine implemented?",
          "ground_truth": "Django's built-in template engine is implemented under django/template, including engine configuration, parsing, loading, and rendering machinery.",
          "expected_sources": ["django/template", "django/template/engine.py", "django/template/base.py"],
          "must_include_any": ["django/template", "Engine", "Template", "render"]
        }
      ]
    }
  ]
}