evaluation/test_dataset.py

"""
Evaluation test dataset: 10 real prompts + 10 edge cases
Used to measure success rate, retries, and reliability metrics.
"""

TEST_PROMPTS = {
    "real_products": [
        {
            "id": "real_1",
            "name": "CRM System",
            "prompt": "Build a CRM with login, contacts, dashboard, role-based access, and premium plan with payments. Admins can see analytics."
        },
        {
            "id": "real_2",
            "name": "E-commerce Platform",
            "prompt": "Create an e-commerce platform with product listing, shopping cart, checkout, payment processing, order tracking, and admin inventory management."
        },
        {
            "id": "real_3",
            "name": "Project Management Tool",
            "prompt": "Build a project management tool with tasks, teams, real-time collaboration, comments, file attachments, and Gantt chart visualization."
        },
        {
            "id": "real_4",
            "name": "Social Network",
            "prompt": "Create a social network with user profiles, friend connections, feed, messaging, notifications, and content sharing."
        },
        {
            "id": "real_5",
            "name": "Booking System",
            "prompt": "Build a booking platform for services with service listings, availability calendar, booking management, payments, and customer reviews."
        },
        {
            "id": "real_6",
            "name": "Learning Platform",
            "prompt": "Create an online learning platform with courses, lessons, quizzes, student progress tracking, certificates, and instructor dashboards."
        },
        {
            "id": "real_7",
            "name": "Chat Application",
            "prompt": "Build a messaging app with direct messages, group chats, file sharing, user presence, and message history."
        },
        {
            "id": "real_8",
            "name": "Analytics Dashboard",
            "prompt": "Create a business analytics dashboard with data visualization, custom reports, filters, export functionality, and real-time metrics."
        },
        {
            "id": "real_9",
            "name": "Healthcare Portal",
            "prompt": "Build a healthcare patient portal with appointment scheduling, medical records, doctor messaging, prescription management, and billing."
        },
        {
            "id": "real_10",
            "name": "HR Management System",
            "prompt": "Create an HR management system with employee records, leave management, attendance tracking, payroll integration, and performance reviews."
        }
    ],
    "edge_cases": [
        {
            "id": "edge_1",
            "category": "vague",
            "name": "Super Vague",
            "prompt": "Build something useful"
        },
        {
            "id": "edge_2",
            "category": "vague",
            "name": "Unclear Requirements",
            "prompt": "App with features and things and stuff"
        },
        {
            "id": "edge_3",
            "category": "conflicting",
            "name": "Conflicting Requirements",
            "prompt": "Build a lightweight app with heavy real-time analytics and complex machine learning. Keep it simple but feature-rich with enterprise-grade security and minimal code."
        },
        {
            "id": "edge_4",
            "category": "conflicting",
            "name": "Contradictory Roles",
            "prompt": "Create a system where guests can manage admin settings but admins have read-only access."
        },
        {
            "id": "edge_5",
            "category": "incomplete",
            "name": "Missing Core Details",
            "prompt": "Build an app"
        },
        {
            "id": "edge_6",
            "category": "incomplete",
            "name": "No Business Rules",
            "prompt": "System with users, products, and orders"
        },
        {
            "id": "edge_7",
            "category": "ambiguous",
            "name": "Ambiguous Scope",
            "prompt": "Build social platform with all standard features"
        },
        {
            "id": "edge_8",
            "category": "ambiguous",
            "name": "Undefined Entities",
            "prompt": "Create a marketplace for things between people with verification and payments"
        },
        {
            "id": "edge_9",
            "category": "complex",
            "name": "Over-specified",
            "prompt": "Build a system with microservices architecture, GraphQL API, React frontend, PostgreSQL database, Redis caching, Kafka message queue, Docker containerization, Kubernetes orchestration, CI/CD pipeline, and monitoring with Prometheus and Grafana. Also include blockchain integration for immutable audit logs."
        },
        {
            "id": "edge_10",
            "category": "technical",
            "name": "Technical Jargon",
            "prompt": "Implement CRUD operations for normalization with ACID compliance, horizontal scalability, and auto-sharding across multiple zones"
        }
    ]
}


def get_test_dataset():
    """Get all test prompts."""
    return TEST_PROMPTS


def get_real_prompts():
    """Get just the real product prompts."""
    return TEST_PROMPTS["real_products"]


def get_edge_cases():
    """Get just the edge case prompts."""
    return TEST_PROMPTS["edge_cases"]


def get_prompt_by_id(prompt_id: str):
    """Get a specific test prompt."""
    for category in TEST_PROMPTS.values():
        for item in category:
            if item["id"] == prompt_id:
                return item
    return None