""" Evaluation test dataset: 10 real prompts + 10 edge cases Used to measure success rate, retries, and reliability metrics. """ TEST_PROMPTS = { "real_products": [ { "id": "real_1", "name": "CRM System", "prompt": "Build a CRM with login, contacts, dashboard, role-based access, and premium plan with payments. Admins can see analytics." }, { "id": "real_2", "name": "E-commerce Platform", "prompt": "Create an e-commerce platform with product listing, shopping cart, checkout, payment processing, order tracking, and admin inventory management." }, { "id": "real_3", "name": "Project Management Tool", "prompt": "Build a project management tool with tasks, teams, real-time collaboration, comments, file attachments, and Gantt chart visualization." }, { "id": "real_4", "name": "Social Network", "prompt": "Create a social network with user profiles, friend connections, feed, messaging, notifications, and content sharing." }, { "id": "real_5", "name": "Booking System", "prompt": "Build a booking platform for services with service listings, availability calendar, booking management, payments, and customer reviews." }, { "id": "real_6", "name": "Learning Platform", "prompt": "Create an online learning platform with courses, lessons, quizzes, student progress tracking, certificates, and instructor dashboards." }, { "id": "real_7", "name": "Chat Application", "prompt": "Build a messaging app with direct messages, group chats, file sharing, user presence, and message history." }, { "id": "real_8", "name": "Analytics Dashboard", "prompt": "Create a business analytics dashboard with data visualization, custom reports, filters, export functionality, and real-time metrics." }, { "id": "real_9", "name": "Healthcare Portal", "prompt": "Build a healthcare patient portal with appointment scheduling, medical records, doctor messaging, prescription management, and billing." }, { "id": "real_10", "name": "HR Management System", "prompt": "Create an HR management system with employee records, leave management, attendance tracking, payroll integration, and performance reviews." } ], "edge_cases": [ { "id": "edge_1", "category": "vague", "name": "Super Vague", "prompt": "Build something useful" }, { "id": "edge_2", "category": "vague", "name": "Unclear Requirements", "prompt": "App with features and things and stuff" }, { "id": "edge_3", "category": "conflicting", "name": "Conflicting Requirements", "prompt": "Build a lightweight app with heavy real-time analytics and complex machine learning. Keep it simple but feature-rich with enterprise-grade security and minimal code." }, { "id": "edge_4", "category": "conflicting", "name": "Contradictory Roles", "prompt": "Create a system where guests can manage admin settings but admins have read-only access." }, { "id": "edge_5", "category": "incomplete", "name": "Missing Core Details", "prompt": "Build an app" }, { "id": "edge_6", "category": "incomplete", "name": "No Business Rules", "prompt": "System with users, products, and orders" }, { "id": "edge_7", "category": "ambiguous", "name": "Ambiguous Scope", "prompt": "Build social platform with all standard features" }, { "id": "edge_8", "category": "ambiguous", "name": "Undefined Entities", "prompt": "Create a marketplace for things between people with verification and payments" }, { "id": "edge_9", "category": "complex", "name": "Over-specified", "prompt": "Build a system with microservices architecture, GraphQL API, React frontend, PostgreSQL database, Redis caching, Kafka message queue, Docker containerization, Kubernetes orchestration, CI/CD pipeline, and monitoring with Prometheus and Grafana. Also include blockchain integration for immutable audit logs." }, { "id": "edge_10", "category": "technical", "name": "Technical Jargon", "prompt": "Implement CRUD operations for normalization with ACID compliance, horizontal scalability, and auto-sharding across multiple zones" } ] } def get_test_dataset(): """Get all test prompts.""" return TEST_PROMPTS def get_real_prompts(): """Get just the real product prompts.""" return TEST_PROMPTS["real_products"] def get_edge_cases(): """Get just the edge case prompts.""" return TEST_PROMPTS["edge_cases"] def get_prompt_by_id(prompt_id: str): """Get a specific test prompt.""" for category in TEST_PROMPTS.values(): for item in category: if item["id"] == prompt_id: return item return None