File size: 5,626 Bytes
d0fdbcd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | """
Evaluation test dataset: 10 real prompts + 10 edge cases
Used to measure success rate, retries, and reliability metrics.
"""
TEST_PROMPTS = {
"real_products": [
{
"id": "real_1",
"name": "CRM System",
"prompt": "Build a CRM with login, contacts, dashboard, role-based access, and premium plan with payments. Admins can see analytics."
},
{
"id": "real_2",
"name": "E-commerce Platform",
"prompt": "Create an e-commerce platform with product listing, shopping cart, checkout, payment processing, order tracking, and admin inventory management."
},
{
"id": "real_3",
"name": "Project Management Tool",
"prompt": "Build a project management tool with tasks, teams, real-time collaboration, comments, file attachments, and Gantt chart visualization."
},
{
"id": "real_4",
"name": "Social Network",
"prompt": "Create a social network with user profiles, friend connections, feed, messaging, notifications, and content sharing."
},
{
"id": "real_5",
"name": "Booking System",
"prompt": "Build a booking platform for services with service listings, availability calendar, booking management, payments, and customer reviews."
},
{
"id": "real_6",
"name": "Learning Platform",
"prompt": "Create an online learning platform with courses, lessons, quizzes, student progress tracking, certificates, and instructor dashboards."
},
{
"id": "real_7",
"name": "Chat Application",
"prompt": "Build a messaging app with direct messages, group chats, file sharing, user presence, and message history."
},
{
"id": "real_8",
"name": "Analytics Dashboard",
"prompt": "Create a business analytics dashboard with data visualization, custom reports, filters, export functionality, and real-time metrics."
},
{
"id": "real_9",
"name": "Healthcare Portal",
"prompt": "Build a healthcare patient portal with appointment scheduling, medical records, doctor messaging, prescription management, and billing."
},
{
"id": "real_10",
"name": "HR Management System",
"prompt": "Create an HR management system with employee records, leave management, attendance tracking, payroll integration, and performance reviews."
}
],
"edge_cases": [
{
"id": "edge_1",
"category": "vague",
"name": "Super Vague",
"prompt": "Build something useful"
},
{
"id": "edge_2",
"category": "vague",
"name": "Unclear Requirements",
"prompt": "App with features and things and stuff"
},
{
"id": "edge_3",
"category": "conflicting",
"name": "Conflicting Requirements",
"prompt": "Build a lightweight app with heavy real-time analytics and complex machine learning. Keep it simple but feature-rich with enterprise-grade security and minimal code."
},
{
"id": "edge_4",
"category": "conflicting",
"name": "Contradictory Roles",
"prompt": "Create a system where guests can manage admin settings but admins have read-only access."
},
{
"id": "edge_5",
"category": "incomplete",
"name": "Missing Core Details",
"prompt": "Build an app"
},
{
"id": "edge_6",
"category": "incomplete",
"name": "No Business Rules",
"prompt": "System with users, products, and orders"
},
{
"id": "edge_7",
"category": "ambiguous",
"name": "Ambiguous Scope",
"prompt": "Build social platform with all standard features"
},
{
"id": "edge_8",
"category": "ambiguous",
"name": "Undefined Entities",
"prompt": "Create a marketplace for things between people with verification and payments"
},
{
"id": "edge_9",
"category": "complex",
"name": "Over-specified",
"prompt": "Build a system with microservices architecture, GraphQL API, React frontend, PostgreSQL database, Redis caching, Kafka message queue, Docker containerization, Kubernetes orchestration, CI/CD pipeline, and monitoring with Prometheus and Grafana. Also include blockchain integration for immutable audit logs."
},
{
"id": "edge_10",
"category": "technical",
"name": "Technical Jargon",
"prompt": "Implement CRUD operations for normalization with ACID compliance, horizontal scalability, and auto-sharding across multiple zones"
}
]
}
def get_test_dataset():
"""Get all test prompts."""
return TEST_PROMPTS
def get_real_prompts():
"""Get just the real product prompts."""
return TEST_PROMPTS["real_products"]
def get_edge_cases():
"""Get just the edge case prompts."""
return TEST_PROMPTS["edge_cases"]
def get_prompt_by_id(prompt_id: str):
"""Get a specific test prompt."""
for category in TEST_PROMPTS.values():
for item in category:
if item["id"] == prompt_id:
return item
return None
|