Kyrlos11's picture
Upload 842 files
22087d0 verified
Raw History Blame Contribute Delete
13.1 kB
"""
Admin API Endpoints for Test Harness
Provides secure endpoints for:
- Starting benchmark tests (core and stress modes)
- Retrieving test results
- Listing previous runs
- Cancelling active tests
"""
import asyncio
from typing import Optional, List
from fastapi import APIRouter, HTTPException, Depends, Query
from pydantic import BaseModel, Field
from core.utils.auth_utils import verify_admin_api_key
from core.utils.logger import logger
from .runner import TestHarnessRunner
from .metrics import MetricsCollector
router = APIRouter(prefix="/admin/test-harness", tags=["admin-test-harness"])
e2e_router = APIRouter(prefix="/admin/tests", tags=["admin-e2e-tests"])
# Pydantic models for request/response
class StartTestRequest(BaseModel):
"""Request model for starting a benchmark test"""
mode: str = Field(..., description="Test mode: 'core_test' or 'stress_test'")
prompt_ids: Optional[List[str]] = Field(None, description="List of prompt IDs to test (None = all)")
concurrency: int = Field(1, description="Number of concurrent requests", ge=1, le=100)
model: str = Field("kortix/basic", description="Model to use (core_test only)")
num_executions: int = Field(100, description="Number of executions (stress_test only)", ge=1, le=10000)
cleanup_threads: bool = Field(True, description="Delete test threads after completion")
metadata: Optional[dict] = Field(None, description="Additional metadata")
class StartTestResponse(BaseModel):
"""Response model for test start"""
run_id: str
status: str
message: str
class RunSummaryResponse(BaseModel):
"""Response model for run summary"""
run_id: str
status: str
run_type: str
model_name: str
concurrency_level: int
total_prompts: int
started_at: str
completed_at: Optional[str]
duration_ms: Optional[int]
metadata: dict
summary: dict
results: List[dict]
class RunListResponse(BaseModel):
"""Response model for listing runs"""
runs: List[dict]
total: int
# Global runner instance
_runner_instance: Optional[TestHarnessRunner] = None
_background_tasks = {}
def get_runner(cleanup_threads: bool = True) -> TestHarnessRunner:
"""Get or create global runner instance"""
global _runner_instance
if _runner_instance is None:
_runner_instance = TestHarnessRunner(cleanup_threads=cleanup_threads)
return _runner_instance
@router.post("/run", response_model=StartTestResponse, summary="Start Benchmark Test")
async def start_benchmark_test(
request: StartTestRequest,
_: bool = Depends(verify_admin_api_key)
):
"""
Start a new benchmark test run
Requires X-Admin-Api-Key header for authentication.
- **core_test**: Real LLM calls with full metrics tracking
- **stress_test**: Mocked LLM for high-concurrency validation
"""
runner = get_runner(cleanup_threads=request.cleanup_threads)
# Validate mode
if request.mode not in ['core_test', 'stress_test']:
raise HTTPException(
status_code=400,
detail=f"Invalid mode: {request.mode}. Must be 'core_test' or 'stress_test'"
)
# Validate prompt IDs if provided
if request.prompt_ids:
from .prompts import get_prompt
invalid_prompts = [pid for pid in request.prompt_ids if not get_prompt(pid)]
if invalid_prompts:
raise HTTPException(
status_code=400,
detail=f"Invalid prompt IDs: {invalid_prompts}"
)
logger.info(f"Starting {request.mode} test: concurrency={request.concurrency}, prompts={request.prompt_ids or 'all'}")
try:
# Start test - it returns run_id immediately after creating DB record
logger.info(f"Starting {request.mode} with prompts: {request.prompt_ids or 'all'}, cleanup: {request.cleanup_threads}")
# Start test and get run_id immediately
if request.mode == 'core_test':
run_id = await runner.run_core_test(
prompt_ids=request.prompt_ids,
concurrency=request.concurrency,
model=request.model,
metadata=request.metadata
)
else: # stress_test
run_id = await runner.run_stress_test(
prompt_ids=request.prompt_ids,
concurrency=request.concurrency,
num_executions=request.num_executions,
metadata=request.metadata
)
return StartTestResponse(
run_id=run_id,
status="running",
message="Test started successfully"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error starting benchmark test: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to start test: {str(e)}"
)
@router.get("/runs/{run_id}", response_model=RunSummaryResponse, summary="Get Test Run Results")
async def get_test_run(
run_id: str,
_: bool = Depends(verify_admin_api_key)
):
"""
Get detailed results and summary for a benchmark test run
Requires X-Admin-Api-Key header for authentication.
"""
metrics = MetricsCollector()
try:
summary = await metrics.get_run_summary(run_id)
return RunSummaryResponse(**summary)
except Exception as e:
logger.error(f"Error retrieving test run {run_id}: {e}", exc_info=True)
if "not found" in str(e).lower():
raise HTTPException(
status_code=404,
detail=f"Test run {run_id} not found"
)
raise HTTPException(
status_code=500,
detail=f"Failed to retrieve test run: {str(e)}"
)
@router.get("/runs", response_model=RunListResponse, summary="List Test Runs")
async def list_test_runs(
limit: int = Query(20, description="Maximum number of runs to return", ge=1, le=100),
run_type: Optional[str] = Query(None, description="Filter by run type: 'core_test' or 'stress_test'"),
_: bool = Depends(verify_admin_api_key)
):
"""
List recent benchmark test runs
Requires X-Admin-Api-Key header for authentication.
"""
metrics = MetricsCollector()
# Validate run_type if provided
if run_type and run_type not in ['core_test', 'stress_test']:
raise HTTPException(
status_code=400,
detail=f"Invalid run_type: {run_type}. Must be 'core_test' or 'stress_test'"
)
try:
runs = await metrics.list_runs(limit=limit, run_type=run_type)
return RunListResponse(
runs=runs,
total=len(runs)
)
except Exception as e:
logger.error(f"Error listing test runs: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to list test runs: {str(e)}"
)
@router.post("/runs/{run_id}/cancel", summary="Cancel Test Run")
async def cancel_test_run(
run_id: str,
_: bool = Depends(verify_admin_api_key)
):
"""
Cancel an active benchmark test run
Requires X-Admin-Api-Key header for authentication.
"""
runner = get_runner()
try:
await runner.cancel_run(run_id)
# Cancel background task if exists
if run_id in _background_tasks:
task = _background_tasks[run_id]
if not task.done():
task.cancel()
_background_tasks.pop(run_id)
return {
"run_id": run_id,
"status": "cancelled",
"message": "Test run cancelled successfully"
}
except Exception as e:
logger.error(f"Error cancelling test run {run_id}: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to cancel test run: {str(e)}"
)
@router.post("/emergency-stop", summary="Emergency Stop All Tests")
async def emergency_stop_all_tests(
_: bool = Depends(verify_admin_api_key)
):
"""
🚨 EMERGENCY STOP - Cancel ALL active benchmark test runs
Use this in case of emergency to stop all running tests immediately.
Requires X-Admin-Api-Key header for authentication.
"""
runner = get_runner()
try:
# Get all active run IDs
active_runs = list(runner._active_runs.keys())
if not active_runs:
return {
"message": "No active test runs to stop",
"cancelled_count": 0
}
logger.warning(f"🚨 EMERGENCY STOP triggered for {len(active_runs)} active runs")
# Cancel all active runs
cancelled = []
errors = []
for run_id in active_runs:
try:
await runner.cancel_run(run_id)
cancelled.append(run_id)
# Cancel background task if exists
if run_id in _background_tasks:
task = _background_tasks[run_id]
if not task.done():
task.cancel()
_background_tasks.pop(run_id)
except Exception as e:
logger.error(f"Failed to cancel run {run_id}: {e}")
errors.append({"run_id": run_id, "error": str(e)})
return {
"message": f"Emergency stop completed - cancelled {len(cancelled)} test runs",
"cancelled_count": len(cancelled),
"cancelled_runs": cancelled,
"errors": errors if errors else None
}
except Exception as e:
logger.error(f"Error during emergency stop: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Emergency stop failed: {str(e)}"
)
@router.get("/prompts", summary="List Available Test Prompts")
async def list_test_prompts(
_: bool = Depends(verify_admin_api_key)
):
"""
List all available test prompts
Requires X-Admin-Api-Key header for authentication.
"""
from .prompts import TEST_PROMPTS
prompts = [
{
'id': p.id,
'text': p.text,
'category': p.category,
'expected_tools': p.expected_tools,
'min_tool_calls': p.min_tool_calls,
'max_duration_ms': p.max_duration_ms,
'description': p.description,
}
for p in TEST_PROMPTS
]
return {
'prompts': prompts,
'total': len(prompts)
}
@e2e_router.post("/e2e", summary="Run E2E API Tests")
async def run_e2e_tests(
test_filter: Optional[str] = Query(None, description="pytest filter expression (e.g., 'test_agents' or 'test_accounts::test_get_accounts')"),
_: bool = Depends(verify_admin_api_key)
):
"""
Trigger pytest E2E API test suite.
Runs the functional E2E tests from backend/tests/ directory.
Requires X-Admin-Api-Key header for authentication.
Args:
test_filter: Optional pytest filter expression to run specific tests.
Examples:
- "test_agents" - run all agent tests
- "test_accounts::test_get_accounts" - run specific test
- "test_full_flow" - run E2E flow tests
Returns:
Test execution results with status, returncode, stdout, and stderr
"""
import subprocess
import os
from pathlib import Path
# Get backend directory path
backend_dir = Path(__file__).parent.parent.parent
# Build pytest command
cmd = ["python", "-m", "pytest", "tests/", "-v", "--tb=short"]
if test_filter:
cmd.extend(["-k", test_filter])
logger.info(f"Running E2E tests: {' '.join(cmd)}")
try:
# Run pytest in backend directory
result = await asyncio.to_thread(
subprocess.run,
cmd,
capture_output=True,
text=True,
cwd=str(backend_dir),
timeout=600.0 # 10 minute timeout
)
return {
"status": "passed" if result.returncode == 0 else "failed",
"returncode": result.returncode,
"stdout": result.stdout,
"stderr": result.stderr,
"command": " ".join(cmd)
}
except subprocess.TimeoutExpired:
logger.error("E2E test execution timed out after 10 minutes")
return {
"status": "timeout",
"returncode": -1,
"stdout": "",
"stderr": "Test execution timed out after 10 minutes",
"command": " ".join(cmd)
}
except Exception as e:
logger.error(f"Error running E2E tests: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to run E2E tests: {str(e)}"
)