Download .github/workflows/scripts/load-test.sh from shimen/yapapaway: direct link, hf CLI and curl.
- Browser
- Download file 35 kB
-
https://huggingface.co/spaces/shimen/yapapaway/resolve/main/.github/workflows/scripts/load-test.sh
- Command line
-
hf download hf://spaces/shimen/yapapaway/.github/workflows/scripts/load-test.sh
-
curl -L -o load-test.sh https://huggingface.co/spaces/shimen/yapapaway/resolve/main/.github/workflows/scripts/load-test.sh
35 kB
| # Load Test Script for Bifrost | |
| # Runs a load test against bifrost-http with a mocker provider | |
| # Usage: ./load-test.sh | |
| # | |
| # This script: | |
| # 1. Builds bifrost-http and mocker locally | |
| # 2. Creates a config.json with mocker provider (OpenAI-style) | |
| # 3. Starts mocker with 0ms latency and bifrost-http | |
| # 4. Runs a calibration (Vegeta -> Mocker direct) to measure Vegeta+network baseline | |
| # 5. Runs the overhead test (Vegeta -> Bifrost -> Mocker) to measure total | |
| # 6. Subtracts calibration from test to isolate Bifrost proxy overhead | |
| # (includes local network hop, JSON parsing/unparsing, plugins, and mocker jitter) | |
| # 7. Restarts mocker with 10s latency for a sustained concurrency stress test | |
| # 8. Asserts overhead < tiered thresholds (per percentile) and stress test has 100% success rate | |
| set -e | |
| # Configuration | |
| SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | |
| REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" | |
| BIFROST_HTTP_DIR="${REPO_ROOT}/transports/bifrost-http" | |
| TRANSPORTS_DIR="${REPO_ROOT}/transports" | |
| WORK_DIR="${SCRIPT_DIR}" | |
| MOCKER_DIR="${REPO_ROOT}/../bifrost-benchmarking/mocker" | |
| BIFROST_PORT=8080 | |
| MOCKER_PORT=8000 | |
| RATE=1000 | |
| MAX_WORKERS=12000 | |
| OVERHEAD_DURATION=30 # overhead measurement duration (seconds) | |
| STRESS_DURATION=30 # stress test duration (seconds) | |
| OVERHEAD_MOCKER_LATENCY_MS=1000 # 1 second latency for overhead measurement | |
| STRESS_MOCKER_LATENCY_MS=1000 # 1 second latency for stress test | |
| # Tiered overhead thresholds (Β΅s) β these cover the full proxy cost: | |
| # local network hop, JSON parsing/unparsing, plugins, and mocker jitter. | |
| # At ${RATE} RPS Γ ${OVERHEAD_MOCKER_LATENCY_MS}ms latency β 1000 concurrent requests. | |
| MAX_OVERHEAD_MEAN_US=5000 # mean overhead threshold (5ms) | |
| MAX_OVERHEAD_P50_US=5000 # p50 overhead threshold (5ms) | |
| MAX_OVERHEAD_P90_US=10000 # p90 overhead threshold (10ms) | |
| MAX_OVERHEAD_P95_US=20000 # p95 overhead threshold (20ms) | |
| MAX_OVERHEAD_P99_US=100000 # p99 overhead threshold (100ms) | |
| # Results storage for summary table | |
| RESULTS_FILE="${WORK_DIR}/load-test-results.md" | |
| RESULTS_JSON="${WORK_DIR}/load-test-results.json" | |
| # Process stats monitoring | |
| STATS_PID="" | |
| STATS_FILE="${WORK_DIR}/bifrost-stats.csv" | |
| # Overhead-phase process stats (saved before bifrost restart) | |
| OVERHEAD_STATS_CPU_AVG="" | |
| OVERHEAD_STATS_CPU_PEAK="" | |
| OVERHEAD_STATS_RSS_AVG="" | |
| OVERHEAD_STATS_RSS_PEAK="" | |
| # Calibration results per bucket (Vegeta -> Mocker direct) | |
| CAL_MIN_NS=0 | |
| CAL_MEAN_NS=0 | |
| CAL_50_NS=0 | |
| CAL_90_NS=0 | |
| CAL_95_NS=0 | |
| CAL_99_NS=0 | |
| CAL_MAX_NS=0 | |
| # Colors for output | |
| RED='\033[0;31m' | |
| GREEN='\033[0;32m' | |
| YELLOW='\033[1;33m' | |
| BLUE='\033[0;34m' | |
| NC='\033[0m' # No Color | |
| log_info() { | |
| echo -e "${BLUE}[INFO]${NC} $1" | |
| } | |
| log_success() { | |
| echo -e "${GREEN}[SUCCESS]${NC} $1" | |
| } | |
| log_warn() { | |
| echo -e "${YELLOW}[WARN]${NC} $1" | |
| } | |
| log_error() { | |
| echo -e "${RED}[ERROR]${NC} $1" | |
| } | |
| # Cleanup function to kill background processes | |
| cleanup() { | |
| log_info "Cleaning up..." | |
| if [ -n "$STATS_PID" ] && kill -0 "$STATS_PID" 2>/dev/null; then | |
| kill "$STATS_PID" 2>/dev/null || true | |
| wait "$STATS_PID" 2>/dev/null || true | |
| fi | |
| if [ -n "$BIFROST_PID" ] && kill -0 "$BIFROST_PID" 2>/dev/null; then | |
| kill "$BIFROST_PID" 2>/dev/null || true | |
| wait "$BIFROST_PID" 2>/dev/null || true | |
| fi | |
| if [ -n "$MOCKER_PID" ] && kill -0 "$MOCKER_PID" 2>/dev/null; then | |
| kill "$MOCKER_PID" 2>/dev/null || true | |
| wait "$MOCKER_PID" 2>/dev/null || true | |
| fi | |
| # Clean up temporary files (keep results files for artifact upload) | |
| rm -f "${WORK_DIR}/config.json" "${WORK_DIR}/logs.db" "${WORK_DIR}/attack.bin" "${WORK_DIR}/calibration.bin" "${WORK_DIR}/stress.bin" "${WORK_DIR}/bifrost.log" "${WORK_DIR}/vegeta-target.json" "${WORK_DIR}/vegeta-target-calibration.json" "${WORK_DIR}/vegeta-target-stress.json" "${WORK_DIR}/vegeta-report.json" "${WORK_DIR}/bifrost-stats.csv" 2>/dev/null || true | |
| log_info "Cleanup complete" | |
| } | |
| trap cleanup EXIT | |
| # Check for required tools | |
| check_dependencies() { | |
| log_info "Checking dependencies..." | |
| if ! command -v go &> /dev/null; then | |
| log_error "Go is not installed. Please install Go 1.24.3 or later." | |
| exit 1 | |
| fi | |
| if ! command -v git &> /dev/null; then | |
| log_error "Git is not installed. Please install Git." | |
| exit 1 | |
| fi | |
| log_success "All dependencies found" | |
| } | |
| # Kill any process listening on a specific port (not processes with connections to it) | |
| kill_port() { | |
| local port=$1 | |
| local pids=$(lsof -ti "TCP:${port}" -sTCP:LISTEN 2>/dev/null) | |
| if [ -n "$pids" ]; then | |
| log_warn "Killing existing process(es) listening on port ${port}: ${pids}" | |
| echo "$pids" | xargs kill -9 2>/dev/null || true | |
| sleep 1 | |
| fi | |
| } | |
| # Kill processes on required ports before starting | |
| cleanup_ports() { | |
| log_info "Checking for processes on required ports..." | |
| kill_port ${MOCKER_PORT} | |
| kill_port ${BIFROST_PORT} | |
| } | |
| # Install Vegeta if not present | |
| install_vegeta() { | |
| if ! command -v vegeta &> /dev/null; then | |
| log_info "Installing Vegeta load testing tool..." | |
| go install github.com/tsenart/vegeta/v12@latest | |
| export PATH="$PATH:$(go env GOPATH)/bin" | |
| if ! command -v vegeta &> /dev/null; then | |
| log_error "Failed to install Vegeta" | |
| exit 1 | |
| fi | |
| log_success "Vegeta installed" | |
| else | |
| log_success "Vegeta already installed" | |
| fi | |
| } | |
| # Build bifrost-http if binary doesn't exist | |
| build_bifrost_http() { | |
| if [ -f "${REPO_ROOT}/tmp/bifrost-http" ]; then | |
| log_success "bifrost-http binary already exists at ${REPO_ROOT}/tmp/bifrost-http" | |
| return 0 | |
| fi | |
| log_info "Building bifrost-http..." | |
| cd "${TRANSPORTS_DIR}" | |
| if go build -o ${REPO_ROOT}/tmp/bifrost-http .; then | |
| log_success "bifrost-http built successfully" | |
| else | |
| log_error "Failed to build bifrost-http" | |
| exit 1 | |
| fi | |
| cd "${WORK_DIR}" | |
| } | |
| # Clone and setup mocker from bifrost-benchmarking | |
| setup_mocker() { | |
| if [ -d "${REPO_ROOT}/../bifrost-benchmarking" ]; then | |
| log_info "Updating bifrost-benchmarking repository..." | |
| cd "${REPO_ROOT}/../bifrost-benchmarking" | |
| git pull --quiet || true | |
| cd "${WORK_DIR}" | |
| else | |
| log_info "Cloning bifrost-benchmarking repository..." | |
| cd "${WORK_DIR}" | |
| git clone --depth 1 https://github.com/maximhq/bifrost-benchmarking.git | |
| fi | |
| log_success "Mocker setup complete" | |
| } | |
| # Build mocker binary (avoids go run overhead) | |
| build_mocker() { | |
| if [ -f "${REPO_ROOT}/tmp/mocker" ]; then | |
| log_success "mocker binary already exists at ${REPO_ROOT}/tmp/mocker" | |
| return 0 | |
| fi | |
| log_info "Building mocker..." | |
| cd "${MOCKER_DIR}" | |
| if go build -o "${REPO_ROOT}/tmp/mocker" .; then | |
| log_success "mocker built successfully" | |
| else | |
| log_error "Failed to build mocker" | |
| exit 1 | |
| fi | |
| cd "${WORK_DIR}" | |
| } | |
| # Create config.json for bifrost with mocker provider | |
| create_config() { | |
| log_info "Creating config.json..." | |
| cat > "${WORK_DIR}/config.json" << 'EOF' | |
| { | |
| "$schema": "https://www.getbifrost.ai/schema", | |
| "client": { | |
| "enable_logging": false, | |
| "initial_pool_size": 20000, | |
| "drop_excess_requests": false, | |
| "allow_direct_keys": false | |
| }, | |
| "config_store": { | |
| "enabled": false | |
| }, | |
| "logs_store": { | |
| "enabled": false | |
| }, | |
| "providers": { | |
| "openai": { | |
| "keys": [ | |
| { | |
| "name": "mocker-key", | |
| "value": "Bearer mocker-key", | |
| "weight": 1 | |
| } | |
| ], | |
| "network_config": { | |
| "base_url": "http://localhost:8000", | |
| "default_request_timeout_in_seconds": 30 | |
| }, | |
| "concurrency_and_buffer_size": { | |
| "concurrency": 20000, | |
| "buffer_size": 40000 | |
| }, | |
| "custom_provider_config": { | |
| "base_provider_type": "openai", | |
| "allowed_requests": { | |
| "list_models": false, | |
| "chat_completion": true, | |
| "chat_completion_stream": true | |
| } | |
| } | |
| } | |
| } | |
| } | |
| EOF | |
| log_success "config.json created" | |
| } | |
| # Start mocker with specified latency | |
| # Arguments: $1 = latency in ms | |
| start_mocker() { | |
| local latency_ms=${1:-0} | |
| log_info "Starting mocker server on port ${MOCKER_PORT} with ${latency_ms}ms latency..." | |
| "${REPO_ROOT}/tmp/mocker" -port ${MOCKER_PORT} -host 0.0.0.0 -latency ${latency_ms} & | |
| MOCKER_PID=$! | |
| # Wait for mocker to be ready | |
| local max_attempts=30 | |
| local attempt=0 | |
| while ! curl -s "http://localhost:${MOCKER_PORT}/v1/chat/completions" -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer mocker-key" \ | |
| -d '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"test"}]}' > /dev/null 2>&1; do | |
| sleep 1 | |
| attempt=$((attempt + 1)) | |
| if [ $attempt -ge $max_attempts ]; then | |
| log_error "Mocker failed to start within ${max_attempts} seconds" | |
| exit 1 | |
| fi | |
| done | |
| log_success "Mocker server started (PID: ${MOCKER_PID})" | |
| } | |
| # Stop mocker | |
| stop_mocker() { | |
| if [ -n "$MOCKER_PID" ] && kill -0 "$MOCKER_PID" 2>/dev/null; then | |
| log_info "Stopping mocker (PID: ${MOCKER_PID})..." | |
| kill "$MOCKER_PID" 2>/dev/null || true | |
| wait "$MOCKER_PID" 2>/dev/null || true | |
| MOCKER_PID="" | |
| sleep 1 | |
| fi | |
| } | |
| # Stop bifrost-http server | |
| stop_bifrost() { | |
| if [ -n "$BIFROST_PID" ] && kill -0 "$BIFROST_PID" 2>/dev/null; then | |
| log_info "Stopping bifrost (PID: ${BIFROST_PID})..." | |
| kill "$BIFROST_PID" 2>/dev/null || true | |
| wait "$BIFROST_PID" 2>/dev/null || true | |
| BIFROST_PID="" | |
| sleep 1 | |
| fi | |
| } | |
| # Start background process stats collection for bifrost | |
| # Samples CPU% and RSS every second, writes to CSV | |
| start_stats_monitor() { | |
| if [ -z "$BIFROST_PID" ] || ! kill -0 "$BIFROST_PID" 2>/dev/null; then | |
| log_warn "Cannot start stats monitor: bifrost not running" | |
| return | |
| fi | |
| echo "timestamp,cpu_pct,rss_mb" > "${STATS_FILE}" | |
| ( | |
| while kill -0 "$BIFROST_PID" 2>/dev/null; do | |
| # ps -o %cpu= -o rss= works on both macOS and Linux | |
| stats=$(ps -p "$BIFROST_PID" -o %cpu=,rss= 2>/dev/null) | |
| if [ -n "$stats" ]; then | |
| cpu=$(echo "$stats" | awk '{print $1}') | |
| rss_kb=$(echo "$stats" | awk '{print $2}') | |
| rss_mb=$(echo "scale=1; ${rss_kb} / 1024" | bc) | |
| echo "$(date +%s),${cpu},${rss_mb}" >> "${STATS_FILE}" | |
| fi | |
| sleep 1 | |
| done | |
| ) & | |
| STATS_PID=$! | |
| log_info "Stats monitor started (PID: ${STATS_PID})" | |
| } | |
| # Stop stats monitor and print summary | |
| stop_stats_monitor() { | |
| if [ -n "$STATS_PID" ] && kill -0 "$STATS_PID" 2>/dev/null; then | |
| kill "$STATS_PID" 2>/dev/null || true | |
| wait "$STATS_PID" 2>/dev/null || true | |
| STATS_PID="" | |
| fi | |
| if [ ! -f "${STATS_FILE}" ] || [ $(wc -l < "${STATS_FILE}") -le 1 ]; then | |
| log_warn "No process stats collected" | |
| return | |
| fi | |
| # Compute peak and average CPU/RSS from CSV (skip header) | |
| if command -v awk &> /dev/null; then | |
| local stats_summary=$(awk -F',' 'NR>1 { | |
| cpu_sum+=$2; rss_sum+=$3; n++; | |
| if($2>cpu_max) cpu_max=$2; | |
| if($3>rss_max) rss_max=$3; | |
| } END { | |
| if(n>0) printf "%.1f,%.1f,%.1f,%.1f,%d", cpu_sum/n, cpu_max, rss_sum/n, rss_max, n | |
| }' "${STATS_FILE}") | |
| STATS_CPU_AVG=$(echo "$stats_summary" | cut -d',' -f1) | |
| STATS_CPU_PEAK=$(echo "$stats_summary" | cut -d',' -f2) | |
| STATS_RSS_AVG=$(echo "$stats_summary" | cut -d',' -f3) | |
| STATS_RSS_PEAK=$(echo "$stats_summary" | cut -d',' -f4) | |
| local samples=$(echo "$stats_summary" | cut -d',' -f5) | |
| echo "" | |
| log_success "Bifrost process stats (single instance, ${samples} samples):" | |
| log_info " CPU: avg=${STATS_CPU_AVG}%, peak=${STATS_CPU_PEAK}%" | |
| log_info " RSS: avg=${STATS_RSS_AVG}MB, peak=${STATS_RSS_PEAK}MB" | |
| fi | |
| } | |
| # Start bifrost-http server | |
| start_bifrost() { | |
| log_info "Starting bifrost-http on port ${BIFROST_PORT}..." | |
| cd "${WORK_DIR}" | |
| local bifrost_log="${WORK_DIR}/bifrost.log" | |
| "${REPO_ROOT}/tmp/bifrost-http" -app-dir "${WORK_DIR}" -port "${BIFROST_PORT}" -host "0.0.0.0" -log-level "info" > "${bifrost_log}" 2>&1 & | |
| BIFROST_PID=$! | |
| # Wait for bifrost to be fully ready (look for "successfully started bifrost" message) | |
| local max_attempts=60 | |
| local attempt=0 | |
| while ! grep -q "successfully started bifrost" "${bifrost_log}" 2>/dev/null; do | |
| sleep 1 | |
| attempt=$((attempt + 1)) | |
| if [ $attempt -ge $max_attempts ]; then | |
| log_error "Bifrost failed to start within ${max_attempts} seconds" | |
| log_error "Bifrost log output:" | |
| cat "${bifrost_log}" 2>/dev/null || true | |
| exit 1 | |
| fi | |
| # Check if process is still running | |
| if ! kill -0 "$BIFROST_PID" 2>/dev/null; then | |
| log_error "Bifrost process died unexpectedly" | |
| log_error "Bifrost log output:" | |
| cat "${bifrost_log}" 2>/dev/null || true | |
| exit 1 | |
| fi | |
| done | |
| log_success "Bifrost-http started (PID: ${BIFROST_PID})" | |
| } | |
| # Extract latencies from a vegeta binary results file | |
| # Arguments: $1 = path to .bin file | |
| # Sets: EXTRACTED_MIN_NS, EXTRACTED_MEAN_NS, EXTRACTED_50_NS, etc. | |
| extract_latencies() { | |
| local bin_file=$1 | |
| local json_report_file="${WORK_DIR}/vegeta-report.json" | |
| vegeta report -type=json < "${bin_file}" > "${json_report_file}" | |
| if command -v jq &> /dev/null; then | |
| EXTRACTED_MIN_NS=$(jq '.latencies.min // 0' "${json_report_file}") | |
| EXTRACTED_MEAN_NS=$(jq '.latencies.mean // 0' "${json_report_file}") | |
| EXTRACTED_50_NS=$(jq '.latencies["50th"] // 0' "${json_report_file}") | |
| EXTRACTED_90_NS=$(jq '.latencies["90th"] // 0' "${json_report_file}") | |
| EXTRACTED_95_NS=$(jq '.latencies["95th"] // 0' "${json_report_file}") | |
| EXTRACTED_99_NS=$(jq '.latencies["99th"] // 0' "${json_report_file}") | |
| EXTRACTED_MAX_NS=$(jq '.latencies.max // 0' "${json_report_file}") | |
| EXTRACTED_SUCCESS=$(jq '.success // 0' "${json_report_file}") | |
| EXTRACTED_RATE=$(jq '.rate // 0' "${json_report_file}") | |
| EXTRACTED_THROUGHPUT=$(jq '.throughput // 0' "${json_report_file}") | |
| elif command -v python3 &> /dev/null; then | |
| EXTRACTED_MIN_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('min', 0))") | |
| EXTRACTED_MEAN_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('mean', 0))") | |
| EXTRACTED_50_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('50th', 0))") | |
| EXTRACTED_90_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('90th', 0))") | |
| EXTRACTED_95_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('95th', 0))") | |
| EXTRACTED_99_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('99th', 0))") | |
| EXTRACTED_MAX_NS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('latencies', {}).get('max', 0))") | |
| EXTRACTED_SUCCESS=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('success', 0))") | |
| EXTRACTED_RATE=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('rate', 0))") | |
| EXTRACTED_THROUGHPUT=$(python3 -c "import json; d=json.load(open('${json_report_file}')); print(d.get('throughput', 0))") | |
| else | |
| log_error "Neither jq nor python3 found. Cannot parse JSON results." | |
| return 1 | |
| fi | |
| rm -f "${json_report_file}" | |
| } | |
| # ============================================================ | |
| # Phase 1: Overhead measurement (mocker at ${OVERHEAD_MOCKER_LATENCY_MS}ms) | |
| # ============================================================ | |
| # Calibration: Vegeta -> Mocker direct (with latency) | |
| # Measures: Vegeta HTTP client + localhost network round-trip + mocker response generation | |
| run_calibration() { | |
| echo "" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "β Calibration: Vegeta -> Mocker (${OVERHEAD_MOCKER_LATENCY_MS}ms, direct) β" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "" | |
| log_info "Measuring Vegeta + network baseline (mocker at ${OVERHEAD_MOCKER_LATENCY_MS}ms latency)" | |
| log_info "Duration: ${OVERHEAD_DURATION}s at ${RATE} RPS, ~$(( RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000 )) concurrent" | |
| echo "" | |
| local target_file="${WORK_DIR}/vegeta-target-calibration.json" | |
| local payload='{"model":"gpt-4o-mini","messages":[{"role":"user","content":"Hello, how are you?"}]}' | |
| cat > "${target_file}" << EOF | |
| {"method": "POST", "url": "http://localhost:${MOCKER_PORT}/v1/chat/completions", "header": {"Content-Type": ["application/json"], "Authorization": ["Bearer mocker-key"]}, "body": "$(echo -n "${payload}" | base64)"} | |
| EOF | |
| vegeta attack \ | |
| -format=json \ | |
| -targets="${target_file}" \ | |
| -rate="${RATE}" \ | |
| -duration="${OVERHEAD_DURATION}s" \ | |
| -timeout="$((OVERHEAD_MOCKER_LATENCY_MS / 1000 + 5))s" \ | |
| -workers=$((RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000)) \ | |
| -max-workers="${MAX_WORKERS}" > "${WORK_DIR}/calibration.bin" | |
| echo "" | |
| log_info "Calibration complete. Results:" | |
| vegeta report < "${WORK_DIR}/calibration.bin" | |
| extract_latencies "${WORK_DIR}/calibration.bin" | |
| log_info "Actual RPS: $(printf "%.0f" $EXTRACTED_RATE) (configured: ${RATE})" | |
| CAL_MIN_NS=$EXTRACTED_MIN_NS | |
| CAL_MEAN_NS=$EXTRACTED_MEAN_NS | |
| CAL_50_NS=$EXTRACTED_50_NS | |
| CAL_90_NS=$EXTRACTED_90_NS | |
| CAL_95_NS=$EXTRACTED_95_NS | |
| CAL_99_NS=$EXTRACTED_99_NS | |
| CAL_MAX_NS=$EXTRACTED_MAX_NS | |
| echo "" | |
| log_success "Calibration baseline (per bucket):" | |
| log_info " Min: $(echo "scale=2; $CAL_MIN_NS / 1000" | bc)Β΅s" | |
| log_info " Mean: $(echo "scale=2; $CAL_MEAN_NS / 1000" | bc)Β΅s" | |
| log_info " P50: $(echo "scale=2; $CAL_50_NS / 1000" | bc)Β΅s" | |
| log_info " P90: $(echo "scale=2; $CAL_90_NS / 1000" | bc)Β΅s" | |
| log_info " P95: $(echo "scale=2; $CAL_95_NS / 1000" | bc)Β΅s" | |
| log_info " P99: $(echo "scale=2; $CAL_99_NS / 1000" | bc)Β΅s" | |
| log_info " Max: $(echo "scale=2; $CAL_MAX_NS / 1000" | bc)Β΅s" | |
| } | |
| # Overhead test: Vegeta -> Bifrost -> Mocker (with latency) | |
| # Same duration/rate as calibration so percentile distributions are comparable | |
| run_overhead_test() { | |
| echo "" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "β Overhead Test: Vegeta -> Bifrost -> Mocker (${OVERHEAD_MOCKER_LATENCY_MS}ms) β" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "" | |
| log_info "Measuring Bifrost overhead (single instance, mocker at ${OVERHEAD_MOCKER_LATENCY_MS}ms latency)" | |
| log_info "Duration: ${OVERHEAD_DURATION}s at ${RATE} RPS, ~$(( RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000 )) concurrent requests through Bifrost" | |
| log_info "Overhead consists of: vegetta overhead and mocker timeout jitter" | |
| echo "" | |
| local target_file="${WORK_DIR}/vegeta-target.json" | |
| local payload='{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"Hello, how are you?"}]}' | |
| cat > "${target_file}" << EOF | |
| {"method": "POST", "url": "http://localhost:${BIFROST_PORT}/v1/chat/completions", "header": {"Content-Type": ["application/json"]}, "body": "$(echo -n "${payload}" | base64)"} | |
| EOF | |
| vegeta attack \ | |
| -format=json \ | |
| -targets="${target_file}" \ | |
| -rate="${RATE}" \ | |
| -duration="${OVERHEAD_DURATION}s" \ | |
| -timeout="$((OVERHEAD_MOCKER_LATENCY_MS / 1000 + 5))s" \ | |
| -workers=$((RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000)) \ | |
| -max-workers="${MAX_WORKERS}" > "${WORK_DIR}/attack.bin" | |
| echo "" | |
| log_info "Overhead test complete. Results:" | |
| vegeta report < "${WORK_DIR}/attack.bin" | |
| echo "" | |
| log_info "Latency histogram:" | |
| vegeta report -type=hist[0,100us,500us,1ms,5ms,10ms,50ms,100ms] < "${WORK_DIR}/attack.bin" || log_warn "Histogram generation failed" | |
| # Extract and compute overhead | |
| extract_latencies "${WORK_DIR}/attack.bin" | |
| log_info " Raw latencies (ns): min=$EXTRACTED_MIN_NS, mean=$EXTRACTED_MEAN_NS, p50=$EXTRACTED_50_NS, p99=$EXTRACTED_99_NS, max=$EXTRACTED_MAX_NS" | |
| log_info " Success rate: $EXTRACTED_SUCCESS" | |
| log_info " Actual RPS: $(printf "%.0f" $EXTRACTED_RATE) (configured: ${RATE})" | |
| if [ -z "$EXTRACTED_MIN_NS" ] || [ "$EXTRACTED_MIN_NS" = "0" ] || [ "$EXTRACTED_MIN_NS" = "null" ]; then | |
| log_error "Failed to extract latency values from vegeta report" | |
| exit 1 | |
| fi | |
| # Subtract calibration per bucket: overhead = through_bifrost - direct_to_mocker | |
| local us_min=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_MIN_NS - $CAL_MIN_NS) / 1000" | bc)) | |
| local us_mean=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_MEAN_NS - $CAL_MEAN_NS) / 1000" | bc)) | |
| local us_50=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_50_NS - $CAL_50_NS) / 1000" | bc)) | |
| local us_90=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_90_NS - $CAL_90_NS) / 1000" | bc)) | |
| local us_95=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_95_NS - $CAL_95_NS) / 1000" | bc)) | |
| local us_99=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_99_NS - $CAL_99_NS) / 1000" | bc)) | |
| local us_max=$(printf "%.2f" $(echo "scale=4; ($EXTRACTED_MAX_NS - $CAL_MAX_NS) / 1000" | bc)) | |
| local success_pct=$(printf "%.2f" $(echo "scale=4; $EXTRACTED_SUCCESS * 100" | bc)) | |
| echo "" | |
| log_success "Bifrost overhead (per bucket):" | |
| log_info " Min: ${us_min}Β΅s" | |
| log_info " Mean: ${us_mean}Β΅s" | |
| log_info " P50: ${us_50}Β΅s" | |
| log_info " P90: ${us_90}Β΅s" | |
| log_info " P95: ${us_95}Β΅s" | |
| log_info " P99: ${us_99}Β΅s" | |
| log_info " Max: ${us_max}Β΅s" | |
| local actual_rps=$(printf "%.0f" $EXTRACTED_RATE) | |
| # Write results | |
| cat > "${RESULTS_FILE}" << EOF | |
| # Bifrost Load Test Results (single instance, ${actual_rps} RPS) | |
| ## Bifrost Processing Overhead | |
| | Metric | Actual RPS | Duration | Concurrent | Success Rate | Min | Mean | P50 | P90 | P95 | P99 | Max | | |
| |--------|-----------|----------|------------|--------------|-----|------|-----|-----|-----|-----|-----| | |
| | Overhead | ${actual_rps} | ${OVERHEAD_DURATION}s | ~$((RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000)) | ${success_pct}% | ${us_min}Β΅s | ${us_mean}Β΅s | ${us_50}Β΅s | ${us_90}Β΅s | ${us_95}Β΅s | ${us_99}Β΅s | ${us_max}Β΅s | | |
| EOF | |
| echo '{"overhead": {"configured_rate": '"${RATE}"', "actual_rate": '"${actual_rps}"', "duration": '"${OVERHEAD_DURATION}"', "concurrent": '$((RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000))', "success_rate": '"${success_pct}"', "latency_us": {"min": '"${us_min}"', "mean": '"${us_mean}"', "p50": '"${us_50}"', "p90": '"${us_90}"', "p95": '"${us_95}"', "p99": '"${us_99}"', "max": '"${us_max}"'}}, "timestamp": "'"$(date -u +"%Y-%m-%dT%H:%M:%SZ")"'"}' > "${RESULTS_JSON}" | |
| # Check tiered thresholds (skip Min/Max β single-point extremes are too noisy) | |
| local failed=0 | |
| local labels=("Mean" "P50" "P90" "P95" "P99") | |
| local real_values=($EXTRACTED_MEAN_NS $EXTRACTED_50_NS $EXTRACTED_90_NS $EXTRACTED_95_NS $EXTRACTED_99_NS) | |
| local cal_values=($CAL_MEAN_NS $CAL_50_NS $CAL_90_NS $CAL_95_NS $CAL_99_NS) | |
| local thresholds=($MAX_OVERHEAD_MEAN_US $MAX_OVERHEAD_P50_US $MAX_OVERHEAD_P90_US $MAX_OVERHEAD_P95_US $MAX_OVERHEAD_P99_US) | |
| local extras=() | |
| for i in "${!real_values[@]}"; do | |
| local overhead_us=$(( (real_values[i] - cal_values[i]) / 1000 )) | |
| if [ "$overhead_us" -gt "${thresholds[i]}" ]; then | |
| extras+=("${labels[i]}:${overhead_us}:${thresholds[i]}") | |
| failed=1 | |
| fi | |
| done | |
| if [ "$failed" -eq 1 ]; then | |
| echo "" | |
| log_error "FAILED: Bifrost overhead exceeded tiered thresholds" | |
| log_error "Overhead consists of: vegetta overhead and mocker timeout jitter. In real-world the P99 overhead will be approximately 100 microseconds." | |
| echo "" | |
| echo -e "${RED}| Bucket | Overhead (Β΅s) | Threshold (Β΅s) |${NC}" | |
| echo -e "${RED}|--------|---------------|----------------|${NC}" | |
| for entry in "${extras[@]}"; do | |
| IFS=: read -r bucket overhead threshold <<< "$entry" | |
| echo -e "${RED}| ${bucket} | ${overhead}Β΅s | ${threshold}Β΅s |${NC}" | |
| done | |
| echo "" | |
| stop_stats_monitor | |
| exit 1 | |
| fi | |
| log_success "All overhead buckets within tiered thresholds (mean<${MAX_OVERHEAD_MEAN_US}Β΅s, p50<${MAX_OVERHEAD_P50_US}Β΅s, p90<${MAX_OVERHEAD_P90_US}Β΅s, p95<${MAX_OVERHEAD_P95_US}Β΅s, p99<${MAX_OVERHEAD_P99_US}Β΅s)" | |
| } | |
| # ============================================================ | |
| # Phase 2: Stress test (mocker at 10s latency) | |
| # ============================================================ | |
| # Arguments: $1 = label (e.g. "Stress #1", "Stress #2") | |
| run_stress_test() { | |
| local label="${1:-Stress}" | |
| local bin_file="${WORK_DIR}/stress.bin" | |
| echo "" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "β ${label}: ${RATE} RPS with ${STRESS_MOCKER_LATENCY_MS}ms mocker latency β" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "" | |
| log_info "Testing single Bifrost instance under sustained concurrency" | |
| log_info "Duration: ${STRESS_DURATION}s at ${RATE} RPS (${STRESS_MOCKER_LATENCY_MS}ms mocker latency)" | |
| log_info "Expected concurrent requests: ~$(( RATE * STRESS_MOCKER_LATENCY_MS / 1000 )) (provider concurrency: 15,000, buffer: 20,000)" | |
| echo "" | |
| local target_file="${WORK_DIR}/vegeta-target-stress.json" | |
| local payload='{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"Hello, how are you?"}]}' | |
| cat > "${target_file}" << EOF | |
| {"method": "POST", "url": "http://localhost:${BIFROST_PORT}/v1/chat/completions", "header": {"Content-Type": ["application/json"]}, "body": "$(echo -n "${payload}" | base64)"} | |
| EOF | |
| vegeta attack \ | |
| -format=json \ | |
| -targets="${target_file}" \ | |
| -rate="${RATE}" \ | |
| -duration="${STRESS_DURATION}s" \ | |
| -timeout="30s" \ | |
| -workers=$((RATE * STRESS_MOCKER_LATENCY_MS / 1000)) \ | |
| -max-workers="${MAX_WORKERS}" > "${bin_file}" | |
| echo "" | |
| log_info "${label} complete. Results:" | |
| vegeta report < "${bin_file}" | |
| echo "" | |
| log_info "Latency histogram:" | |
| vegeta report -type=hist[0,1ms,5ms,10ms,50ms,100ms,500ms,1s,5s,10s,15s] < "${bin_file}" || log_warn "Histogram generation failed" | |
| # Check success rate | |
| extract_latencies "${bin_file}" | |
| local success_pct=$(printf "%.2f" $(echo "scale=4; $EXTRACTED_SUCCESS * 100" | bc)) | |
| log_info "Actual RPS: $(printf "%.0f" $EXTRACTED_RATE) (configured: ${RATE})" | |
| local stress_actual_rps=$(printf "%.0f" $EXTRACTED_RATE) | |
| # Append stress test results to results file | |
| cat >> "${RESULTS_FILE}" << EOF | |
| ## ${label} (${STRESS_MOCKER_LATENCY_MS}ms mocker latency) | |
| | Metric | Actual RPS | Duration | Concurrent | Success Rate | Min | Mean | P50 | P90 | P95 | P99 | Max | | |
| |--------|-----------|----------|------------|--------------|-----|------|-----|-----|-----|-----|-----| | |
| | ${label} | ${stress_actual_rps} | ${STRESS_DURATION}s | ~$((RATE * STRESS_MOCKER_LATENCY_MS / 1000)) | ${success_pct}% | $(echo "scale=2; $EXTRACTED_MIN_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_MEAN_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_50_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_90_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_95_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_99_NS / 1000000" | bc)ms | $(echo "scale=2; $EXTRACTED_MAX_NS / 1000000" | bc)ms | | |
| EOF | |
| if [ "$success_pct" != "100.00" ]; then | |
| echo "" | |
| log_error "FAILED: ${label} success rate is ${success_pct}% (expected 100%)" | |
| exit 1 | |
| fi | |
| log_success "${label} passed: ${success_pct}% success rate" | |
| } | |
| # ============================================================ | |
| # Finalize | |
| # ============================================================ | |
| finalize_results() { | |
| # Append process stats if available | |
| local has_overhead_stats=false | |
| local has_stress_stats=false | |
| if [ -n "$OVERHEAD_STATS_CPU_PEAK" ]; then | |
| has_overhead_stats=true | |
| fi | |
| if [ -n "$STATS_CPU_PEAK" ]; then | |
| has_stress_stats=true | |
| fi | |
| if [ "$has_overhead_stats" = true ] || [ "$has_stress_stats" = true ]; then | |
| cat >> "${RESULTS_FILE}" << 'EOF' | |
| ## Bifrost Process Stats (single instance) | |
| | Phase | CPU Avg | CPU Peak | RSS Avg | RSS Peak | | |
| |-------|---------|----------|---------|----------| | |
| EOF | |
| if [ "$has_overhead_stats" = true ]; then | |
| echo "| Overhead | ${OVERHEAD_STATS_CPU_AVG}% | ${OVERHEAD_STATS_CPU_PEAK}% | ${OVERHEAD_STATS_RSS_AVG}MB | ${OVERHEAD_STATS_RSS_PEAK}MB |" >> "${RESULTS_FILE}" | |
| fi | |
| if [ "$has_stress_stats" = true ]; then | |
| echo "| Stress | ${STATS_CPU_AVG}% | ${STATS_CPU_PEAK}% | ${STATS_RSS_AVG}MB | ${STATS_RSS_PEAK}MB |" >> "${RESULTS_FILE}" | |
| fi | |
| fi | |
| cat >> "${RESULTS_FILE}" << EOF | |
| ## Method | |
| - **Single instance**: All tests run against one bifrost-http process at ${RATE} RPS | |
| - **Overhead measurement**: Mocker at ${OVERHEAD_MOCKER_LATENCY_MS}ms latency, calibration (Vegeta->Mocker) subtracted from test (Vegeta->Bifrost->Mocker) | |
| - **Stress test**: Mocker at ${STRESS_MOCKER_LATENCY_MS}ms latency, verifies 100% success under sustained concurrency | |
| ## Notes | |
| - Overhead values are in microseconds (Β΅s), stress test values in milliseconds (ms) | |
| - Overhead ignores the mocker jitter, local network request queuing. In real-world the P99 overhead will be approximately 100 microseconds. | |
| - Tiered overhead thresholds: mean<${MAX_OVERHEAD_MEAN_US}Β΅s, p50<${MAX_OVERHEAD_P50_US}Β΅s, p90<${MAX_OVERHEAD_P90_US}Β΅s, p95<${MAX_OVERHEAD_P95_US}Β΅s, p99<${MAX_OVERHEAD_P99_US}Β΅s | |
| - P50/P90/P95/P99 represent percentile latencies | |
| --- | |
| *Generated by Bifrost Load Test Script* | |
| EOF | |
| # Update JSON with stress results and process stats | |
| local tmp_json=$(mktemp) | |
| if command -v jq &> /dev/null; then | |
| jq --arg sr "$(printf "%.2f" $(echo "scale=4; $EXTRACTED_SUCCESS * 100" | bc))" \ | |
| --arg cpu_avg "${STATS_CPU_AVG:-0}" --arg cpu_peak "${STATS_CPU_PEAK:-0}" \ | |
| --arg rss_avg "${STATS_RSS_AVG:-0}" --arg rss_peak "${STATS_RSS_PEAK:-0}" \ | |
| --arg oh_cpu_avg "${OVERHEAD_STATS_CPU_AVG:-0}" --arg oh_cpu_peak "${OVERHEAD_STATS_CPU_PEAK:-0}" \ | |
| --arg oh_rss_avg "${OVERHEAD_STATS_RSS_AVG:-0}" --arg oh_rss_peak "${OVERHEAD_STATS_RSS_PEAK:-0}" \ | |
| '.stress = {"rate": '"${RATE}"', "duration": '"${STRESS_DURATION}"', "mocker_latency_ms": '"${STRESS_MOCKER_LATENCY_MS}"', "success_rate": ($sr | tonumber)} | .process_stats = {"overhead": {"cpu_avg_pct": ($oh_cpu_avg | tonumber), "cpu_peak_pct": ($oh_cpu_peak | tonumber), "rss_avg_mb": ($oh_rss_avg | tonumber), "rss_peak_mb": ($oh_rss_peak | tonumber)}, "stress": {"cpu_avg_pct": ($cpu_avg | tonumber), "cpu_peak_pct": ($cpu_peak | tonumber), "rss_avg_mb": ($rss_avg | tonumber), "rss_peak_mb": ($rss_peak | tonumber)}}' \ | |
| "${RESULTS_JSON}" > "${tmp_json}" | |
| mv "${tmp_json}" "${RESULTS_JSON}" | |
| fi | |
| log_success "Results saved to:" | |
| log_info " - Markdown: ${RESULTS_FILE}" | |
| log_info " - JSON: ${RESULTS_JSON}" | |
| } | |
| # Main execution | |
| main() { | |
| echo "" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "β Bifrost Load Test (single instance, ${RATE} RPS) β" | |
| echo "βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "" | |
| log_info "Configuration: single bifrost-http instance, ${RATE} RPS" | |
| log_info "Provider concurrency: 15,000 (buffer: 20,000)" | |
| log_info "Overhead thresholds: mean<${MAX_OVERHEAD_MEAN_US}Β΅s, p50<${MAX_OVERHEAD_P50_US}Β΅s, p90<${MAX_OVERHEAD_P90_US}Β΅s, p95<${MAX_OVERHEAD_P95_US}Β΅s, p99<${MAX_OVERHEAD_P99_US}Β΅s" | |
| log_info "Phase 1: Overhead measurement β ${OVERHEAD_MOCKER_LATENCY_MS}ms mocker, ${OVERHEAD_DURATION}s, ~$(( RATE * OVERHEAD_MOCKER_LATENCY_MS / 1000 )) concurrent requests" | |
| log_info "Phase 2: Stress test β ${STRESS_MOCKER_LATENCY_MS}ms mocker, ${STRESS_DURATION}s, ~$(( RATE * STRESS_MOCKER_LATENCY_MS / 1000 )) concurrent requests" | |
| check_dependencies | |
| install_vegeta | |
| build_bifrost_http | |
| setup_mocker | |
| build_mocker | |
| create_config | |
| cleanup_ports | |
| # ββ Phase 1: Overhead measurement with ${OVERHEAD_MOCKER_LATENCY_MS}ms mocker ββ | |
| start_mocker ${OVERHEAD_MOCKER_LATENCY_MS} | |
| start_bifrost | |
| start_stats_monitor | |
| run_calibration | |
| run_overhead_test | |
| # ββ Collect process stats from overhead phase ββ | |
| stop_stats_monitor | |
| OVERHEAD_STATS_CPU_AVG="${STATS_CPU_AVG}" | |
| OVERHEAD_STATS_CPU_PEAK="${STATS_CPU_PEAK}" | |
| OVERHEAD_STATS_RSS_AVG="${STATS_RSS_AVG}" | |
| OVERHEAD_STATS_RSS_PEAK="${STATS_RSS_PEAK}" | |
| # ββ Phase 2: Stress test with high-latency mocker ββ | |
| # Restart both mocker and bifrost to ensure a clean fasthttp connection pool. | |
| # Without restarting bifrost, stale TCP connections from the overhead phase | |
| # (which used a different mocker process) cause immediate 400s on POST requests | |
| # because fasthttp does not retry non-idempotent methods on broken connections. | |
| stop_mocker | |
| stop_bifrost | |
| start_mocker ${STRESS_MOCKER_LATENCY_MS} | |
| start_bifrost | |
| start_stats_monitor | |
| run_stress_test "Stress #1" | |
| echo "" | |
| log_info "Waiting 30s before second stress test (idle period)..." | |
| sleep 30 | |
| run_stress_test "Stress #2" | |
| # ββ Collect process stats from stress phase ββ | |
| stop_stats_monitor | |
| # ββ Finalize ββ | |
| finalize_results | |
| cleanup_ports | |
| echo "" | |
| # Print final summary | |
| echo "ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "β FINAL RESULTS SUMMARY β" | |
| echo "ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ" | |
| echo "" | |
| cat "${RESULTS_FILE}" | |
| echo "" | |
| log_success "All tests passed!" | |
| } | |
| main "$@" | |