File size: 4,297 Bytes
c2d547f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | #!/usr/bin/env bash
# Fully automated baseline-vs-optimized benchmark for zai-org/GLM-5.2 (ROCm).
# Starts the baseline server, benchmarks it, tears it down, starts the
# optimized server, benchmarks it, tears it down, then prints a
# Metric | Baseline | Optimized | Delta table. No manual steps.
#
# Usage: ./run_benchmark.sh [baseline|optimized|both] (default: both)
# Env overrides: MAX_CONCURRENCY, SEED, SERVER_READY_TIMEOUT
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODEL="zai-org/GLM-5.2"
HOST="127.0.0.1"
PORT="8000"
MAX_CONCURRENCY="${MAX_CONCURRENCY:-512}"
SEED="${SEED:-0}"
SERVER_READY_TIMEOUT="${SERVER_READY_TIMEOUT:-3600}"
MODE="${1:-both}"
CURRENT_SERVER_PID=""
stop_current_server() {
[ -n "$CURRENT_SERVER_PID" ] || return 0
kill -TERM -- "-$CURRENT_SERVER_PID" 2>/dev/null || kill -TERM "$CURRENT_SERVER_PID" 2>/dev/null || true
for _ in $(seq 1 30); do
kill -0 "$CURRENT_SERVER_PID" 2>/dev/null || { CURRENT_SERVER_PID=""; return 0; }
sleep 1
done
kill -KILL -- "-$CURRENT_SERVER_PID" 2>/dev/null || kill -KILL "$CURRENT_SERVER_PID" 2>/dev/null || true
CURRENT_SERVER_PID=""
}
trap stop_current_server EXIT INT TERM
wait_for_server() {
python3 - "$HOST" "$PORT" "$SERVER_READY_TIMEOUT" <<'PY'
import sys, time, urllib.request
host, port, timeout = sys.argv[1], sys.argv[2], int(sys.argv[3])
url = f"http://{host}:{port}/health"
deadline = time.time() + timeout
while time.time() < deadline:
try:
urllib.request.urlopen(url, timeout=5)
sys.exit(0)
except Exception:
time.sleep(10)
sys.exit(1)
PY
}
run_one() {
local tag="$1"
echo "=== [$tag] starting server (log: server_${tag}.log) ==="
setsid "$SCRIPT_DIR/launch_server.sh" "$tag" >"server_${tag}.log" 2>&1 &
CURRENT_SERVER_PID=$!
echo "=== [$tag] waiting for server (up to ${SERVER_READY_TIMEOUT}s) ==="
if ! wait_for_server; then
echo "[$tag] server did not become healthy in time -- see server_${tag}.log" >&2
stop_current_server
exit 1
fi
echo "=== [$tag] benchmarking ==="
local args=(
--backend openai
--host "$HOST" --port "$PORT"
--model "$MODEL"
--dataset-name random
--random-input-len 2048
--random-output-len 256
--random-range-ratio 0.8
--request-rate inf
--max-concurrency "$MAX_CONCURRENCY"
--seed "$SEED"
)
vllm bench serve "${args[@]}" --num-prompts 10 >/dev/null # warm-up, discarded
vllm bench serve "${args[@]}" --num-prompts 1000 \
--save-result --result-filename "benchmark_${tag}.json"
echo "=== [$tag] stopping server ==="
stop_current_server
}
case "$MODE" in
baseline|optimized) run_one "$MODE" ;;
both) run_one baseline; run_one optimized ;;
*)
echo "Usage: $0 [baseline|optimized|both]" >&2
exit 1
;;
esac
python3 - "$MODE" <<'PY'
import json, os, sys
mode = sys.argv[1]
tags = ["baseline", "optimized"] if mode == "both" else [mode]
data = {}
for tag in tags:
path = f"benchmark_{tag}.json"
if os.path.exists(path):
with open(path) as f:
data[tag] = json.load(f)
metrics = [
("Request throughput (req/s)", "request_throughput"),
("Output tok/s", "output_throughput"),
("Total tok/s", "total_token_throughput"),
("TTFT mean (ms)", "mean_ttft_ms"),
("TTFT median (ms)", "median_ttft_ms"),
("TTFT p99 (ms)", "p99_ttft_ms"),
("TPOT mean (ms)", "mean_tpot_ms"),
("TPOT median (ms)", "median_tpot_ms"),
("TPOT p99 (ms)", "p99_tpot_ms"),
("ITL mean (ms)", "mean_itl_ms"),
("ITL median (ms)", "median_itl_ms"),
("ITL p99 (ms)", "p99_itl_ms"),
("Duration (s)", "duration"),
]
def fmt(v):
return f"{v:.1f}" if isinstance(v, (int, float)) else "n/a"
print()
if mode == "both":
print("| Metric | Baseline | Optimized | Delta |")
print("|---|---|---|---|")
for name, key in metrics:
b, o = data["baseline"].get(key), data["optimized"].get(key)
delta = f"{(o - b) / b * 100:+.1f}%" if isinstance(b, (int, float)) and isinstance(o, (int, float)) and b else "n/a"
print(f"| {name} | {fmt(b)} | {fmt(o)} | {delta} |")
else:
print(f"| Metric | {mode.capitalize()} |")
print("|---|---|")
for name, key in metrics:
print(f"| {name} | {fmt(data[mode].get(key))} |")
PY
|