File size: 2,303 Bytes
42b02cd d861a68 42b02cd d861a68 42b02cd d861a68 350d99c 42b02cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | #!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODEL="${1:-qwen38}"
case "$MODEL" in
qwen38) MODEL_PATH="$ROOT/models/Qwen3.8-27B-Q6_K.gguf"; EXPECTED_SIZE=22563871392 ;;
qwen36) MODEL_PATH="$ROOT/models/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf"; EXPECTED_SIZE=26456194016 ;;
*) echo "Usage: $0 [qwen38|qwen36]" >&2; exit 2 ;;
esac
[[ -f "$MODEL_PATH" && "$(stat -c %s "$MODEL_PATH")" == "$EXPECTED_SIZE" ]] || {
echo "Model missing or incomplete: $MODEL_PATH" >&2
exit 1
}
BIN="$ROOT/runtime/b11177/llama-server"
[[ -x "$BIN" ]] || BIN="$ROOT/runtime/b11177/build/bin/llama-server"
[[ -x "$BIN" ]] || { echo 'llama-server missing. Run scripts/install.sh.' >&2; exit 1; }
mkdir -p "$ROOT/secrets"
KEY_FILE="$ROOT/secrets/api_keys.txt"
if [[ ! -f "$KEY_FILE" ]]; then
umask 077
python -c 'import secrets; print(secrets.token_urlsafe(32))' > "$KEY_FILE"
fi
export LD_LIBRARY_PATH="$ROOT/runtime/b11177:/opt/conda/lib:${LD_LIBRARY_PATH:-}"
export OMP_NUM_THREADS=4
export OMP_THREAD_LIMIT=4
export OMP_DYNAMIC=FALSE
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
export BLIS_NUM_THREADS=1
export VECLIB_MAXIMUM_THREADS=1
CPU_SET="$(python -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0))[:32])))')"
SPEC_ARGS=()
if [[ "${LLM_SPEC_MTP:-0}" == 1 ]]; then
[[ "$MODEL" == qwen38 ]] || { echo 'MTP draft is only configured for qwen38.' >&2; exit 1; }
DRAFT="$ROOT/models/mtp-Qwen3.8-27B-Q8_0.gguf"
[[ -f "$DRAFT" && "$(stat -c %s "$DRAFT")" == 3164006816 ]] || {
echo 'MTP draft missing. Run WITH_MTP=1 bash scripts/install.sh.' >&2
exit 1
}
SPEC_ARGS=(--spec-type draft-mtp --model-draft "$DRAFT" --spec-draft-n-max 2)
fi
exec taskset -c "$CPU_SET" "$BIN" \
--model "$MODEL_PATH" \
--alias "$MODEL" \
--host "${LLM_HOST:-127.0.0.1}" \
--port "${LLM_PORT:-8080}" \
--api-key-file "$KEY_FILE" \
--n-gpu-layers all \
--fit off \
--flash-attn on \
--cache-type-k q8_0 \
--cache-type-v q8_0 \
--ctx-size "${LLM_CONTEXT:-65536}" \
--parallel 4 \
--threads 16 \
--threads-batch 16 \
--threads-http 4 \
--cont-batching \
--reasoning-effort "${LLM_REASONING_EFFORT:-medium}" \
--batch-size 2048 \
--ubatch-size 512 \
--jinja \
"${SPEC_ARGS[@]}"
|