File size: 3,009 Bytes
350d99c 48f42a4 350d99c 48f42a4 350d99c 48f42a4 350d99c 48f42a4 350d99c d861a68 350d99c d861a68 350d99c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | #!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
QWEN38="$ROOT/models/Qwen3.8-27B-Q6_K.gguf"
QWEN36="$ROOT/models/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf"
[[ -f "$QWEN38" && "$(stat -c %s "$QWEN38")" == 22563871392 ]] || { echo "Missing or incomplete: $QWEN38" >&2; exit 1; }
[[ -f "$QWEN36" && "$(stat -c %s "$QWEN36")" == 26456194016 ]] || { echo "Missing or incomplete: $QWEN36" >&2; exit 1; }
BIN="$ROOT/runtime/b11177/llama-server"
[[ -x "$BIN" ]] || BIN="$ROOT/runtime/b11177/build/bin/llama-server"
[[ -x "$BIN" ]] || { echo 'llama-server missing. Run bash scripts/install.sh.' >&2; exit 1; }
mkdir -p "$ROOT/secrets" "$ROOT/runtime"
KEY_FILE="$ROOT/secrets/api_keys.txt"
if [[ ! -s "$KEY_FILE" ]]; then
umask 077
python -c 'import secrets; print(secrets.token_urlsafe(32))' > "$KEY_FILE"
fi
chmod 600 "$KEY_FILE"
PRESET="$ROOT/runtime/models.ini"
cat > "$PRESET" <<EOF
version = 1
[*]
n-gpu-layers = all
fit = off
flash-attn = on
cache-type-k = q8_0
cache-type-v = q8_0
[qwen38]
model = $QWEN38
ctx-size = 65536
parallel = 4
[qwen36]
model = $QWEN36
ctx-size = 65536
parallel = 4
EOF
GLM_Q8="$ROOT/models/GLM-4.7-Flash-Q8_0.gguf"
if [[ -f "$GLM_Q8" && "$(stat -c %s "$GLM_Q8")" == 31842799232 ]]; then
cat >> "$PRESET" <<EOF
[glm47q8]
model = $GLM_Q8
ctx-size = 32768
parallel = 2
EOF
else
echo "Optional model glm47q8 not installed yet; expected models/GLM-4.7-Flash-Q8_0.gguf (31,842,799,232 bytes)." >&2
fi
GPTOSS_1="$ROOT/models/gpt-oss-120b-MXFP4-00001-of-00002.gguf"
GPTOSS_2="$ROOT/models/gpt-oss-120b-MXFP4-00002-of-00002.gguf"
if [[ -f "$GPTOSS_1" && -f "$GPTOSS_2" \
&& "$(stat -c %s "$GPTOSS_1")" == 39815566336 \
&& "$(stat -c %s "$GPTOSS_2")" == 23571779104 ]]; then
cat >> "$PRESET" <<EOF
[gptoss120]
model = $GPTOSS_1
ctx-size = 16384
parallel = 1
cpu-moe = true
EOF
else
echo "Optional model gptoss120 not installed yet; expected both MXFP4 shards in models/." >&2
fi
export LD_LIBRARY_PATH="$ROOT/runtime/b11177:/opt/conda/lib:${LD_LIBRARY_PATH:-}"
# llama.cpp b11177 otherwise defaults its HTTP thread pool to visible CPUs.
# Keep nested/optional library pools small; llama.cpp owns the main compute pool.
export OMP_NUM_THREADS="${LLM_OMP_THREADS:-4}"
export OMP_THREAD_LIMIT="${LLM_OMP_THREADS:-4}"
export OMP_DYNAMIC=FALSE
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
export BLIS_NUM_THREADS=1
export VECLIB_MAXIMUM_THREADS=1
CPU_SET="$(python -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0))[:32])))')"
exec taskset -c "$CPU_SET" "$BIN" \
--models-preset "$PRESET" \
--models-max 1 \
--models-autoload \
--sleep-idle-seconds "${LLM_IDLE_SECONDS:-900}" \
--host "${LLM_HOST:-127.0.0.1}" \
--port "${LLM_PORT:-8080}" \
--api-key-file "$KEY_FILE" \
--threads 16 \
--threads-batch 16 \
--threads-http 4 \
--cont-batching \
--reasoning-effort "${LLM_REASONING_EFFORT:-medium}" \
--batch-size 2048 \
--ubatch-size 512 \
--jinja
|