scripts/router.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
QWEN38="$ROOT/models/Qwen3.8-27B-Q6_K.gguf"
QWEN36="$ROOT/models/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf"
[[ -f "$QWEN38" && "$(stat -c %s "$QWEN38")" == 22563871392 ]] || { echo "Missing or incomplete: $QWEN38" >&2; exit 1; }
[[ -f "$QWEN36" && "$(stat -c %s "$QWEN36")" == 26456194016 ]] || { echo "Missing or incomplete: $QWEN36" >&2; exit 1; }

BIN="$ROOT/runtime/b11177/llama-server"
[[ -x "$BIN" ]] || BIN="$ROOT/runtime/b11177/build/bin/llama-server"
[[ -x "$BIN" ]] || { echo 'llama-server missing. Run bash scripts/install.sh.' >&2; exit 1; }

mkdir -p "$ROOT/secrets" "$ROOT/runtime"
KEY_FILE="$ROOT/secrets/api_keys.txt"
if [[ ! -s "$KEY_FILE" ]]; then
  umask 077
  python -c 'import secrets; print(secrets.token_urlsafe(32))' > "$KEY_FILE"
fi
chmod 600 "$KEY_FILE"

PRESET="$ROOT/runtime/models.ini"
cat > "$PRESET" <<EOF
version = 1

[*]
n-gpu-layers = all
fit = off
flash-attn = on
cache-type-k = q8_0
cache-type-v = q8_0

[qwen38]
model = $QWEN38
ctx-size = 65536
parallel = 4

[qwen36]
model = $QWEN36
ctx-size = 65536
parallel = 4
EOF

GLM_Q8="$ROOT/models/GLM-4.7-Flash-Q8_0.gguf"
if [[ -f "$GLM_Q8" && "$(stat -c %s "$GLM_Q8")" == 31842799232 ]]; then
  cat >> "$PRESET" <<EOF

[glm47q8]
model = $GLM_Q8
ctx-size = 32768
parallel = 2
EOF
else
  echo "Optional model glm47q8 not installed yet; expected models/GLM-4.7-Flash-Q8_0.gguf (31,842,799,232 bytes)." >&2
fi

GPTOSS_1="$ROOT/models/gpt-oss-120b-MXFP4-00001-of-00002.gguf"
GPTOSS_2="$ROOT/models/gpt-oss-120b-MXFP4-00002-of-00002.gguf"
if [[ -f "$GPTOSS_1" && -f "$GPTOSS_2" \
   && "$(stat -c %s "$GPTOSS_1")" == 39815566336 \
   && "$(stat -c %s "$GPTOSS_2")" == 23571779104 ]]; then
  cat >> "$PRESET" <<EOF

[gptoss120]
model = $GPTOSS_1
ctx-size = 16384
parallel = 1
cpu-moe = true
EOF
else
  echo "Optional model gptoss120 not installed yet; expected both MXFP4 shards in models/." >&2
fi

export LD_LIBRARY_PATH="$ROOT/runtime/b11177:/opt/conda/lib:${LD_LIBRARY_PATH:-}"
# llama.cpp b11177 otherwise defaults its HTTP thread pool to visible CPUs.
# Keep nested/optional library pools small; llama.cpp owns the main compute pool.
export OMP_NUM_THREADS="${LLM_OMP_THREADS:-4}"
export OMP_THREAD_LIMIT="${LLM_OMP_THREADS:-4}"
export OMP_DYNAMIC=FALSE
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
export BLIS_NUM_THREADS=1
export VECLIB_MAXIMUM_THREADS=1
CPU_SET="$(python -c 'import os; print(",".join(map(str, sorted(os.sched_getaffinity(0))[:32])))')"
exec taskset -c "$CPU_SET" "$BIN" \
  --models-preset "$PRESET" \
  --models-max 1 \
  --models-autoload \
  --sleep-idle-seconds "${LLM_IDLE_SECONDS:-900}" \
  --host "${LLM_HOST:-127.0.0.1}" \
  --port "${LLM_PORT:-8080}" \
  --api-key-file "$KEY_FILE" \
  --threads 16 \
  --threads-batch 16 \
  --threads-http 4 \
  --cont-batching \
  --reasoning-effort "${LLM_REASONING_EFFORT:-medium}" \
  --batch-size 2048 \
  --ubatch-size 512 \
  --jinja