Merge llama-swap branch into main

.env.example

@@ -1,20 +1,56 @@
+# Copy this file to .env and replace the placeholder values.
+# Keep .env private: it may contain access tokens and API keys.
+
+# -----------------------------------------------------------------------------
+# Paths
+# -----------------------------------------------------------------------------
 MODELS_ROOT="/path/to/MODELS"
-PRESETS_DIR="${MODELS_ROOT}/MODELS/presets"
-LLAMA_DIR="${MODELS_ROOT}/MODELS/llama.cpp"
-MODELS_DIR="${MODELS_ROOT}/MODELS/models"
-RESULTS_DIR="${MODELS_ROOT}/MODELS/presets/bench-results"
-BENCH_STATE_DIR="${MODELS_ROOT}/MODELS/presets/.bench-state"
+PRESETS_DIR="${MODELS_ROOT}/presets"
+LLAMA_DIR="${MODELS_ROOT}/llama.cpp"
+MODELS_DIR="${MODELS_ROOT}/models"
+
+# -----------------------------------------------------------------------------
+# CUDA and Docker images
+# -----------------------------------------------------------------------------
 CUDA_IMAGE="nvidia/cuda:12.9.1-devel-ubuntu24.04"
 BUILDER_IMAGE="llama-cuda12-builder:local"
+CUDA_BUILD="12"
+CUDA_ARCHITECTURES="52;75"
+CUDA_FLAGS="-Wno-deprecated-gpu-targets"
+
+# -----------------------------------------------------------------------------
+# Container names
+# -----------------------------------------------------------------------------
 BUILD_CONTAINER_NAME="llama-build"
 SERVER_CONTAINER_NAME="llama-server"
 BENCH_CONTAINER_PREFIX="llama-bench"
+LLAMA_SWAP_CONTAINER_NAME="llama-swap"
+
+# -----------------------------------------------------------------------------
+# llama.cpp server and RPC server
+# -----------------------------------------------------------------------------
 SERVER_HOST="0.0.0.0"
 SERVER_PORT="8080"
-CUDA_ARCHITECTURES="52;75"
-CUDA_FLAGS="-Wno-deprecated-gpu-targets"
-BENCH_REPETITIONS="3"
 SERVER_INTERNAL_PORT="8080"
-RPC_HOSTS="HOST:PORT,HOST:PORT"
+RPC_HOSTS="HOST"
+RPC_PORT="50052"
+
+# -----------------------------------------------------------------------------
+# Authentication and Hugging Face access
+# -----------------------------------------------------------------------------
 HF_TOKEN=""
-LLAMA_API_KEY=""
+LLAMA_API_KEY=""
+
+# -----------------------------------------------------------------------------
+# llama-swap gateway
+# -----------------------------------------------------------------------------
+LLAMA_SWAP_VERSION="unified-cuda"
+LLAMA_SWAP_IMAGE="ghcr.io/mostlygeek/llama-swap:${LLAMA_SWAP_VERSION}"
+LLAMA_SWAP_PORT="8080"
+LLAMA_SWAP_API_KEY="${LLAMA_API_KEY}"
+
+# -----------------------------------------------------------------------------
+# Service lifecycle
+# -----------------------------------------------------------------------------
+# Graceful Docker stop timeout used by server.sh and llama-swap.sh.
+STOP_TIMEOUT="30"

.gitignore

@@ -1,5 +1,7 @@
 __pycache__/
 *.log
+llama-swap-logs/
+llama-swap-state/
 
 .env
 .bench-state/

README.md

@@ -103,24 +103,116 @@ The image includes `libssl-dev`, so llama.cpp can download Hugging Face models o
 ### Server
 
 ```bash
-./server.sh                 # Start llama.cpp server with preset.ini
+./server.sh                 # Start in the background with preset.ini
+./server.sh start            # Same as above
+./server.sh start custom.ini
+./server.sh status
+./server.sh restart custom.ini
+./server.sh stop
 ```
 
+`start` and `daemon` launch the Docker container in detached mode, so the
+script returns while the server continues running in the background. The
+container name is used for lifecycle tracking; no host PID file is needed.
+The legacy form `./server.sh custom.ini` is also supported.
+
 `server.sh` uses the prebuilt CUDA 12.9 image, exposes port `8080`, and serves both GPUs. From another computer:
 
 ```text
 http://<host-ip>:8080
 ```
 
-The server runs in router mode and loads models on demand. Set an API key before exposing it beyond a trusted LAN; current presets intentionally have no key configured.
+The server runs in router mode and loads models on demand. Set `LLAMA_API_KEY`
+before exposing the legacy server beyond a trusted LAN.
+
+### llama-swap gateway
+
+`llama-swap.sh` runs the five active models from `llama-swap.yaml` through the
+custom CUDA build, swapping the corresponding `llama-server` process on demand:
+
+- `Qwen3.6-35B-MTP`
+- `Ornith-1.5-35B-A3B-MTP`
+- `gemma-4-12B-it-qat`
+- `gemma-4-26B-A4B-it-qat-MTP`
+- `Qwopus3.6-35B-A3B-Coder-MTP`
+
+On the target server, from the presets directory configured in `.env`:
+
+```bash
+./llama-swap.sh start
+./llama-swap.sh status
+./llama-swap.sh restart
+./llama-swap.sh stop
+```
+
+`./llama-swap.sh` and `./llama-swap.sh daemon` are aliases for `start`. The
+gateway runs as a detached Docker container and remains active after the
+shell script exits.
+
+The gateway is available at `http://<server-host>:<port>/v1`. Its API key is read
+from `LLAMA_API_KEY`; the key is passed to llama-swap only, while the upstream
+llama-server processes remain bound to the container loopback interface. Each
+model receives an available internal port from llama-swap, and the proxy uses
+that same assigned port.
+
+Authentication is API-key based; there is no separate llama-swap username or
+password. The web UI and API endpoints require the key:
+
+- In a browser prompt, enter any non-empty username and use the value of
+  `LLAMA_API_KEY` as the password.
+- For OpenAI-compatible clients, set the base URL to
+  `http://<server-host>:<port>/v1` and use the same value as the API key. The client
+  should send it as `Authorization: Bearer <API key>`.
+- `http://<server-host>:<port>/health` is unauthenticated and returns `OK`.
+
+The derived single-model presets are in `llama-swap-presets/`, and model logs
+are written to `${PRESETS_DIR}/llama-swap-logs/`. Activity history and metrics
+are stored in `${PRESETS_DIR}/llama-swap-state/activity.sqlite`, so the
+llama-swap Activity page retains previous requests and runs across restarts.
+
+The state directory is created automatically by `llama-swap.sh` and is mounted
+into the container through the existing `/presets` volume. The currently loaded
+model process is still intentionally restarted and must be loaded again after a
+container restart; only historical activity and metrics are persisted.
+
+Models are automatically unloaded after 30 minutes of inactivity because
+`globalTTL` is set to `1800` seconds. Increase this value to reduce reloads, or
+set it to `0` to keep the active model loaded indefinitely.
+
+The gateway is configured with an exclusive `single-model` group. The Pi agent
+selects the model by sending its exact model ID in each OpenAI-compatible
+request. When that ID changes, llama-swap unloads the current model before
+starting the requested one; the Pi agent must not call `/models/sse`.
+
+To discover the available IDs, call `GET /v1/models` with the API key. A chat
+request should use the selected ID in its `model` field:
+
+```json
+{
+  "model": "Qwen3.6-35B-MTP",
+  "messages": [
+    {"role": "user", "content": "Hello"}
+  ]
+}
+```
 
 ## Quick Start
 
 ```bash
-cd /mnt/MODELS/presets
+cd <presets-directory>
 ./create-build-image.sh
 ./build-llama.cpp.sh
-./server.sh
+./server.sh start
+./server.sh status
+```
+
+Then use the OpenAI-compatible API on port 8080. Run `server.sh` and
+`llama-swap.sh` separately; they both use port `8080` by default. To use the
+gateway instead, stop the standalone server and run:
+
+```bash
+./server.sh stop
+./llama-swap.sh start
 ```
 
-Then use the OpenAI-compatible API on port 8080.
+Then use the authenticated gateway at `http://<server-host>:<port>/v1`.

docker-compose.yml

@@ -20,10 +20,10 @@ services:
       bash -c '
         set -euo pipefail
         git config --global --add safe.directory /src
-        cmake -S /src -B /src/build-${CUDA_BUILD} \
+        cmake -S /src -B /src/build-${CUDA_BUILD:-12} \
           -DGGML_CUDA=ON -DGGML_CUDA_GRAPH=ON -DGGML_VULKAN=ON \
           -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHITECTURES}" \
           -DCMAKE_CUDA_FLAGS="${CUDA_FLAGS}" \
           -DBUILD_SHARED_LIBS=OFF -DGGML_RPC=ON
-        cmake --build /src/build-${CUDA_BUILD} --config Release -j$$(nproc)
+        cmake --build /src/build-${CUDA_BUILD:-12} --config Release -j$$(nproc)
       '

llama-swap-presets/gemma-4-12b-it-qat.ini

@@ -0,0 +1,24 @@
+version = 1
+
+[*]
+fa = on
+jinja = true
+metrics = false
+reasoning = on
+reasoning-budget-message = "OK, I've thought long enough. Let's answer."
+np = 1
+c = 131072
+image-min-tokens = 1024
+device = CUDA0,CUDA1
+split-mode = layer
+main-gpu = 0
+device-draft = CUDA1
+mmproj-device = CUDA1
+models-max = 1
+
+[gemma-4-12B-it-qat]
+hf = unsloth/gemma-4-12B-it-qat-GGUF:UD-Q4_K_XL
+spec-type = draft-mtp
+spec-draft-n-max = 2
+load-mode = none
+lazy-mode = auto

llama-swap-presets/gemma-4-26b-a4b-it-qat-mtp.ini

@@ -0,0 +1,24 @@
+version = 1
+
+[*]
+fa = on
+jinja = true
+metrics = false
+reasoning = on
+reasoning-budget-message = "OK, I've thought long enough. Let's answer."
+np = 1
+c = 131072
+image-min-tokens = 1024
+device = CUDA0,CUDA1
+split-mode = layer
+main-gpu = 0
+device-draft = CUDA1
+mmproj-device = CUDA1
+models-max = 1
+
+[gemma-4-26B-A4B-it-qat-MTP]
+hf = unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL
+spec-type = draft-mtp
+spec-draft-n-max = 2
+load-mode = none
+lazy-mode = auto

llama-swap-presets/ornith-1.5-35b-a3b-mtp.ini

@@ -0,0 +1,33 @@
+version = 1
+
+[*]
+fa = on
+jinja = true
+metrics = false
+reasoning = on
+reasoning-budget-message = "OK, I've thought long enough. Let's answer."
+np = 1
+c = 131072
+image-min-tokens = 1024
+device = CUDA0,CUDA1
+split-mode = layer
+main-gpu = 0
+device-draft = CUDA1
+mmproj-device = CUDA1
+models-max = 1
+
+[Ornith-1.5-35B-A3B-MTP]
+hf = ornith-ai/Ornith-1.5-35B-A3B-GGUF:Q4_K_M
+temp = 0.85
+top-p = 0.95
+top-k = 40
+min-p = 0.05
+repeat-penalty = 1.1
+presence-penalty = 0.0
+spec-type = draft-mtp
+spec-draft-n-max = 2
+load-mode = none
+lazy-mode = auto
+device = CUDA0
+split-mode = layer
+main-gpu = 0

llama-swap-presets/qwen3.6-35b-mtp.ini

@@ -0,0 +1,33 @@
+version = 1
+
+[*]
+fa = on
+jinja = true
+metrics = false
+reasoning = on
+reasoning-budget-message = "OK, I've thought long enough. Let's answer."
+np = 1
+c = 131072
+image-min-tokens = 1024
+device = CUDA0,CUDA1
+split-mode = layer
+main-gpu = 0
+device-draft = CUDA1
+mmproj-device = CUDA1
+models-max = 1
+
+[Qwen3.6-35B-MTP]
+hf = unsloth/Qwen3.6-35B-A3B-MTP-GGUF:UD-Q4_K_XL
+temp = 0.6
+top-p = 0.95
+top-k = 20
+presence-penalty = 0.0
+min-p = 0.00
+repeat-penalty = 1.0
+spec-type = draft-mtp
+spec-draft-n-max = 2
+device = CUDA0
+split-mode = layer
+main-gpu = 0
+load-mode = none
+lazy-mode = auto

llama-swap-presets/qwopus3.5-35b-mtp.ini

@@ -0,0 +1,33 @@
+version = 1
+
+[*]
+fa = on
+jinja = true
+metrics = false
+reasoning = on
+reasoning-budget-message = "OK, I've thought long enough. Let's answer."
+np = 1
+c = 131072
+image-min-tokens = 1024
+device = CUDA0,CUDA1
+split-mode = layer
+main-gpu = 0
+device-draft = CUDA1
+mmproj-device = CUDA1
+models-max = 1
+
+[Qwopus3.6-35B-A3B-Coder-MTP-GGUF]
+hf = Jackrong/Qwopus3.6-35B-A3B-Coder-MTP-GGUF:Q4_K_M
+temp = 0.6
+top-p = 0.95
+top-k = 20
+presence-penalty = 0.0
+min-p = 0.00
+repeat-penalty = 1.0
+spec-type = draft-mtp
+spec-draft-n-max = 2
+device = CUDA0
+split-mode = layer
+main-gpu = 0
+load-mode = none
+lazy-mode = auto

llama-swap.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${SCRIPT_DIR}/.env"
+if [[ ! -f "${ENV_FILE}" ]]; then
+  printf 'Missing environment file: %s\n' "${ENV_FILE}" >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "${ENV_FILE}"
+set +a
+
+CUDA_BUILD="${CUDA_BUILD:-12}"
+DOCKER=(sudo docker)
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [command]
+
+Commands:
+  start           Start llama-swap in the background (default)
+  daemon          Alias for start
+  stop            Gracefully stop llama-swap
+  restart         Stop and start llama-swap
+  status          Show the Docker container state and published port
+EOF
+}
+
+container_state() {
+  local state
+  if state="$("${DOCKER[@]}" inspect --format='{{.State.Status}}' "${LLAMA_SWAP_CONTAINER_NAME}" 2>/dev/null)"; then
+    printf '%s\n' "${state}"
+  else
+    printf 'missing\n'
+  fi
+}
+
+remove_stale_container() {
+  if [[ "$(container_state)" != "missing" ]]; then
+    "${DOCKER[@]}" rm --force "${LLAMA_SWAP_CONTAINER_NAME}" >/dev/null
+  fi
+}
+
+start_llama_swap() {
+  : "${MODELS_ROOT:?MODELS_ROOT must be set in .env}"
+  : "${LLAMA_DIR:?LLAMA_DIR must be set in .env}"
+  : "${PRESETS_DIR:?PRESETS_DIR must be set in .env}"
+  : "${MODELS_DIR:?MODELS_DIR must be set in .env}"
+  : "${BUILDER_IMAGE:?BUILDER_IMAGE must be set in .env}"
+  : "${LLAMA_SWAP_CONTAINER_NAME:?LLAMA_SWAP_CONTAINER_NAME must be set in .env}"
+  : "${LLAMA_SWAP_PORT:?LLAMA_SWAP_PORT must be set in .env}"
+  : "${LLAMA_SWAP_API_KEY:?LLAMA_SWAP_API_KEY must be set in .env}"
+  : "${LLAMA_SWAP_IMAGE:?LLAMA_SWAP_IMAGE must be set in .env}"
+
+  if [[ ! -f "${PRESETS_DIR}/llama-swap.yaml" ]]; then
+    printf 'Missing llama-swap configuration: %s\n' "${PRESETS_DIR}/llama-swap.yaml" >&2
+    return 1
+  fi
+
+  if [[ ! -x "${LLAMA_DIR}/build-${CUDA_BUILD}/bin/llama-server" ]]; then
+    printf 'Missing custom llama-server: %s\n' "${LLAMA_DIR}/build-${CUDA_BUILD}/bin/llama-server" >&2
+    printf 'Build llama.cpp before starting llama-swap.\n' >&2
+    return 1
+  fi
+
+  if [[ "$(container_state)" == "running" ]]; then
+    printf '%s is already running\n' "${LLAMA_SWAP_CONTAINER_NAME}" >&2
+    return 1
+  fi
+
+  mkdir -p "${PRESETS_DIR}/llama-swap-logs" "${PRESETS_DIR}/llama-swap-state"
+  remove_stale_container
+
+  "${DOCKER[@]}" run --detach --rm \
+    --name "${LLAMA_SWAP_CONTAINER_NAME}" \
+    --gpus all \
+    --device=/dev/dri \
+    --ulimit memlock=-1:-1 \
+    --publish "${LLAMA_SWAP_PORT}:8080" \
+    --env "CUDA_BUILD=${CUDA_BUILD}" \
+    --env "HF_HOME=/models" \
+    --env "HF_TOKEN=${HF_TOKEN:-}" \
+    --env "LLAMA_SWAP_API_KEY=${LLAMA_SWAP_API_KEY}" \
+    --env "LLAMA_SWAP_CONFIG=/presets/llama-swap.yaml" \
+    --volume "${LLAMA_DIR}:/src:ro" \
+    --volume "${PRESETS_DIR}:/presets" \
+    --volume "${MODELS_DIR}:/models" \
+    --workdir /models \
+    "${LLAMA_SWAP_IMAGE}"
+  printf 'Started %s in the background\n' "${LLAMA_SWAP_CONTAINER_NAME}"
+}
+
+stop_llama_swap() {
+  local state timeout
+  state="$(container_state)"
+  timeout="${STOP_TIMEOUT:-30}"
+
+  if [[ "${state}" == "missing" ]]; then
+    printf '%s is not running\n' "${LLAMA_SWAP_CONTAINER_NAME}"
+    return 0
+  fi
+
+  if [[ "${state}" == "running" || "${state}" == "created" || "${state}" == "restarting" ]]; then
+    "${DOCKER[@]}" stop --time="${timeout}" "${LLAMA_SWAP_CONTAINER_NAME}" >/dev/null
+  else
+    remove_stale_container
+  fi
+  printf 'Stopped %s\n' "${LLAMA_SWAP_CONTAINER_NAME}"
+}
+
+status_llama_swap() {
+  local state port
+  state="$(container_state)"
+  if [[ "${state}" == "missing" ]]; then
+    printf '%s: stopped (container missing)\n' "${LLAMA_SWAP_CONTAINER_NAME}"
+    return 1
+  fi
+
+  port="$("${DOCKER[@]}" port "${LLAMA_SWAP_CONTAINER_NAME}" 8080/tcp 2>/dev/null || true)"
+  if [[ -n "${port}" ]]; then
+    printf '%s: %s (%s)\n' "${LLAMA_SWAP_CONTAINER_NAME}" "${state}" "${port}"
+  else
+    printf '%s: %s\n' "${LLAMA_SWAP_CONTAINER_NAME}" "${state}"
+  fi
+  [[ "${state}" == "running" ]]
+}
+
+if [[ $# -gt 0 ]]; then
+  COMMAND="$1"
+else
+  COMMAND="start"
+fi
+case "${COMMAND}" in
+  start|daemon|restart)
+    if [[ $# -gt 1 ]]; then
+      printf 'Unexpected argument for %s: %s\n' "${COMMAND}" "$2" >&2
+      usage >&2
+      exit 2
+    fi
+    ;;
+  stop|status)
+    if [[ $# -gt 1 ]]; then
+      printf 'Unexpected argument for %s: %s\n' "${COMMAND}" "$2" >&2
+      usage >&2
+      exit 2
+    fi
+    ;;
+  help|-h|--help)
+    usage
+    exit 0
+    ;;
+  *)
+    printf 'Unknown command: %s\n' "${COMMAND}" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+: "${LLAMA_SWAP_CONTAINER_NAME:?LLAMA_SWAP_CONTAINER_NAME must be set in .env}"
+case "${COMMAND}" in
+  start|daemon)
+    start_llama_swap
+    ;;
+  stop)
+    stop_llama_swap
+    ;;
+  restart)
+    stop_llama_swap
+    start_llama_swap
+    ;;
+  status)
+    status_llama_swap
+    ;;
+esac

llama-swap.yaml

@@ -0,0 +1,106 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
+# The API key is injected by llama-swap.sh and is intentionally not stored here.
+healthCheckTimeout: 1800
+logLevel: info
+# Store the Activity page's request history and metrics in a file-backed SQLite
+# database so they survive llama-swap/container restarts.
+store:
+  path: /presets/llama-swap-state/activity.sqlite
+# Keep the active model warm for 30 minutes, then reclaim its VRAM when idle.
+globalTTL: 1800
+unloadTimeout: 30
+globalConcurrencyLimit: 1
+sendLoadingState: true
+logToStdout: proxy
+compat:
+  # ignoreWebsockets: do not let websocket connections load a stopped model
+  # or count toward concurrency, TTL activity, or swap decisions
+  # - optional, default: false
+  ignoreWebsockets: true
+apiKeys:
+  - "${env.LLAMA_SWAP_API_KEY}"
+
+# Shared command fragments keep model definitions consistent. The environment
+# macro is expanded by llama-swap inside the container, and ${PORT} is assigned
+# separately for each model.
+macros:
+  llama_server: "/src/build-${env.CUDA_BUILD}/bin/llama-server"
+  llama_server_args: >-
+    --host 127.0.0.1
+    --port ${PORT}
+
+groups:
+  single-model:
+    swap: true
+    exclusive: true
+    members:
+      - "Qwen3.6-35B-MTP"
+      - "Ornith-1.5-35B-A3B-MTP"
+      - "gemma-4-12B-it-qat"
+      - "gemma-4-26B-A4B-it-qat-MTP"
+      - "Qwopus3.6-35B-A3B-Coder-MTP"
+
+models:
+  "Qwen3.6-35B-MTP":
+    cmd: |
+      ${llama_server}
+      ${llama_server_args}
+      --models-preset /presets/llama-swap-presets/qwen3.6-35b-mtp.ini
+      --log-file /presets/llama-swap-logs/qwen3.6-35b-mtp.log
+    proxy: http://127.0.0.1:${PORT}
+    checkEndpoint: /health
+    concurrencyLimit: 0
+    capabilities:
+      tools: true
+      disableAuto: false
+
+  "Qwopus3.6-35B-A3B-Coder-MTP":
+    cmd: |
+      ${llama_server}
+      ${llama_server_args}
+      --models-preset /presets/llama-swap-presets/qwopus3.6-35b-mtp.ini
+      --log-file /presets/llama-swap-logs/qwopus3.6-35b-mtp.log
+    proxy: http://127.0.0.1:${PORT}
+    checkEndpoint: /health
+    concurrencyLimit: 0
+    capabilities:
+      tools: true
+      disableAuto: false
+
+  "Ornith-1.5-35B-A3B-MTP":
+    cmd: |
+      ${llama_server}
+      ${llama_server_args}
+      --models-preset /presets/llama-swap-presets/ornith-1.5-35b-a3b-mtp.ini
+      --log-file /presets/llama-swap-logs/ornith-1.5-35b-a3b-mtp.log
+    proxy: http://127.0.0.1:${PORT}
+    checkEndpoint: /health
+    concurrencyLimit: 0
+    capabilities:
+      tools: true
+      disableAuto: false
+
+  "gemma-4-12B-it-qat":
+    cmd: |
+      ${llama_server}
+      ${llama_server_args}
+      --models-preset /presets/llama-swap-presets/gemma-4-12b-it-qat.ini
+      --log-file /presets/llama-swap-logs/gemma-4-12b-it-qat.log
+    proxy: http://127.0.0.1:${PORT}
+    checkEndpoint: /health
+    capabilities:
+      tools: true
+      disableAuto: false
+
+  "gemma-4-26B-A4B-it-qat-MTP":
+    cmd: |
+      ${llama_server}
+      ${llama_server_args}
+      --models-preset /presets/llama-swap-presets/gemma-4-26b-a4b-it-qat-mtp.ini
+      --log-file /presets/llama-swap-logs/gemma-4-26b-a4b-it-qat-mtp.log
+    proxy: http://127.0.0.1:${PORT}
+    checkEndpoint: /health
+    concurrencyLimit: 0
+    capabilities:
+      tools: true
+      disableAuto: false

preset.ini

@@ -108,3 +108,19 @@ spec-type = draft-mtp
 spec-draft-n-max = 2
 load-mode = none
 lazy-mode = auto
+
+[Qwopus3.6-35B-A3B-Coder-MTP]
+hf = Jackrong/Qwopus3.6-35B-A3B-Coder-MTP-GGUF:Q4_K_M
+temp = 0.6
+top-p = 0.95
+top-k = 20
+presence-penalty = 0.0
+min-p = 0.00
+repeat-penalty = 1.0
+spec-type = draft-mtp
+spec-draft-n-max = 2
+device = CUDA0
+split-mode = layer
+main-gpu = 0
+load-mode = none
+lazy-mode = auto

server.sh

@@ -1,29 +1,190 @@
 #!/usr/bin/env bash
 set -euo pipefail
+
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${SCRIPT_DIR}/.env"
+if [[ ! -f "${ENV_FILE}" ]]; then
+  printf 'Missing environment file: %s\n' "${ENV_FILE}" >&2
+  exit 1
+fi
+
 set -a
-source "${SCRIPT_DIR}/.env"
+# shellcheck disable=SC1090
+source "${ENV_FILE}"
 set +a
 
-SRC_DIR="${LLAMA_DIR}"
-IMAGE="${BUILDER_IMAGE}"
-PRESET="${1:-"preset.ini"}"
-
-sudo docker run -d --rm --name "${SERVER_CONTAINER_NAME}" --gpus all \
-  --device=/dev/dri \
-  --ulimit memlock=-1:-1 \
-  -p "${SERVER_PORT}:${SERVER_INTERNAL_PORT}" \
-  -e "HF_HOME=/models" \
-  -e "LLAMA_API_KEY=${LLAMA_API_KEY}" \
-  -e "HF_TOKEN=${HF_TOKEN}" \
-  -v "${SRC_DIR}:/src" \
-  -v "${PRESETS_DIR}:/presets" \
-  -v "${MODELS_DIR}:/models" \
-  -w /models \
-  "${IMAGE}" \
-  /src/build-"${CUDA_BUILD}"/bin/llama-server \
-    --host "${SERVER_HOST}" \
-    --port "${SERVER_PORT}" \
-    --models-preset "/presets/${PRESET}" \
-    --log-file /presets/llama-server.log
+CUDA_BUILD="${CUDA_BUILD:-12}"
+DOCKER=(sudo docker)
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [command] [preset]
+
+Commands:
+  start [preset]   Start the server in the background (default)
+  daemon [preset]  Alias for start
+  stop             Gracefully stop the server
+  restart [preset] Stop and start the server
+  status           Show the Docker container state and published port
+
+Legacy form:
+  $(basename "$0") [preset]
+EOF
+}
+
+container_state() {
+  local state
+  if state="$("${DOCKER[@]}" inspect --format='{{.State.Status}}' "${SERVER_CONTAINER_NAME}" 2>/dev/null)"; then
+    printf '%s\n' "${state}"
+  else
+    printf 'missing\n'
+  fi
+}
+
+remove_stale_container() {
+  if [[ "$(container_state)" != "missing" ]]; then
+    "${DOCKER[@]}" rm --force "${SERVER_CONTAINER_NAME}" >/dev/null
+  fi
+}
+
+start_server() {
+  : "${LLAMA_DIR:?LLAMA_DIR must be set in .env}"
+  : "${PRESETS_DIR:?PRESETS_DIR must be set in .env}"
+  : "${MODELS_DIR:?MODELS_DIR must be set in .env}"
+  : "${BUILDER_IMAGE:?BUILDER_IMAGE must be set in .env}"
+  : "${SERVER_HOST:?SERVER_HOST must be set in .env}"
+  : "${SERVER_PORT:?SERVER_PORT must be set in .env}"
+  : "${SERVER_INTERNAL_PORT:?SERVER_INTERNAL_PORT must be set in .env}"
+
+  if [[ "$(container_state)" == "running" ]]; then
+    printf '%s is already running\n' "${SERVER_CONTAINER_NAME}" >&2
+    return 1
+  fi
+
+  if [[ ! -f "${PRESETS_DIR}/${PRESET}" ]]; then
+    printf 'Missing server preset: %s\n' "${PRESETS_DIR}/${PRESET}" >&2
+    return 1
+  fi
+
+  if [[ ! -x "${LLAMA_DIR}/build-${CUDA_BUILD}/bin/llama-server" ]]; then
+    printf 'Missing custom llama-server: %s\n' \
+      "${LLAMA_DIR}/build-${CUDA_BUILD}/bin/llama-server" >&2
+    printf 'Build llama.cpp before starting the server.\n' >&2
+    return 1
+  fi
+
+  remove_stale_container
+
+  "${DOCKER[@]}" run --detach --rm \
+    --name "${SERVER_CONTAINER_NAME}" \
+    --gpus all \
+    --device=/dev/dri \
+    --ulimit memlock=-1:-1 \
+    --publish "${SERVER_PORT}:${SERVER_INTERNAL_PORT}" \
+    --env "HF_HOME=/models" \
+    --env "LLAMA_API_KEY=${LLAMA_API_KEY:-}" \
+    --env "HF_TOKEN=${HF_TOKEN:-}" \
+    --volume "${LLAMA_DIR}:/src" \
+    --volume "${PRESETS_DIR}:/presets" \
+    --volume "${MODELS_DIR}:/models" \
+    --workdir /models \
+    "${BUILDER_IMAGE}" \
+    "/src/build-${CUDA_BUILD}/bin/llama-server" \
+      --host "${SERVER_HOST}" \
+      --port "${SERVER_PORT}" \
+      --models-preset "/presets/${PRESET}" \
+      --log-file /presets/llama-server.log
+  printf 'Started %s in the background with preset %s\n' \
+    "${SERVER_CONTAINER_NAME}" "${PRESET}"
+}
+
+stop_server() {
+  local state timeout
+  state="$(container_state)"
+  timeout="${STOP_TIMEOUT:-30}"
+
+  if [[ "${state}" == "missing" ]]; then
+    printf '%s is not running\n' "${SERVER_CONTAINER_NAME}"
+    return 0
+  fi
+
+  if [[ "${state}" == "running" || "${state}" == "created" || "${state}" == "restarting" ]]; then
+    "${DOCKER[@]}" stop --time="${timeout}" "${SERVER_CONTAINER_NAME}" >/dev/null
+  else
+    remove_stale_container
+  fi
+  printf 'Stopped %s\n' "${SERVER_CONTAINER_NAME}"
+}
+
+status_server() {
+  local state port
+  state="$(container_state)"
+  if [[ "${state}" == "missing" ]]; then
+    printf '%s: stopped (container missing)\n' "${SERVER_CONTAINER_NAME}"
+    return 1
+  fi
+
+  port="$("${DOCKER[@]}" port "${SERVER_CONTAINER_NAME}" "${SERVER_INTERNAL_PORT}/tcp" 2>/dev/null || true)"
+  if [[ -n "${port}" ]]; then
+    printf '%s: %s (%s)\n' "${SERVER_CONTAINER_NAME}" "${state}" "${port}"
+  else
+    printf '%s: %s\n' "${SERVER_CONTAINER_NAME}" "${state}"
+  fi
+  [[ "${state}" == "running" ]]
+}
+
+if [[ $# -gt 0 ]]; then
+  COMMAND="$1"
+else
+  COMMAND="start"
+fi
+PRESET="preset.ini"
+case "${COMMAND}" in
+  start|daemon|restart)
+    if [[ $# -gt 0 ]]; then
+      shift
+    fi
+    PRESET="${1:-preset.ini}"
+    ;;
+  stop|status)
+    shift
+    if [[ $# -gt 0 ]]; then
+      printf 'Unexpected argument for %s: %s\n' "${COMMAND}" "$1" >&2
+      usage >&2
+      exit 2
+    fi
+    ;;
+  help|-h|--help)
+    usage
+    exit 0
+    ;;
+  *)
+    # Preserve the original ./server.sh [preset] interface.
+    PRESET="${COMMAND}"
+    COMMAND="start"
+    ;;
+esac
+
+if [[ ("${COMMAND}" == "start" || "${COMMAND}" == "daemon" || "${COMMAND}" == "restart") && $# -gt 1 ]]; then
+  printf 'Too many arguments for %s\n' "${COMMAND}" >&2
+  usage >&2
+  exit 2
+fi
+
+: "${SERVER_CONTAINER_NAME:?SERVER_CONTAINER_NAME must be set in .env}"
+case "${COMMAND}" in
+  start|daemon)
+    start_server
+    ;;
+  stop)
+    stop_server
+    ;;
+  restart)
+    stop_server
+    start_server
+    ;;
+  status)
+    status_server
+    ;;
+esac
 #    --rpc "${RPC_HOSTS}:${RPC_PORT}"