File size: 1,070 Bytes
43aaf2b 0bc758e 43aaf2b 0bc758e 43aaf2b 8b15c4a 43aaf2b 23413c3 ab1399c 6ca36c7 5f06832 ab1399c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/bin/sh
set -eu
MODEL_SPEC="${MODEL_SPEC:-openbmb/MiniCPM-V-4.7-1B-gguf:Q8_0}"
HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-7860}"
CTX_SIZE="${CTX_SIZE:-16384}"
THREADS="${THREADS:-2}"
PARALLEL="${PARALLEL:-1}"
REASONING_MODE="${REASONING_MODE:-off}"
CACHE_TYPE_K="${CACHE_TYPE_K:-q8_0}"
CACHE_TYPE_V="${CACHE_TYPE_V:-q8_0}"
FLASH_ATTN="${FLASH_ATTN:-on}"
# Speculative decoding désactivé par défaut : draft-mtp est spécifique à Gemma
# et MiniCPM-V n'a pas de tête MTP. Mets SPEC_TYPE=ngram-cache dans les
# secrets du Space pour essayer l'auto-spéculation (sans modèle draft).
SPEC_TYPE="${SPEC_TYPE:-none}"
SPEC_FLAGS=""
if [ "$SPEC_TYPE" != "none" ]; then
SPEC_FLAGS="--spec-type $SPEC_TYPE"
fi
# shellcheck disable=SC2086
exec /app/llama-server \
-hf "$MODEL_SPEC" \
--host "$HOST" \
--port "$PORT" \
--ctx-size "$CTX_SIZE" \
--threads "$THREADS" \
--threads-batch "$THREADS" \
--parallel "$PARALLEL" \
--cache-type-k "$CACHE_TYPE_K" \
--cache-type-v "$CACHE_TYPE_V" \
--reasoning "$REASONING_MODE" \
-fa "$FLASH_ATTN" \
$SPEC_FLAGS
|