embeddinggemma-2-GGUF / reproduce /scripts /start_validation_runner.sh
homerquan's picture
Add standard Q4_K_M text model and Q8_0 vision/audio companion
336c50f verified
Raw History Blame Contribute Delete
700 Bytes
#!/usr/bin/env bash
set -euo pipefail
TASK_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
docker run -d --name embeddinggemma2-q4-validation \
--gpus all \
--publish 127.0.0.1:12435:12434 \
--mount type=volume,src=embeddinggemma2-q4-validation-models,dst=/models \
--mount "type=bind,src=${TASK_ROOT}/llama.cpp/build-cuda/bin,dst=/opt/llama,readonly" \
--env MODEL_RUNNER_PORT=12434 \
--env LLAMA_SERVER_PATH=/opt/llama \
--env LD_LIBRARY_PATH=/opt/llama:/app \
--env DISABLE_SERVER_UPDATE=1 \
--env 'LLAMA_ARGS=-ngl 999 --pooling mean -b 2048 -ub 2048 --parallel 1 -t 10' \
docker/model-runner@sha256:9967b6c1092ff01e7cf68119ab25067a1f58e3d1b06873bc16fe1e758bcd914f