File size: 700 Bytes
336c50f
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env bash
set -euo pipefail
TASK_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
docker run -d --name embeddinggemma2-q4-validation \
  --gpus all \
  --publish 127.0.0.1:12435:12434 \
  --mount type=volume,src=embeddinggemma2-q4-validation-models,dst=/models \
  --mount "type=bind,src=${TASK_ROOT}/llama.cpp/build-cuda/bin,dst=/opt/llama,readonly" \
  --env MODEL_RUNNER_PORT=12434 \
  --env LLAMA_SERVER_PATH=/opt/llama \
  --env LD_LIBRARY_PATH=/opt/llama:/app \
  --env DISABLE_SERVER_UPDATE=1 \
  --env 'LLAMA_ARGS=-ngl 999 --pooling mean -b 2048 -ub 2048 --parallel 1 -t 10' \
  docker/model-runner@sha256:9967b6c1092ff01e7cf68119ab25067a1f58e3d1b06873bc16fe1e758bcd914f