File size: 934 Bytes
7acb327 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | #!/usr/bin/env bash
# Start a chat server for a model, then open http://localhost:8080 in your browser.
#
# ./run.sh models/<file>.gguf [extra llama-server options]
#
# Stop it with Ctrl-C.
set -euo pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
MODEL=${1:?usage: ./run.sh models/<file>.gguf}
shift
SERVER="$HERE/build/bin/llama-server"
[[ -x "$SERVER" ]] || { echo "llama.cpp is not built yet. Run ./install.sh first."; exit 1; }
[[ -f "$MODEL" ]] || { echo "No model at $MODEL. Download one with ./get-model.sh first."; exit 1; }
OPTS=(-m "$MODEL" -ngl 99 -c 8192 --port 8080)
# On Linux, read the whole model into memory instead of paging it from disk. A model paged
# from a slow disk can make every token wait on the drive. Macs page from fast SSD, so skip it.
[[ "$(uname -s)" != Darwin ]] && OPTS+=(--no-mmap)
echo "Starting. When you see 'server is listening', open http://localhost:8080"
exec "$SERVER" "${OPTS[@]}" "$@"
|