| # Start a chat server for a model, then open http://localhost:8080 in your browser. | |
| # | |
| # ./run.sh models/<file>.gguf [extra llama-server options] | |
| # | |
| # Stop it with Ctrl-C. | |
| set -euo pipefail | |
| HERE=$(cd "$(dirname "$0")" && pwd) | |
| MODEL=${1:?usage: ./run.sh models/<file>.gguf} | |
| shift | |
| SERVER="$HERE/build/bin/llama-server" | |
| [[ -x "$SERVER" ]] || { echo "llama.cpp is not built yet. Run ./install.sh first."; exit 1; } | |
| [[ -f "$MODEL" ]] || { echo "No model at $MODEL. Download one with ./get-model.sh first."; exit 1; } | |
| OPTS=(-m "$MODEL" -ngl 99 -c 8192 --port 8080) | |
| # On Linux, read the whole model into memory instead of paging it from disk. A model paged | |
| # from a slow disk can make every token wait on the drive. Macs page from fast SSD, so skip it. | |
| [[ "$(uname -s)" != Darwin ]] && OPTS+=(--no-mmap) | |
| echo "Starting. When you see 'server is listening', open http://localhost:8080" | |
| exec "$SERVER" "${OPTS[@]}" "$@" | |