cazyundee commited on
Commit
707d14d
·
verified ·
1 Parent(s): d55cd28

Add standalone CPU server entrypoint

Browse files
Files changed (1) hide show
  1. server_entrypoint.sh +28 -0
server_entrypoint.sh ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+ set -eu
3
+
4
+ MODEL_DIR=/data/models/tinyllama
5
+ MODEL_PATH="$MODEL_DIR/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
6
+ mkdir -p "$MODEL_DIR"
7
+
8
+ if [ ! -f "$MODEL_PATH" ]; then
9
+ echo "Downloading TinyLlama GGUF..."
10
+ hf download \
11
+ TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF \
12
+ tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf \
13
+ --repo-type model \
14
+ --local-dir "$MODEL_DIR"
15
+ fi
16
+
17
+ # Keep this process as PID 1: no Gradio/Python subprocess wrapper.
18
+ exec /usr/local/bin/llama-server \
19
+ --model "$MODEL_PATH" \
20
+ --host 0.0.0.0 \
21
+ --port 7860 \
22
+ --n-gpu-layers 0 \
23
+ --threads 16 \
24
+ --threads-batch 16 \
25
+ --ctx-size 2048 \
26
+ --parallel 1 \
27
+ --no-mmap \
28
+ --no-warmup