KnowLine-4B-Gen2: 0.5 Gen1 + 0.25 mix F step 4500 + 0.25 mix F step 5000 (bf16)

c7ca0de verified 15 days ago

1.54 kB

#!/bin/bash
# Serve KnowLine-4B-Gen2 behind /v1/systemone with the settings of our Decision Index run (see INFERENCE.md).
#   bash serve_knowline.sh <model dir or HF repo id> [gpu=0] [port=8080]
# Starts the SGLang engine (FP8 at load; port = front-end port + 1000), then knowline_server.py (chat style,
# temperature 1, 16 scoring threads), which needs only transformers and requests.
# MEM (default 0.72) is SGLang's --mem-fraction-static: it only sizes the KV cache, scores do not depend on it.
# HOST (default 127.0.0.1) is the front end's bind address.
# Without SGLang: python knowline_server.py --model <model> --backend hf --port 8080 (transformers, slower).
set -euo pipefail
MODEL=${1:?usage: serve_knowline.sh <model dir or HF repo id> [gpu] [port]}
GPU=${2:-0}; PORT=${3:-8080}; SGL_PORT=$((PORT + 1000)); MEM=${MEM:-0.72}; HOST=${HOST:-127.0.0.1}
HERE=$(cd "$(dirname "$0")" && pwd)
CUDA_VISIBLE_DEVICES=$GPU python -m sglang.launch_server --model-path "$MODEL" --served-model-name m --tp 1 \
  --quantization fp8 --mem-fraction-static "$MEM" --mamba-radix-cache-strategy extra_buffer --enable-fp32-lm-head \
  --port "$SGL_PORT" &
SGL=$!
trap 'kill "$SGL" 2>/dev/null || true' EXIT
until curl -sf -m 3 "http://127.0.0.1:$SGL_PORT/health" > /dev/null; do
  sleep 5; kill -0 "$SGL" 2>/dev/null || { echo "SGLang exited" >&2; exit 1; }
done
python "$HERE/knowline_server.py" --model "$MODEL" --backend sglang --url "http://127.0.0.1:$SGL_PORT" \
  --served-model-name m --temperature 1 --workers 16 --host "$HOST" --port "$PORT"

	#!/bin/bash
	# Serve KnowLine-4B-Gen2 behind /v1/systemone with the settings of our Decision Index run (see INFERENCE.md).
	# bash serve_knowline.sh <model dir or HF repo id> [gpu=0] [port=8080]
	# Starts the SGLang engine (FP8 at load; port = front-end port + 1000), then knowline_server.py (chat style,
	# temperature 1, 16 scoring threads), which needs only transformers and requests.
	# MEM (default 0.72) is SGLang's --mem-fraction-static: it only sizes the KV cache, scores do not depend on it.
	# HOST (default 127.0.0.1) is the front end's bind address.
	# Without SGLang: python knowline_server.py --model <model> --backend hf --port 8080 (transformers, slower).
	set -euo pipefail
	MODEL=${1:?usage: serve_knowline.sh <model dir or HF repo id> [gpu] [port]}
	GPU=${2:-0}; PORT=${3:-8080}; SGL_PORT=$((PORT + 1000)); MEM=${MEM:-0.72}; HOST=${HOST:-127.0.0.1}
	HERE=$(cd "$(dirname "$0")" && pwd)
	CUDA_VISIBLE_DEVICES=$GPU python -m sglang.launch_server --model-path "$MODEL" --served-model-name m --tp 1 \
	--quantization fp8 --mem-fraction-static "$MEM" --mamba-radix-cache-strategy extra_buffer --enable-fp32-lm-head \
	--port "$SGL_PORT" &
	SGL=$!
	trap 'kill "$SGL" 2>/dev/null \|\| true' EXIT
	until curl -sf -m 3 "http://127.0.0.1:$SGL_PORT/health" > /dev/null; do
	sleep 5; kill -0 "$SGL" 2>/dev/null \|\| { echo "SGLang exited" >&2; exit 1; }
	done
	python "$HERE/knowline_server.py" --model "$MODEL" --backend sglang --url "http://127.0.0.1:$SGL_PORT" \
	--served-model-name m --temperature 1 --workers 16 --host "$HOST" --port "$PORT"