#!/bin/bash # Serve KnowLine-31B-Gen2 behind /v1/systemone with the settings of our Decision Index run (see INFERENCE.md). # bash serve_knowline.sh [gpu=0] [port=8080] # Starts the SGLang engine (bf16; port = front-end port + 1000), then knowline_server.py (chat style, # temperature 1, 16 scoring threads, read twice: on by default in knowline_server.py, per-type temperatures from # temperatures.json: score questions / 0.7874, so their effective temperature is 1.0; box questions in Gemma's # box_2d format read y first, with box_calibration.json when present), which needs only transformers # and requests; knowline_layout.py must sit next to it. # MEM (default 0.85) is SGLang's --mem-fraction-static: it only sizes the KV cache, scores do not depend on it. # HOST (default 127.0.0.1) is the front end's bind address. # Without SGLang: python knowline_server.py --model --backend hf --port 8080 (transformers, slower). set -euo pipefail MODEL=${1:?usage: serve_knowline.sh [gpu] [port]} GPU=${2:-0}; PORT=${3:-8080}; SGL_PORT=$((PORT + 1000)); MEM=${MEM:-0.85}; HOST=${HOST:-127.0.0.1} HERE=$(cd "$(dirname "$0")" && pwd) CUDA_VISIBLE_DEVICES=$GPU python -m sglang.launch_server --model-path "$MODEL" --served-model-name m --tp 1 \ --mem-fraction-static "$MEM" --enable-fp32-lm-head \ --port "$SGL_PORT" & SGL=$! trap 'kill "$SGL" 2>/dev/null || true' EXIT until curl -sf -m 3 "http://127.0.0.1:$SGL_PORT/health" > /dev/null; do sleep 5; kill -0 "$SGL" 2>/dev/null || { echo "SGLang exited" >&2; exit 1; } done python "$HERE/knowline_server.py" --model "$MODEL" --backend sglang --url "http://127.0.0.1:$SGL_PORT" \ --served-model-name m --temperature 1 --workers 16 --host "$HOST" --port "$PORT" \ --temperatures "$HERE/temperatures.json" --box-prompt gemma --box-coords yxyx \ $([ -f "$HERE/box_calibration.json" ] && echo --box-calibration "$HERE/box_calibration.json")