File size: 2,204 Bytes
b4b0f75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | #!/usr/bin/env bash
set -euo pipefail
bundle_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
mode="${1:-serve}"
if [[ "$mode" == serve || "$mode" == decide ]]; then
shift || true
else
mode=serve
fi
profile=current
args=("$@")
for ((i=0; i<${#args[@]}; i++)); do
if [[ "${args[$i]}" == --profile && $((i+1)) -lt ${#args[@]} ]]; then
profile="${args[$((i+1))]}"
fi
done
if [[ "$profile" != current && "$profile" != base ]]; then
echo 'Profile must be current or base' >&2
exit 2
fi
native="$bundle_dir/native/bin/llama-server"
model="$bundle_dir/models/Ternary-Bonsai-2-27B-PQ2_0.gguf"
adapter="$bundle_dir/adapters/broad-lr0.0001.adapter"
[[ -f "$native" && -f "$model" ]] || { echo 'Bundle is incomplete: native server or model missing' >&2; exit 2; }
# Some repository download clients materialize links as files or omit them.
ensure_link() {
local name="$1" target="$2"
if [[ ! -e "$bundle_dir/native/bin/$name" && ! -L "$bundle_dir/native/bin/$name" ]]; then
ln -s "$target" "$bundle_dir/native/bin/$name"
fi
}
for library in libggml-base libggml-cpu libggml-cuda libggml; do
ensure_link "$library.so.0" "$library.so.0.21.0"
ensure_link "$library.so" "$library.so.0"
done
for library in libllama-common libllama libmtmd; do
ensure_link "$library.so.0" "$library.so.0.2.0"
ensure_link "$library.so" "$library.so.0"
done
if [[ ! -x "$native" ]]; then
chmod u+x "$native" || { echo 'Native server needs executable permission' >&2; exit 2; }
fi
extra=()
if [[ "$profile" == current ]]; then
[[ -f "$adapter" ]] || { echo 'Bundle is incomplete: adapter missing' >&2; exit 2; }
extra+=(--adapter "$adapter")
fi
cd "$bundle_dir"
export PYTHONPATH="$bundle_dir/source${PYTHONPATH:+:$PYTHONPATH}"
export PYTHONDONTWRITEBYTECODE=1
python_bin="${BONSAI_PYTHON:-python3}"
if [[ "$mode" == decide ]]; then
exec "$python_bin" -c 'from bonsai_runtime.service import cli; cli()' \
--launch --profile "$profile" --model "$model" --binary "$native" "${extra[@]}" "${args[@]}"
fi
exec "$python_bin" -c 'from bonsai_runtime.service import http_cli; http_cli()' \
--launch --profile "$profile" --model "$model" --binary "$native" "${extra[@]}" "${args[@]}"
|