File size: 1,237 Bytes
87aeaf1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | FROM debian:bookworm-slim AS build
RUN apt-get update && apt-get install -y --no-install-recommends \
git cmake g++ make libgomp1 ca-certificates && rm -rf /var/lib/apt/lists/*
WORKDIR /src
RUN git clone https://github.com/ggml-org/llama.cpp.git . && \
git checkout 3de52ac38818978d6da8f386eda43a856e53714a
# BitNet b1.58 uses squared ReLU in the FFN; upstream graph hard-codes SiLU.
RUN sed -i 's/LLM_FFN_SILU, LLM_FFN_PAR, il);/LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);/' src/models/bitnet.cpp
RUN cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF \
-DGGML_NATIVE=OFF -DGGML_OPENMP=ON -DLLAMA_CURL=OFF \
-DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_SERVER=ON && \
cmake --build build --config Release --target llama-server -j2
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
libgomp1 libssl3 libzstd1 ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=build /src/build/bin/llama-server /usr/local/bin/llama-server
COPY model.gguf /models/model.gguf
EXPOSE 7860
CMD ["llama-server", "-m", "/models/model.gguf", "--host", "0.0.0.0", "--port", "7860", \
"--ctx-size", "256", "--threads", "2", "--parallel", "1", "--no-mmap"]
|