| FROM python:3.11-slim |
|
|
| ENV DEBIAN_FRONTEND=noninteractive \ |
| MODEL_REPO=mradermacher/Llama3.3-8B-Instruct-Thinking-Heretic-Uncensored-Claude-4.5-Opus-High-Reasoning-i1-GGUF \ |
| QUANT=Q4_K_M \ |
| MODEL_FILE=auto \ |
| MMPROJ_FILE=auto \ |
| MMPROJ_PREFER=Q8_0,f16,F16 \ |
| MODEL_DIR=/data/models/llama3 \ |
| LLAMA_VERSION=b11277 \ |
| LLAMA_DIR=/opt/llama.cpp \ |
| LLAMA_SERVER_BIN=/opt/llama.cpp/llama-server \ |
| LD_LIBRARY_PATH=/opt/llama.cpp \ |
| LLAMA_HOST=0.0.0.0 \ |
| LLAMA_PORT=7860 \ |
| THREADS=8 \ |
| CTX_SIZE=8192 \ |
| BATCH_SIZE=default \ |
| UBATCH_SIZE=default \ |
| FLASH_ATTN=default \ |
| CACHE_TYPE_K=default \ |
| CACHE_TYPE_V=default \ |
| IMAGE_MAX_TOKENS=default \ |
| CHAT_TEMPLATE_FILE=default \ |
| GPU_LAYERS=0 \ |
| TEMPERATURE=0.2 \ |
| TOP_P=0.95 \ |
| TOP_K=64 \ |
| REPEAT_PENALTY=1.08 \ |
| HF_XET_HIGH_PERFORMANCE=1 \ |
| PYTHONUNBUFFERED=1 |
|
|
| RUN apt-get update && apt-get install -y --no-install-recommends \ |
| ca-certificates \ |
| curl \ |
| libgomp1 \ |
| libstdc++6 \ |
| && rm -rf /var/lib/apt/lists/* |
|
|
| RUN mkdir -p "${LLAMA_DIR}" \ |
| && curl -fL "https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_VERSION}/llama-${LLAMA_VERSION}-bin-ubuntu-x64.tar.gz" \ |
| | tar -xz --strip-components=1 -C "${LLAMA_DIR}" \ |
| && chmod +x "${LLAMA_SERVER_BIN}" |
|
|
| |
| RUN find "${LLAMA_DIR}" -maxdepth 3 | head -50 \ |
| && ldd "$(find "${LLAMA_DIR}" -name llama-server -type f | head -1)" |
|
|
| RUN pip install --no-cache-dir huggingface_hub |
|
|
| WORKDIR /app |
| COPY app.py /app/app.py |
|
|
| EXPOSE 7860 |
| CMD ["python", "app.py"] |
|
|
