Dockerfile

FROM nvidia/cuda:12.1.0-devel-ubuntu22.04

ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHONUNBUFFERED=1
ENV CMAKE_ARGS="-DLLAMA_CUDA=on -DGGML_CUDA=on"
ENV FORCE_CMAKE=1

RUN apt-get update && apt-get install -y \
    python3-pip \
    python3-dev \
    python3-venv \
    git \
    git-lfs \
    wget \
    build-essential \
    cmake \
    && rm -rf /var/lib/apt/lists/* \
    && git lfs install

WORKDIR /app

# Install deps minus llama-cpp-python
COPY requirements.txt /app/requirements-temp.txt
RUN grep -v "llama-cpp-python" /app/requirements-temp.txt > /app/requirements.txt && \
    pip install --no-cache-dir -r requirements.txt

# Install llama-cpp-python with CUDA wheel fallback
RUN pip install --no-cache-dir \
    llama-cpp-python \
    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 \
    || pip install --no-cache-dir llama-cpp-python

COPY . /app

EXPOSE 7860

ENV MODEL_REPO=DavidAU/Qwen3.6-27B-Heretic-Uncensored-FINETUNE-NEO-CODE-Di-IMatrix-MAX-GGUF
ENV MODEL_FILE=Qwen3.6-27B-NEO-CODE-HERE-2T-OT-Q4_K_M.gguf
ENV N_CTX=4096
ENV N_GPU_LAYERS=-1

CMD ["python", "app.py"]