Dockerfile

# CPU + FREE-tier RAG Space (Docker SDK).
#   Embeddings : BAAI/bge-small-en-v1.5   (fastembed / ONNX, no torch)
#   Vector DB  : FAISS (in-memory)
#   LLM        : Qwen3.5-0.8B (MoE)        (llama.cpp GGUF; small + fast on CPU)
#   API        : OpenAI-compatible  ->  /v1/chat/completions  (+ web UI at /)
#
# NOTE: Qwen3.5 uses the `qwen35` arch -> requires llama-cpp-python >= 0.3.32
#       (llama.cpp >= b9616). See requirements.txt.
#
# Everything is CPU-only. No GPU, no paid hardware required.
# Models are baked into the image so the Space cold-starts instantly.

FROM python:3.11-slim

# build-essential + cmake to compile llama-cpp-python; libopenblas for fast
# prompt processing (prefill), which dominates RAG latency (large prompts).
RUN apt-get update && apt-get install -y --no-install-recommends \
        build-essential cmake git curl ca-certificates \
        libopenblas-dev pkg-config && rm -rf /var/lib/apt/lists/*

# Hugging Face Spaces run the container as non-root UID 1000.
RUN useradd -m -u 1000 user
USER user
ENV HOME=/home/user \
    PATH=/home/user/.local/bin:$PATH
WORKDIR /home/user/app

# --- Python deps ---
# Build llama-cpp-python 0.3.2 FROM SOURCE (--no-binary) so we get a glibc
# binary WITHOUT the newer "set_rows" CPU decode crash. The prebuilt wheels
# can't give us that combo (old=musl, new=buggy). GGML_NATIVE=OFF keeps the
# binary portable. Everything else installs as a fast prebuilt wheel.
COPY --chown=user requirements.txt .
# GGML_NATIVE=OFF avoids AVX-512 "-march=native" (portable), but we explicitly
# turn AVX2/FMA/F16C back ON — HF's Xeon CPUs all support them, and they give
# llama.cpp a ~3-4x CPU speedup. Without them decode crawls at ~1-2 tok/s.
ENV CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_AVX=ON -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" \
    FORCE_CMAKE=1
RUN pip install --no-cache-dir --user --no-binary=llama-cpp-python -r requirements.txt

# --- Model config ------------------------------------------------------------
# Qwen3.5-0.8B (MoE): tiny active-param count -> fast on CPU, punches above a
# dense 0.5B. Q4_K_M (~533 MB) is the CPU sweet spot. Text-only GGUF (no vision).
ENV LLM_REPO=unsloth/Qwen3.5-0.8B-GGUF \
    LLM_FILE=Qwen3.5-0.8B-Q4_K_M.gguf \
    MODEL_DIR=/home/user/models \
    EMBED_MODEL=BAAI/bge-small-en-v1.5 \
    FASTEMBED_CACHE=/home/user/.cache/fastembed \
    HF_HOME=/home/user/.cache/huggingface

# Bake the LLM (~1 GB) into the image -> instant cold start.
RUN python -c "from huggingface_hub import hf_hub_download; \
    hf_hub_download(repo_id='${LLM_REPO}', filename='${LLM_FILE}', local_dir='${MODEL_DIR}')"

# Bake the embedding model (ONNX ~130 MB) into the image too.
RUN python -c "import os; from fastembed import TextEmbedding; \
    TextEmbedding(os.environ['EMBED_MODEL'], cache_dir=os.environ['FASTEMBED_CACHE'])"

# --- App ---------------------------------------------------------------------
COPY --chown=user . .

# Runtime knobs (free tier = 2 vCPU). N_CTX=2048 is ample for RAG prompts and
# keeps the KV cache small -> less RAM, faster per-token.
ENV N_CTX=2048 \
    N_THREADS=2 \
    TOP_K=4 \
    DOCS_DIR=documents

EXPOSE 7860
CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]