docker-compose.gpu.yml

# =============================================================================
# GPU override — layer on top of the base file:
#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
#
# Swaps images to CUDA variants, offloads layers to the GPU, and reserves the
# GPU for the containers. Use with profiles/gpu.env (sets NGL, CTX, models).
#
# NOTE: keep --no-mmproj-offload in the llama-swap macro so the BF16 projector
#       stays on CPU and spares VRAM for the model weights (weak-GPU friendly).
# =============================================================================

services:
  llama-swap:
    image: ${LLAMASWAP_IMAGE:-ghcr.io/mostlygeek/llama-swap:cuda}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

  tei-embed:
    # Default CUDA image targets Turing+; pick an arch-specific tag if needed.
    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:1.9}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

  tei-rerank:
    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:1.9}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]