File size: 6,451 Bytes
89bf59d 476ec1a 89bf59d 476ec1a c181693 89bf59d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | # =============================================================================
# Gemma 4 CPU-first stack — base (CPU) compose file.
#
# Services:
# llama-swap : OpenAI endpoint that hot-swaps Gemma 4 GGUF profiles
# tei-embed : HF Text-Embeddings-Inference — Persian embedder (Hakim/bge-m3)
# tei-rerank : HF Text-Embeddings-Inference — reranker (bge-reranker-v2-m3)
# openwebui : chat UI + RAG orchestrator + native-audio Pipe host
#
# Usage (CPU): docker compose up -d
# Usage (GPU): docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
#
# All tunables come from .env (copy one of profiles/*.env → .env first).
# =============================================================================
x-common: &common
restart: unless-stopped
networks: [stack]
services:
# ---------------------------------------------------------------------------
# LLM gateway: llama-swap renders its config from .env, then loads Gemma 4
# llama-server on demand (one model resident at a time, idle-unloaded by ttl).
# ---------------------------------------------------------------------------
llama-swap:
<<: *common
image: ${LLAMASWAP_IMAGE:-ghcr.io/mostlygeek/llama-swap:cpu}
container_name: llama-swap
entrypoint: ["/bin/sh", "/config/entrypoint.sh"]
environment:
# Rendered into config.yaml by entrypoint.sh (sed), per-machine:
NGL: ${NGL:-0} # GPU layers to offload (0 = pure CPU)
THREADS: ${THREADS:-4} # physical cores, not hyperthreads
CTX: ${CTX:-8192} # context window
DEFAULT_MODEL: ${DEFAULT_MODEL:-gemma-e4b} # preloaded at startup (no cold first message)
HF_TOKEN: ${HF_TOKEN:-} # for gated Gemma 4 pulls via -hf
HF_HOME: /models/hf-cache
LLAMA_CACHE: /models/llama-cache # persist GGUFs fetched by -hf across restarts
volumes:
- ./llama-swap:/config
- ./models:/models
ports:
- "${LLAMASWAP_PORT:-8080}:8080"
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://localhost:8080/v1/models || exit 1"]
interval: 30s
timeout: 5s
retries: 3
start_period: 40s
# ---------------------------------------------------------------------------
# RAG embedder (Persian). Default Hakim (FaMTEB #1); fallback bge-m3.
# Container listens on :80. OpenAI-compatible /v1/embeddings + native /embed.
# ---------------------------------------------------------------------------
tei-embed:
<<: *common
image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:cpu-1.9}
container_name: tei-embed
command: >-
--model-id ${TEI_EMBED_MODEL:-BAAI/bge-m3}
--pooling ${TEI_EMBED_POOLING:-cls}
--dtype ${TEI_DTYPE:-float16}
--max-batch-tokens 4096
--max-client-batch-size 8
--auto-truncate
environment:
HF_TOKEN: ${HF_TOKEN:-}
volumes:
- ./models/tei:/data
ports:
- "${TEI_EMBED_PORT:-8081}:80"
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://localhost:80/health || exit 1"]
interval: 30s
timeout: 5s
retries: 5
start_period: 60s
# ---------------------------------------------------------------------------
# RAG reranker (cross-encoder). bge-reranker-v2-m3 covers Persian well.
# Exposes /rerank (Open WebUI external reranker points here).
# ---------------------------------------------------------------------------
tei-rerank:
<<: *common
image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:cpu-1.9}
container_name: tei-rerank
command: >-
--model-id ${TEI_RERANK_MODEL:-BAAI/bge-reranker-v2-m3}
--dtype ${TEI_DTYPE:-float16}
--max-batch-tokens 4096
--auto-truncate
environment:
HF_TOKEN: ${HF_TOKEN:-}
volumes:
- ./models/tei:/data
ports:
- "${TEI_RERANK_PORT:-8082}:80"
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://localhost:80/health || exit 1"]
interval: 30s
timeout: 5s
retries: 5
start_period: 60s
# ---------------------------------------------------------------------------
# Front-end + RAG orchestrator. Talks to llama-swap (chat/vision) and the
# two TEI services (embed/rerank). Native audio handled by the imported Pipe.
# ---------------------------------------------------------------------------
openwebui:
<<: *common
image: ghcr.io/open-webui/open-webui:main
container_name: openwebui
environment:
# --- chat backend -------------------------------------------------------
OPENAI_API_BASE_URL: http://llama-swap:8080/v1
OPENAI_API_KEY: ${OPENAI_API_KEY:-sk-local}
ENABLE_OLLAMA_API: "false"
DEFAULT_MODELS: ${DEFAULT_MODEL:-gemma-e4b}
# --- RAG embeddings via TEI (set RAG_* explicitly; OWUI does not inherit
# OPENAI_* — see Open WebUI issues #8697 / #22084) ------------------
RAG_EMBEDDING_ENGINE: openai
RAG_OPENAI_API_BASE_URL: http://tei-embed:80/v1
RAG_OPENAI_API_KEY: ${TEI_API_KEY:-x}
RAG_EMBEDDING_MODEL: ${TEI_EMBED_MODEL:-MCINext/Hakim}
RAG_EMBEDDING_BATCH_SIZE: "16"
# --- hybrid retrieval + external reranker via TEI ----------------------
ENABLE_RAG_HYBRID_SEARCH: "true"
RAG_RERANKING_ENGINE: external
RAG_EXTERNAL_RERANKER_URL: http://tei-rerank:80/rerank
RAG_EXTERNAL_RERANKER_API_KEY: ${TEI_API_KEY:-x}
RAG_TOP_K: "8"
RAG_TOP_K_RERANKER: "4"
# --- native audio is the Pipe's job; turn STT off ----------------------
AUDIO_STT_ENGINE: ""
# --- kill hidden background LLM calls (title/tags/follow-up/autocomplete
# each fire an extra chat completion on the same slow CPU backend) ---
ENABLE_TITLE_GENERATION: "false"
ENABLE_TAGS_GENERATION: "false"
ENABLE_FOLLOW_UP_GENERATION: "false"
ENABLE_AUTOCOMPLETE_GENERATION: "false"
ENABLE_RETRIEVAL_QUERY_GENERATION: "false"
ENABLE_SEARCH_QUERY_GENERATION: "false"
WEBUI_NAME: ${WEBUI_NAME:-Gemma 4 Local}
volumes:
- ./openwebui/data:/app/backend/data
# functions/ is mounted for convenience; import the Pipe via Admin → Functions
- ./openwebui/functions:/app/backend/data/functions-src:ro
ports:
- "${WEBUI_PORT:-3000}:8080"
depends_on:
- llama-swap
- tei-embed
- tei-rerank
networks:
stack:
driver: bridge
|