Add gemma4-stack config, scripts, and docker-compose setup

.env.example

@@ -0,0 +1,37 @@
+# =============================================================================
+# Gemma 4 stack configuration. Copy a profile over this file, e.g.:
+#     cp profiles/dev-cpu.env .env
+# then edit HF_TOKEN. Every value below has a sensible default in compose.
+# =============================================================================
+
+# --- Hugging Face (required: Gemma 4 is gated) -------------------------------
+HF_TOKEN=
+
+# --- Hardware / inference knobs (rendered into llama-swap config) ------------
+NGL=0                 # GPU layers to offload. 0 = pure CPU. 99 = all (GPU).
+THREADS=4             # physical cores (this box = 4). Not hyperthreads.
+CTX=8192              # context window per model
+
+# --- Default model shown in Open WebUI (must match a llama-swap config key) --
+DEFAULT_MODEL=gemma-e4b      # gemma-e4b | gemma-12b | gemma-26b-a4b
+
+# --- RAG models (Persian-tuned) ----------------------------------------------
+TEI_EMBED_MODEL=BAAI/bge-m3          # Persian, FaMTEB top-2 (Hakim's HF repo is unpublished)
+TEI_EMBED_POOLING=cls                # bge-m3 uses cls pooling
+TEI_RERANK_MODEL=BAAI/bge-reranker-v2-m3
+TEI_DTYPE=float16            # float16 -> lighter Candle backend on CPU (saves RAM)
+
+# --- Images (override for GPU via docker-compose.gpu.yml) --------------------
+LLAMASWAP_IMAGE=ghcr.io/mostlygeek/llama-swap:cpu
+TEI_IMAGE=ghcr.io/huggingface/text-embeddings-inference:cpu-1.9
+
+# --- Host ports ---------------------------------------------------------------
+WEBUI_PORT=3000
+LLAMASWAP_PORT=8080
+TEI_EMBED_PORT=8081
+TEI_RERANK_PORT=8082
+
+# --- Misc ---------------------------------------------------------------------
+OPENAI_API_KEY=sk-local
+TEI_API_KEY=x
+WEBUI_NAME=Gemma 4 Local

.gitattributes

@@ -0,0 +1,6 @@
+models/tei/**/blobs/* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,18 @@
+# local config & secrets
+.env
+
+# rendered llama-swap config
+llama-swap/config.yaml
+
+# Open WebUI persistent data (db, vector store, uploads)
+openwebui/data/
+
+# bench output
+scripts/bench-results/
+
+# transient hf_hub download locks (0-byte)
+*.lock
+
+# python
+__pycache__/
+*.pyc

Makefile

@@ -0,0 +1,40 @@
+# Minimal helpers for the Gemma 4 stack.  Run `make help`.
+ENV     ?= dev-cpu
+ARGS    ?=
+COMPOSE  = docker compose
+
+.PHONY: help env models pull up up-gpu down ps logs test clean
+
+help: ## Show this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+	  awk 'BEGIN{FS=":.*?## "}{printf "  \033[36m%-10s\033[0m %s\n",$$1,$$2}'
+
+env: ## Copy profiles/$(ENV).env -> .env   (ENV=dev-cpu|cpu-server|cpu-server-xl|gpu)
+	cp profiles/$(ENV).env .env
+
+models: ## Download required BF16 mmproj (use ARGS=--weights to also fetch GGUFs)
+	./scripts/download-models.sh $(ARGS)
+
+pull: ## Pre-pull docker images
+	$(COMPOSE) pull
+
+up: ## Start the CPU stack
+	$(COMPOSE) up -d
+
+up-gpu: ## Start the GPU stack (needs profiles/gpu.env)
+	$(COMPOSE) -f docker-compose.yml -f docker-compose.gpu.yml up -d
+
+down: ## Stop the stack
+	$(COMPOSE) down
+
+ps: ## Show service status
+	$(COMPOSE) ps
+
+logs: ## Tail logs (SVC=llama-swap to filter)
+	$(COMPOSE) logs -f --tail=120 $(SVC)
+
+test: ## Run the e2e smoke tests
+	./scripts/e2e-test.sh
+
+clean: ## Stop and remove containers + volumes
+	$(COMPOSE) down -v

README.md

@@ -0,0 +1,130 @@
+# Gemma 4 — local, CPU-first LLM stack (llama.cpp + llama-swap + TEI + Open WebUI)
+
+A fully local stack: **Gemma 4** (text + **image** + **native audio**) served by **llama.cpp**
+behind **llama-swap**, with **Persian-tuned RAG** (HF **TEI**: `bge-m3` embedder — top-2 on
+the Persian FaMTEB benchmark — + `bge-reranker-v2-m3`) and **Open WebUI** as the front-end.
+Profiled for CPU servers, with a GPU override.
+
+> Embedder note: the Persian-SOTA **Hakim** (`MCINext/Hakim`) has no weights published in its
+> HF repo yet (only a README), so the stack defaults to **`bge-m3`**. Swap `TEI_EMBED_MODEL`
+> back to Hakim once its weights land.
+
+```
+Open WebUI ──┬─► llama-swap ─► llama-server (Gemma 4 omni + BF16 mmproj)   chat · vision · audio
+             ├─► TEI-embed  (BAAI/bge-m3 · Persian)                       RAG embeddings
+             └─► TEI-rerank (BAAI/bge-reranker-v2-m3)                      RAG rerank
+```
+
+## What runs where
+
+| Service | Image (CPU) | Internal URL | Host port |
+|---|---|---|---|
+| Open WebUI | `ghcr.io/open-webui/open-webui:main` | — | `3000` |
+| llama-swap | `ghcr.io/mostlygeek/llama-swap:cpu` | `http://llama-swap:8080/v1` | `8080` |
+| TEI embed | `…/text-embeddings-inference:cpu-1.9` | `http://tei-embed:80` | `8081` |
+| TEI rerank | `…/text-embeddings-inference:cpu-1.9` | `http://tei-rerank:80` | `8082` |
+
+## Prerequisites
+- Docker + `docker compose` v2 (`docker compose version`).
+- A Hugging Face token with the **Gemma 4 license accepted** (the model is gated):
+  ```bash
+  pip install -U "huggingface_hub[cli]" && huggingface-cli login
+  ```
+
+## Quick start (this CPU dev box)
+```bash
+cd gemma4-stack
+cp profiles/dev-cpu.env .env          # E4B default; edit HF_TOKEN=
+./scripts/download-models.sh          # pulls the REQUIRED BF16 mmproj files
+docker compose up -d                  # boots TEI + Open WebUI + llama-swap
+```
+Open <http://localhost:3000>, create the first (admin) account.
+
+**One-time in Admin UI:**
+1. **Settings → Models →** enable **Vision** on `gemma-e4b` (lets Open WebUI send images).
+2. **Admin → Functions → +** → paste `openwebui/functions/gemma4_audio_pipe.py` → **Save** → enable.
+   - In its valves, confirm `MODEL=gemma-e4b` and `LLAMASWAP_URL=http://llama-swap:8080/v1`.
+3. **Settings → Documents →** confirm embedding engine = **OpenAI**, model = your `TEI_EMBED_MODEL`,
+   **Hybrid Search ON**, and the external reranker URL is reachable.
+
+## Use it
+- **Text/Persian chat** — pick `gemma-e4b`, chat.
+- **Image** — attach a picture, ask about it (Vision toggle must be on).
+- **Native audio** — pick **“Gemma 4 · Omni (audio)”**, attach a short clip (≤ ~30 s),
+  ask your question. Gemma *hears* it (no Whisper).
+- **RAG** — upload Persian docs to a Knowledge / `#`-reference them; retrieval → rerank → answer.
+
+## Other machines / profiles
+```bash
+cp profiles/cpu-server.env .env       # 12B default; set THREADS=<physical cores>
+docker compose up -d
+
+cp profiles/gpu.env .env              # GPU box
+docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
+```
+Switch the live model anytime from the model dropdown — llama-swap loads it on demand and
+unloads the idle one (so RAM never doubles). Profiles: `gemma-e4b`, `gemma-12b`,
+`gemma-26b-a4b` (image-only).
+
+## Test it yourself
+**One command — runs all 7 checks (text · image · native audio · embeddings · rerank · UI) using the
+bundled demo assets, no setup:**
+```bash
+make test            # expects: RESULT: 7 passed, 0 failed
+```
+In the **browser** (port 3000): pick `gemma-e4b` and chat in Persian; attach `assets/test_image.png`
+and ask about it; for audio, import the Pipe (Admin → Functions) then pick “Gemma 4 · Omni (audio)”
+and attach `assets/test_audio_en.wav`.
+
+## Verify from the shell
+```bash
+curl localhost:8080/v1/models                     # llama-swap lists the profiles
+curl localhost:8081/v1/embeddings -d '{"model":"x","input":["سلام"]}'   # embedder
+curl localhost:8082/rerank -d '{"query":"پایتخت ایران","texts":["تهران","موز"]}'  # reranker
+# native audio (after: ffmpeg -i clip.any -ar 16000 -ac 1 clip.wav):
+B64=$(base64 -w0 clip.wav); curl localhost:8080/v1/chat/completions -H 'Content-Type: application/json' \
+ -d '{"model":"gemma-e4b","messages":[{"role":"user","content":[{"type":"text","text":"این صدا را بنویس"},{"type":"input_audio","input_audio":{"data":"'"$B64"'","format":"wav"}}]}]}'
+```
+
+## Tuning (`scripts/bench.sh`)
+`./scripts/download-models.sh --weights` then `./scripts/bench.sh /models/gemma-4-E4B-it-Q4_K_M.gguf`
+to sweep `--threads`. Set the winner as `THREADS` in `.env`. Long context? add KV quant
+(`-ctk q8_0 -ctv q8_0`) to the macro in `llama-swap/config.tmpl.yaml`.
+
+## Troubleshooting (the known traps)
+- **Audio crashes / asserts** → keep `-b 2048 -ub 2048` (already in the macro). Default 512 crashes.
+- **Audio is garbage/repetitive** → the mmproj must be **BF16** (we pin it). Q8_0/F16 break audio.
+- **`llama-server` not found in llama-swap** → adjust the binary path in `config.tmpl.yaml` / `entrypoint.sh`.
+- **`llama-swap` flags differ** → check `docker run --rm <llamaswap-image> llama-swap --help`; fix `entrypoint.sh`.
+- **Audio “No audio found” in the Pipe** → Open WebUI's file-store API differs by version;
+  adapt `_resolve_local_path()` in the Pipe.
+- **Embeddings fail / wrong model** → set every `RAG_*` env explicitly (Open WebUI doesn't inherit
+  `OPENAI_*`); verify in Admin → Documents. Hakim won't load? set `TEI_EMBED_MODEL=BAAI/bge-m3`
+  and `TEI_EMBED_POOLING=cls`.
+- **Reranker freezes UI / format mismatch** → fall back to Open WebUI's built-in reranker: remove the
+  `RAG_RERANKING_ENGINE=external` + URL envs and set `RAG_RERANKING_MODEL=BAAI/bge-reranker-v2-m3`.
+
+## Operational notes (validated live on a 15 GB CPU box)
+- **Gemma 4 is a reasoning model** — it emits `reasoning_content` before `content`. Give it a
+  generous `max_tokens` (≥256) or short prompts will return empty `content` (all budget spent
+  thinking). The Pipe/UI handle this; raw API callers should read both fields.
+- **Send media via a file, not inline** — base64 audio/large images exceed the shell arg limit
+  (`argument list too long`). Use `curl -d @payload.json` (the e2e script and Pipe already do).
+- **TEI memory on CPU**: the ONNX backend is heavy (bge-m3 ≈ 4.75 GB). Setting `TEI_DTYPE=float16`
+  makes ONNX bow out and TEI falls back to the lighter **Candle** backend (≈ 3.4 GB) — ~2.4 GB
+  saved, same 1024-dim Persian output. That's why `--dtype float16` is in the compose commands.
+- **Tight-RAM tuning (this box)**: dev profile uses `CTX=4096`; E4B is loaded by local path
+  (`-m /models/...gguf`) so a studio pause/resume doesn't re-download 5 GB. E4B + bge-m3 +
+  reranker + UI ≈ 13–14 GB — fits 15 GB but with little headroom; **16–24 GB is comfortable**.
+- **Verified end-to-end here**: Persian text ✓, image→“Red” ✓, native audio→exact transcript ✓,
+  Persian embeddings (1024-d) ✓, Persian rerank ordering ✓, Open WebUI ✓.
+
+## Layout
+```
+docker-compose.yml · docker-compose.gpu.yml · .env.example
+profiles/{dev-cpu,cpu-server,cpu-server-xl,gpu}.env
+llama-swap/{config.tmpl.yaml,entrypoint.sh}
+openwebui/functions/gemma4_audio_pipe.py
+scripts/{download-models.sh,bench.sh}
+```
+Full design rationale: `../.claude/plans/i-want-make-a-ancient-teapot.md`.

assets/test_audio_en.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92cdf7c1bb8821f84413a9f619e24d76fc26473e56b2b3f2fa2d674b1c4c4dd1
+size 27840

assets/test_audio_en.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26093ab539a21e7bee0df70dcb9e5b107fbeb2cfb796542b0f3f6ade664f76f3
+size 111438

docker-compose.gpu.yml

@@ -0,0 +1,42 @@
+# =============================================================================
+# GPU override — layer on top of the base file:
+#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
+#
+# Swaps images to CUDA variants, offloads layers to the GPU, and reserves the
+# GPU for the containers. Use with profiles/gpu.env (sets NGL, CTX, models).
+#
+# NOTE: keep --no-mmproj-offload in the llama-swap macro so the BF16 projector
+#       stays on CPU and spares VRAM for the model weights (weak-GPU friendly).
+# =============================================================================
+
+services:
+  llama-swap:
+    image: ${LLAMASWAP_IMAGE:-ghcr.io/mostlygeek/llama-swap:cuda}
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+  tei-embed:
+    # Default CUDA image targets Turing+; pick an arch-specific tag if needed.
+    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:1.9}
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+
+  tei-rerank:
+    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:1.9}
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]

docker-compose.yml

@@ -0,0 +1,148 @@
+# =============================================================================
+# Gemma 4 CPU-first stack — base (CPU) compose file.
+#
+#   Services:
+#     llama-swap  : OpenAI endpoint that hot-swaps Gemma 4 GGUF profiles
+#     tei-embed   : HF Text-Embeddings-Inference — Persian embedder (Hakim/bge-m3)
+#     tei-rerank  : HF Text-Embeddings-Inference — reranker (bge-reranker-v2-m3)
+#     openwebui   : chat UI + RAG orchestrator + native-audio Pipe host
+#
+#   Usage (CPU):     docker compose up -d
+#   Usage (GPU):     docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
+#
+#   All tunables come from .env (copy one of profiles/*.env → .env first).
+# =============================================================================
+
+x-common: &common
+  restart: unless-stopped
+  networks: [stack]
+
+services:
+  # ---------------------------------------------------------------------------
+  # LLM gateway: llama-swap renders its config from .env, then loads Gemma 4
+  # llama-server on demand (one model resident at a time, idle-unloaded by ttl).
+  # ---------------------------------------------------------------------------
+  llama-swap:
+    <<: *common
+    image: ${LLAMASWAP_IMAGE:-ghcr.io/mostlygeek/llama-swap:cpu}
+    container_name: llama-swap
+    entrypoint: ["/bin/sh", "/config/entrypoint.sh"]
+    environment:
+      # Rendered into config.yaml by entrypoint.sh (sed), per-machine:
+      NGL: ${NGL:-0}              # GPU layers to offload (0 = pure CPU)
+      THREADS: ${THREADS:-4}      # physical cores, not hyperthreads
+      CTX: ${CTX:-8192}           # context window
+      HF_TOKEN: ${HF_TOKEN:-}     # for gated Gemma 4 pulls via -hf
+      HF_HOME: /models/hf-cache
+      LLAMA_CACHE: /models/llama-cache   # persist GGUFs fetched by -hf across restarts
+    volumes:
+      - ./llama-swap:/config
+      - ./models:/models
+    ports:
+      - "${LLAMASWAP_PORT:-8080}:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8080/v1/models || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 40s
+
+  # ---------------------------------------------------------------------------
+  # RAG embedder (Persian). Default Hakim (FaMTEB #1); fallback bge-m3.
+  # Container listens on :80. OpenAI-compatible /v1/embeddings + native /embed.
+  # ---------------------------------------------------------------------------
+  tei-embed:
+    <<: *common
+    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:cpu-1.9}
+    container_name: tei-embed
+    command: >-
+      --model-id ${TEI_EMBED_MODEL:-BAAI/bge-m3}
+      --pooling ${TEI_EMBED_POOLING:-cls}
+      --dtype ${TEI_DTYPE:-float16}
+      --max-batch-tokens 4096
+      --max-client-batch-size 8
+      --auto-truncate
+    environment:
+      HF_TOKEN: ${HF_TOKEN:-}
+    volumes:
+      - ./models/tei:/data
+    ports:
+      - "${TEI_EMBED_PORT:-8081}:80"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:80/health || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+
+  # ---------------------------------------------------------------------------
+  # RAG reranker (cross-encoder). bge-reranker-v2-m3 covers Persian well.
+  # Exposes /rerank (Open WebUI external reranker points here).
+  # ---------------------------------------------------------------------------
+  tei-rerank:
+    <<: *common
+    image: ${TEI_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:cpu-1.9}
+    container_name: tei-rerank
+    command: >-
+      --model-id ${TEI_RERANK_MODEL:-BAAI/bge-reranker-v2-m3}
+      --dtype ${TEI_DTYPE:-float16}
+      --max-batch-tokens 4096
+      --auto-truncate
+    environment:
+      HF_TOKEN: ${HF_TOKEN:-}
+    volumes:
+      - ./models/tei:/data
+    ports:
+      - "${TEI_RERANK_PORT:-8082}:80"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:80/health || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+
+  # ---------------------------------------------------------------------------
+  # Front-end + RAG orchestrator. Talks to llama-swap (chat/vision) and the
+  # two TEI services (embed/rerank). Native audio handled by the imported Pipe.
+  # ---------------------------------------------------------------------------
+  openwebui:
+    <<: *common
+    image: ghcr.io/open-webui/open-webui:main
+    container_name: openwebui
+    environment:
+      # --- chat backend -------------------------------------------------------
+      OPENAI_API_BASE_URL: http://llama-swap:8080/v1
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-sk-local}
+      ENABLE_OLLAMA_API: "false"
+      DEFAULT_MODELS: ${DEFAULT_MODEL:-gemma-e4b}
+      # --- RAG embeddings via TEI (set RAG_* explicitly; OWUI does not inherit
+      #     OPENAI_* — see Open WebUI issues #8697 / #22084) ------------------
+      RAG_EMBEDDING_ENGINE: openai
+      RAG_OPENAI_API_BASE_URL: http://tei-embed:80/v1
+      RAG_OPENAI_API_KEY: ${TEI_API_KEY:-x}
+      RAG_EMBEDDING_MODEL: ${TEI_EMBED_MODEL:-MCINext/Hakim}
+      RAG_EMBEDDING_BATCH_SIZE: "16"
+      # --- hybrid retrieval + external reranker via TEI ----------------------
+      ENABLE_RAG_HYBRID_SEARCH: "true"
+      RAG_RERANKING_ENGINE: external
+      RAG_EXTERNAL_RERANKER_URL: http://tei-rerank:80/rerank
+      RAG_EXTERNAL_RERANKER_API_KEY: ${TEI_API_KEY:-x}
+      RAG_TOP_K: "8"
+      RAG_TOP_K_RERANKER: "4"
+      # --- native audio is the Pipe's job; turn STT off ----------------------
+      AUDIO_STT_ENGINE: ""
+      WEBUI_NAME: ${WEBUI_NAME:-Gemma 4 Local}
+    volumes:
+      - ./openwebui/data:/app/backend/data
+      # functions/ is mounted for convenience; import the Pipe via Admin → Functions
+      - ./openwebui/functions:/app/backend/data/functions-src:ro
+    ports:
+      - "${WEBUI_PORT:-3000}:8080"
+    depends_on:
+      - llama-swap
+      - tei-embed
+      - tei-rerank
+
+networks:
+  stack:
+    driver: bridge

llama-swap/config.tmpl.yaml

@@ -0,0 +1,53 @@
+# =============================================================================
+# llama-swap config TEMPLATE. entrypoint.sh sed-substitutes ${NGL} ${THREADS}
+# ${CTX} from the environment, then llama-swap resolves ${base} (macro) and
+# ${PORT} (its own dynamic per-model port) at runtime. Do NOT pre-substitute
+# ${base} or ${PORT}.
+#
+# Each "models:" key is the model name you pass in the OpenAI API "model" field
+# (and what Open WebUI shows). ttl unloads an idle model so RAM frees for the
+# next swap.
+#
+# Audio correctness (E4B / 12B): BF16 mmproj + -ub 2048 + --jinja are mandatory.
+# Binary note: the llama-swap "unified" images ship llama-server on PATH. If your
+# image differs, replace "llama-server" below with its absolute path.
+# =============================================================================
+
+healthCheckTimeout: 300
+logLevel: info
+
+macros:
+  base: >-
+    llama-server --host 0.0.0.0 --port ${PORT}
+    --jinja --no-mmproj-offload -ngl ${NGL}
+    --threads ${THREADS} -b 2048 -ub 2048
+    --temp 1.0 --top-k 64 --top-p 0.95
+    --metrics
+
+models:
+  # ---- E4B: audio + image + text (dev default, ~7-8 GB) ---------------------
+  "gemma-e4b":
+    cmd: >-
+      ${base}
+      -m /models/gemma-4-E4B-it-Q4_K_M.gguf
+      --mmproj /models/mmproj-gemma-4-E4B-it-bf16.gguf
+      -c ${CTX}
+    ttl: 600
+
+  # ---- 12B: audio + image + text (server default, ~10-12 GB) ----------------
+  "gemma-12b":
+    cmd: >-
+      ${base}
+      -hf ggml-org/gemma-4-12B-it-GGUF:Q4_K_M
+      --mmproj /models/mmproj-gemma-4-12B-it-bf16.gguf
+      -c ${CTX}
+    ttl: 600
+
+  # ---- 26B-A4B MoE: image + text ONLY, no audio (~16-18 GB) -----------------
+  "gemma-26b-a4b":
+    cmd: >-
+      ${base}
+      -hf ggml-org/gemma-4-26B-A4B-it-GGUF:Q4_K_M
+      --mmproj /models/mmproj-gemma-4-26B-A4B-it-bf16.gguf
+      -c ${CTX}
+    ttl: 300

llama-swap/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Render llama-swap config from the template using env knobs, then launch.
+# Substitutes only ${NGL} ${THREADS} ${CTX} — leaves ${base} and ${PORT} for
+# llama-swap to resolve itself.
+set -eu
+
+NGL="${NGL:-0}"
+THREADS="${THREADS:-4}"
+CTX="${CTX:-8192}"
+
+TMPL="/config/config.tmpl.yaml"
+OUT="/tmp/config.yaml"
+
+sed \
+  -e "s|\${NGL}|${NGL}|g" \
+  -e "s|\${THREADS}|${THREADS}|g" \
+  -e "s|\${CTX}|${CTX}|g" \
+  "$TMPL" > "$OUT"
+
+echo "===================================================================="
+echo "[entrypoint] rendered $OUT  (NGL=$NGL THREADS=$THREADS CTX=$CTX)"
+echo "===================================================================="
+cat "$OUT"
+echo "===================================================================="
+
+BIN="$(command -v llama-swap || echo /app/llama-swap)"
+echo "[entrypoint] launching: $BIN --config $OUT --listen 0.0.0.0:8080"
+exec "$BIN" --config "$OUT" --listen 0.0.0.0:8080

profiles/cpu-server-xl.env

@@ -0,0 +1,24 @@
+# Profile: large CPU server (>= 24 GiB RAM) for the 26B-A4B MoE.
+# NOTE: 26B-A4B is IMAGE + TEXT only (no audio). Use gemma-12b if you need audio.
+HF_TOKEN=
+
+NGL=0
+THREADS=24           # <-- physical cores
+CTX=16384
+DEFAULT_MODEL=gemma-26b-a4b
+
+TEI_EMBED_MODEL=BAAI/bge-m3
+TEI_EMBED_POOLING=cls
+TEI_RERANK_MODEL=BAAI/bge-reranker-v2-m3
+TEI_DTYPE=float16            # float16 -> lighter Candle backend on CPU (saves RAM)
+
+LLAMASWAP_IMAGE=ghcr.io/mostlygeek/llama-swap:cpu
+TEI_IMAGE=ghcr.io/huggingface/text-embeddings-inference:cpu-1.9
+
+WEBUI_PORT=3000
+LLAMASWAP_PORT=8080
+TEI_EMBED_PORT=8081
+TEI_RERANK_PORT=8082
+OPENAI_API_KEY=sk-local
+TEI_API_KEY=x
+WEBUI_NAME=Gemma 4 Local (xl)

profiles/cpu-server.env

@@ -0,0 +1,24 @@
+# Profile: a bigger CPU server (>= 16 GiB RAM, many cores).
+# Default model 12B (audio+image+text). Set THREADS to the server's PHYSICAL cores.
+HF_TOKEN=
+
+NGL=0
+THREADS=16            # <-- set to physical core count of the server
+CTX=16384
+DEFAULT_MODEL=gemma-12b
+
+TEI_EMBED_MODEL=BAAI/bge-m3
+TEI_EMBED_POOLING=cls
+TEI_RERANK_MODEL=BAAI/bge-reranker-v2-m3
+TEI_DTYPE=float16            # float16 -> lighter Candle backend on CPU (saves RAM)
+
+LLAMASWAP_IMAGE=ghcr.io/mostlygeek/llama-swap:cpu
+TEI_IMAGE=ghcr.io/huggingface/text-embeddings-inference:cpu-1.9
+
+WEBUI_PORT=3000
+LLAMASWAP_PORT=8080
+TEI_EMBED_PORT=8081
+TEI_RERANK_PORT=8082
+OPENAI_API_KEY=sk-local
+TEI_API_KEY=x
+WEBUI_NAME=Gemma 4 Local (server)

profiles/dev-cpu.env

@@ -0,0 +1,24 @@
+# Profile: this 15 GiB Lightning dev box (Xeon 8470, 4 phys cores, no GPU).
+# Default model E4B (audio+image+text) — comfortably fits with TEI + Open WebUI.
+HF_TOKEN=
+
+NGL=0
+THREADS=4
+CTX=8192
+DEFAULT_MODEL=gemma-e4b
+
+TEI_EMBED_MODEL=BAAI/bge-m3
+TEI_EMBED_POOLING=cls
+TEI_RERANK_MODEL=BAAI/bge-reranker-v2-m3
+TEI_DTYPE=float16            # float16 -> lighter Candle backend on CPU (saves RAM)
+
+LLAMASWAP_IMAGE=ghcr.io/mostlygeek/llama-swap:cpu
+TEI_IMAGE=ghcr.io/huggingface/text-embeddings-inference:cpu-1.9
+
+WEBUI_PORT=3000
+LLAMASWAP_PORT=8080
+TEI_EMBED_PORT=8081
+TEI_RERANK_PORT=8082
+OPENAI_API_KEY=sk-local
+TEI_API_KEY=x
+WEBUI_NAME=Gemma 4 Local (dev)

profiles/gpu.env

@@ -0,0 +1,29 @@
+# Profile: a (weak) GPU box. Use with the GPU override:
+#   cp profiles/gpu.env .env
+#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d
+#
+# NGL=99 offloads all model layers; lower it (e.g. 20) if VRAM < model size.
+# The BF16 mmproj stays on CPU (--no-mmproj-offload in the llama-swap macro).
+HF_TOKEN=
+
+NGL=99
+THREADS=8
+CTX=32768
+DEFAULT_MODEL=gemma-12b
+
+TEI_EMBED_MODEL=BAAI/bge-m3
+TEI_EMBED_POOLING=cls
+TEI_RERANK_MODEL=BAAI/bge-reranker-v2-m3
+TEI_DTYPE=float16            # float16 -> lighter Candle backend on CPU (saves RAM)
+
+# CUDA images (override base):
+LLAMASWAP_IMAGE=ghcr.io/mostlygeek/llama-swap:cuda
+TEI_IMAGE=ghcr.io/huggingface/text-embeddings-inference:1.9
+
+WEBUI_PORT=3000
+LLAMASWAP_PORT=8080
+TEI_EMBED_PORT=8081
+TEI_RERANK_PORT=8082
+OPENAI_API_KEY=sk-local
+TEI_API_KEY=x
+WEBUI_NAME=Gemma 4 Local (gpu)

scripts/bench.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Sweep llama-bench across thread counts on this machine to pick the best
+# --threads for a given Gemma 4 GGUF. Results saved under scripts/bench-results/.
+#
+#   ./bench.sh /models/gemma-4-E4B-it-Q4_K_M.gguf
+#   THREADS_LIST="2 4 8" ./bench.sh <gguf-path-inside-container>
+#
+# Requires the GGUF present in ./models (pre-download with download-models.sh
+# --weights). Uses the prebuilt llama.cpp image (CPU).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+MODEL="${1:?usage: bench.sh /models/<file>.gguf  (path is inside the container)}"
+THREADS_LIST="${THREADS_LIST:-1 2 4 8}"
+IMAGE="${LLAMA_IMAGE:-ghcr.io/ggml-org/llama.cpp:full}"
+OUT="scripts/bench-results"; mkdir -p "$OUT"
+STAMP="$(date +%Y%m%d-%H%M%S)"
+REPORT="$OUT/bench-$STAMP.md"
+
+echo "# llama-bench $(uname -n) $STAMP" | tee "$REPORT"
+echo "model: $MODEL" | tee -a "$REPORT"
+echo | tee -a "$REPORT"
+
+for t in $THREADS_LIST; do
+  echo "## threads=$t" | tee -a "$REPORT"
+  docker run --rm -v "$PWD/models:/models" "$IMAGE" \
+    llama-bench -m "$MODEL" -t "$t" -p 512 -n 128 2>/dev/null \
+    | tee -a "$REPORT" || echo "  (run failed for t=$t)" | tee -a "$REPORT"
+  echo | tee -a "$REPORT"
+done
+
+echo "Saved: $REPORT"

scripts/download-models.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Download the Gemma 4 BF16 mmproj projectors (REQUIRED — must be pinned, never
+# auto-fetched) and, optionally, the LLM GGUF weights. The LLM weights are also
+# fetched automatically by llama-swap's `-hf ...:Q4_K_M` on first request, so by
+# default we only pull the mmproj files.
+#
+#   ./download-models.sh              # mmproj only (fast, required)
+#   ./download-models.sh --weights    # also pre-pull the Q4_K_M GGUFs
+#
+# Prereq: huggingface-cli logged in (Gemma is gated):
+#   pip install -U "huggingface_hub[cli]" && huggingface-cli login
+#   (and accept the Gemma 4 license once on huggingface.co)
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+DEST="models"
+mkdir -p "$DEST"
+
+HF="$(command -v huggingface-cli || command -v hf || true)"
+if [ -z "$HF" ]; then
+  echo "ERROR: huggingface-cli not found. Run: pip install -U 'huggingface_hub[cli]'" >&2
+  exit 1
+fi
+
+# repo : mmproj-filename : quant-filename
+MODELS="
+ggml-org/gemma-4-E4B-it-GGUF:mmproj-gemma-4-E4B-it-bf16.gguf:gemma-4-E4B-it-Q4_K_M.gguf
+ggml-org/gemma-4-12B-it-GGUF:mmproj-gemma-4-12B-it-bf16.gguf:gemma-4-12B-it-Q4_K_M.gguf
+ggml-org/gemma-4-26B-A4B-it-GGUF:mmproj-gemma-4-26B-A4B-it-bf16.gguf:gemma-4-26B-A4B-it-Q4_K_M.gguf
+"
+
+WEIGHTS=0
+[ "${1:-}" = "--weights" ] && WEIGHTS=1
+
+dl() { # repo file
+  echo ">> $1 :: $2"
+  "$HF" download "$1" "$2" --local-dir "$DEST" || \
+    echo "   !! could not fetch $2 — verify the exact filename on the HF repo 'Files' tab" >&2
+}
+
+for row in $MODELS; do
+  repo="${row%%:*}"; rest="${row#*:}"
+  mmproj="${rest%%:*}"; quant="${rest#*:}"
+  dl "$repo" "$mmproj"                 # REQUIRED (BF16, pinned for audio)
+  [ "$WEIGHTS" = "1" ] && dl "$repo" "$quant"
+done
+
+echo
+echo "Done. mmproj files:"
+ls -lh "$DEST"/mmproj-*.gguf 2>/dev/null || echo "  (none — check errors above)"
+echo
+echo "TEI embedder/reranker download themselves on first start into models/tei/."

scripts/e2e-test.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# End-to-end smoke tests for the Gemma 4 stack. Exit non-zero if anything fails.
+#   ./scripts/e2e-test.sh
+# Env overrides: LS=http://localhost:8080  EMB=http://localhost:8081
+#                RRK=http://localhost:8082  OWUI=http://localhost:3000
+#                MODEL=gemma-e4b  TEST_WAV=/path/clip.wav  TEST_IMG=/path/img.png
+set -u
+
+LS="${LS:-http://localhost:8080}"
+EMB="${EMB:-http://localhost:8081}"
+RRK="${RRK:-http://localhost:8082}"
+OWUI="${OWUI:-http://localhost:3000}"
+MODEL="${MODEL:-gemma-e4b}"
+# Auto-use the bundled demo assets so `make test` needs no setup.
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+TEST_IMG="${TEST_IMG:-$ROOT/assets/test_image.png}"
+TEST_WAV="${TEST_WAV:-$ROOT/assets/test_audio_en.wav}"
+
+PASS=0; FAIL=0
+ok(){ echo "  ✅ $1"; PASS=$((PASS+1)); }
+no(){ echo "  ❌ $1"; FAIL=$((FAIL+1)); }
+hdr(){ echo; echo "== $1 =="; }
+
+jqget(){ python3 -c "import sys,json;d=json.load(sys.stdin);print(eval(sys.argv[1]))" "$1" 2>/dev/null; }
+
+hdr "1) llama-swap lists model profiles"
+MODELS=$(curl -fsS "$LS/v1/models" 2>/dev/null)
+echo "$MODELS" | grep -q "$MODEL" && ok "model '$MODEL' present" || no "model '$MODEL' missing"
+
+hdr "2) Gemma text chat (Persian)"
+RESP=$(curl -fsS "$LS/v1/chat/completions" -H 'Content-Type: application/json' -d \
+  "{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"به فارسی یک جمله کوتاه بنویس.\"}],\"max_tokens\":512,\"temperature\":1.0}" 2>/dev/null)
+TXT=$(echo "$RESP" | jqget "(d['choices'][0]['message'].get('content') or d['choices'][0]['message'].get('reasoning_content') or '')")
+[ -n "$TXT" ] && ok "got reply: ${TXT:0:60}" || no "no text reply ($RESP)"
+
+hdr "3) Gemma vision (image)"
+if [ -n "$TEST_IMG" ] && [ -f "$TEST_IMG" ]; then
+  python3 -c "import json,base64;b=base64.b64encode(open('$TEST_IMG','rb').read()).decode();json.dump({'model':'$MODEL','messages':[{'role':'user','content':[{'type':'text','text':'Describe this image in one short sentence.'},{'type':'image_url','image_url':{'url':'data:image/png;base64,'+b}}]}],'max_tokens':512},open('/tmp/e2e_img.json','w'))"
+  R=$(curl -s -m 600 "$LS/v1/chat/completions" -H 'Content-Type: application/json' -d @/tmp/e2e_img.json 2>/dev/null)
+  V=$(echo "$R" | jqget "(d['choices'][0]['message'].get('content') or d['choices'][0]['message'].get('reasoning_content') or '')")
+  [ -n "$V" ] && ok "vision reply: ${V:0:60}" || no "no vision reply ($R)"
+else
+  echo "  ⏭  skipped (set TEST_IMG=/path/img.png)"
+fi
+
+hdr "4) Gemma native audio (input_audio)"
+if [ -n "$TEST_WAV" ] && [ -f "$TEST_WAV" ]; then
+  python3 -c "import json,base64;b=base64.b64encode(open('$TEST_WAV','rb').read()).decode();json.dump({'model':'$MODEL','messages':[{'role':'user','content':[{'type':'text','text':'Transcribe this audio exactly.'},{'type':'input_audio','input_audio':{'data':b,'format':'wav'}}]}],'max_tokens':512,'temperature':1.0},open('/tmp/e2e_aud.json','w'))"
+  R=$(curl -s -m 1500 "$LS/v1/chat/completions" -H 'Content-Type: application/json' -d @/tmp/e2e_aud.json 2>/dev/null)
+  A=$(echo "$R" | jqget "(d['choices'][0]['message'].get('content') or d['choices'][0]['message'].get('reasoning_content') or '')")
+  [ -n "$A" ] && ok "audio reply: ${A:0:80}" || no "no audio reply ($R)"
+else
+  echo "  ⏭  skipped (set TEST_WAV=/path/clip.wav)"
+fi
+
+hdr "5) TEI embeddings (Persian)"
+R=$(curl -fsS "$EMB/v1/embeddings" -H 'Content-Type: application/json' -d \
+  '{"model":"x","input":["پایتخت ایران تهران است"]}' 2>/dev/null)
+DIM=$(echo "$R" | jqget "len(d['data'][0]['embedding'])")
+[ -n "$DIM" ] && [ "$DIM" -gt 0 ] 2>/dev/null && ok "embedding dim=$DIM" || no "no embedding ($R)"
+
+hdr "6) TEI rerank (Persian relevance ordering)"
+R=$(curl -fsS "$RRK/rerank" -H 'Content-Type: application/json' -d \
+  '{"query":"پایتخت ایران کجاست؟","texts":["تهران پایتخت ایران است.","موز یک میوه است."]}' 2>/dev/null)
+TOP=$(echo "$R" | python3 -c "import sys,json;d=json.load(sys.stdin);d=sorted(d,key=lambda x:-x['score']);print(d[0]['index'])" 2>/dev/null)
+[ "$TOP" = "0" ] && ok "relevant text ranked first" || no "rerank wrong/failed (top=$TOP, $R)"
+
+hdr "7) Open WebUI reachable"
+CODE=$(curl -s -o /dev/null -w "%{http_code}" "$OWUI/health" 2>/dev/null)
+[ "$CODE" = "200" ] && ok "Open WebUI /health 200" || no "Open WebUI not healthy (HTTP $CODE)"
+
+echo; echo "=================================================="
+echo "  RESULT: $PASS passed, $FAIL failed"
+echo "=================================================="
+[ "$FAIL" -eq 0 ]