Tune for free CPU tier: smaller ctx, trimmed retrieval, no re-rank, capped tokens

Dockerfile

@@ -21,9 +21,9 @@ ENV HOME=/home/user \
     PORT=7860 \
     OLLAMA_URL=http://localhost:11434 \
     OLLAMA_MODEL=qwen2.5:1.5b \
-    EMBED_MODEL=nomic-embed-text \
     INDEX_PATH=/tmp/mru_budget_index.parquet \
-    OMP_NUM_THREADS=4 \
+    RERANK=off \
+    OMP_NUM_THREADS=2 \
     OLLAMA_NUM_PARALLEL=1
 
 WORKDIR /home/user/app

config.yaml

@@ -14,13 +14,13 @@ ollama:
   model: null              # null = first installed model
   request_timeout: 300
   keep_alive: 30m          # keep the model resident between questions
-  num_ctx: 8192            # context window: fits the excerpts + answer without truncation
-  num_predict: null        # max answer tokens (null = model default)
+  num_ctx: 4096            # context window: smaller = much faster prefill on CPU
+  num_predict: 512         # cap answer tokens so long answers don't stall on CPU
   warm_on_start: true      # preload the model at startup so the first answer is fast
 
 retrieval:
-  top_k: 6
-  context_budget: 11000    # chars of excerpt context handed to the model
-  rerank: auto             # embedding re-rank of BM25 hits: auto | on | off
+  top_k: 4
+  context_budget: 5000     # chars of excerpt context handed to the model (smaller = faster on CPU)
+  rerank: off              # embedding re-rank of BM25 hits: auto | on | off (off = no extra round-trip)
   embed_model: nomic-embed-text  # Ollama embedding model (auto-used only if installed)
   rerank_k: 30             # BM25 candidates to re-rank by embedding similarity