Add Fast / Thinking answer modes (per-request LLM toggle)

- Fast: extractive engine — instant, offline, free (no API key).
- Thinking: LLM reasons on the fly with Claude adaptive thinking
(thinking={"type":"adaptive"}, no temperature — correct for current models),
grounded in the same retrieved context with citations + abstention.
Transparently falls back to Fast when no API key is set, so it never breaks.
- AskRequest.mode ("fast"|"thinking", validated -> 422 on bad value); /ask
returns the engine that actually answered.
- Frontend: segmented Fast/Thinking toggle in the Ask view; mode passed to API.
- Thinking model default = claude-sonnet-4-6 (supports adaptive thinking).
- 21 tests pass incl. new thinking->fast fallback test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/.env.example

@@ -1,19 +1,17 @@
-# Copy this file to ".env" and fill in values if you want LLM-synthesized answers.
-# The app works fully WITHOUT any keys using the built-in extractive answer mode.
+# Copy this file to ".env" to enable "Thinking" mode. The app works fully
+# WITHOUT any key: "Fast" mode (extractive) needs nothing, and "Thinking" mode
+# automatically falls back to Fast when no key is present.
 
-# Answer mode: "extractive" (default, no API needed) or "llm"
-ANSWER_MODE=extractive
-
-# LLM provider when ANSWER_MODE=llm: "openai" or "anthropic"
+# Provider for Thinking mode: "anthropic" (Claude) or "openai"
 LLM_PROVIDER=anthropic
 
-# API keys (only needed for ANSWER_MODE=llm)
-OPENAI_API_KEY=
+# API keys (only needed for Thinking mode). Leave blank to stay on Fast.
 ANTHROPIC_API_KEY=
+OPENAI_API_KEY=
 
-# Model names (sensible defaults are used if left blank)
-OPENAI_MODEL=gpt-4o-mini
-ANTHROPIC_MODEL=claude-haiku-4-5-20251001
+# Models used by Thinking mode (Sonnet 4.6 supports adaptive thinking).
+ANTHROPIC_MODEL=claude-sonnet-4-6
+OPENAI_MODEL=gpt-4o
 
 # Retrieval tuning
 TOP_K=4

backend/app/answering.py

@@ -99,15 +99,21 @@ def build_citations(scored: List[ScoredChunk], question: str) -> List[Citation]:
     return citations
 
 
-def answer_question(question: str, chunks: List[Chunk]):
-    """Return (answer, abstained, mode, citations)."""
+def answer_question(question: str, chunks: List[Chunk], mode: str = "fast"):
+    """Return (answer, abstained, mode, citations).
+
+    mode="fast"     -> extractive engine (instant, offline, free).
+    mode="thinking" -> LLM reasons on the fly; falls back to fast if no API key
+                       or the call fails. The returned mode reflects what actually
+                       produced the answer ("fast" | "thinking").
+    """
     scored = rank_chunks(question, chunks, top_k=config.TOP_K)
     top_score = scored[0].score if scored else 0.0
 
     # Anti-hallucination guard #1: the best evidence must clear a score floor.
     if not scored or top_score < config.SCORE_THRESHOLD:
         logger.info("Abstain (low score %.3f) for question: %s", top_score, question)
-        return config.ABSTAIN_MESSAGE, True, "extractive", []
+        return config.ABSTAIN_MESSAGE, True, mode, []
 
     # Anti-hallucination guard #2: term coverage. A genuine answer normally
     # shares at least two distinct content words with the source. This stops a
@@ -126,18 +132,21 @@ def answer_question(question: str, chunks: List[Chunk]):
             len(q_terms),
             question,
         )
-        return config.ABSTAIN_MESSAGE, True, "extractive", []
+        return config.ABSTAIN_MESSAGE, True, mode, []
 
     # Keep only chunks that carry real signal for the citations/context.
     relevant = [s for s in scored if s.score >= config.SCORE_THRESHOLD] or scored[:1]
     citations = build_citations(relevant, question)
 
-    # Try optional LLM synthesis; fall back to extractive on any issue.
-    contexts = [s.chunk.text for s in relevant]
-    llm_answer = llm.synthesize(question, contexts)
-    if llm_answer:
-        abstained = llm_answer.strip() == config.ABSTAIN_MESSAGE
-        return llm_answer, abstained, "llm", ([] if abstained else citations)
+    # Thinking mode: let the LLM reason over the same grounded context.
+    if mode == "thinking":
+        contexts = [s.chunk.text for s in relevant]
+        llm_answer = llm.synthesize(question, contexts)
+        if llm_answer:
+            abstained = llm_answer.strip() == config.ABSTAIN_MESSAGE
+            return llm_answer, abstained, "thinking", ([] if abstained else citations)
+        # No key / call failed -> transparently fall back to the fast engine.
+        logger.info("Thinking unavailable; using fast (extractive) for: %s", question)
 
     answer = _focused_answer(question, [s.chunk for s in relevant])
-    return answer, False, "extractive", citations
+    return answer, False, "fast", citations

backend/app/config.py

@@ -24,12 +24,24 @@ DATABASE_URL = f"sqlite:///{DB_PATH}"
 SAMPLE_DOCS_DIR = BACKEND_DIR.parent / "sample_docs"
 
 # Answer behaviour.
-ANSWER_MODE = os.getenv("ANSWER_MODE", "extractive").strip().lower()
+# Two request-time answer modes:
+#   "fast"     -> extractive engine: instant, offline, free (no API key needed).
+#   "thinking" -> LLM reasons on the fly with adaptive thinking (needs an API
+#                 key); automatically falls back to "fast" if no key is set.
+ANSWER_MODE = os.getenv("ANSWER_MODE", "extractive").strip().lower()  # legacy/health only
 LLM_PROVIDER = os.getenv("LLM_PROVIDER", "anthropic").strip().lower()
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
 ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
-OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini").strip()
-ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-haiku-4-5-20251001").strip()
+
+# Model used by "thinking" mode (reasons with adaptive thinking on Anthropic).
+ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6").strip()
+OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
+
+# True if any provider key is configured (so "thinking" mode is actually usable).
+LLM_AVAILABLE = bool(
+    (LLM_PROVIDER == "anthropic" and ANTHROPIC_API_KEY)
+    or (LLM_PROVIDER == "openai" and OPENAI_API_KEY)
+)
 
 # Retrieval tuning.
 TOP_K = int(os.getenv("TOP_K", "4"))

backend/app/llm.py

@@ -1,19 +1,29 @@
-"""Optional LLM synthesis layer.
+"""Optional LLM synthesis layer for "thinking" mode.
 
-The app NEVER requires this. If ANSWER_MODE=llm and a provider key is present,
-we ask the model to answer strictly from the supplied context and to reply with
-the abstain message when the context is insufficient. Any failure (missing
-package, bad key, network error) falls back to extractive mode upstream.
+The app NEVER requires this. When the user picks Thinking mode AND a provider key
+is configured, we ask the model to reason and answer strictly from the supplied
+context, replying with the abstain message when the context is insufficient.
+Any failure (missing package, bad key, network error) returns None so the caller
+falls back to the extractive Fast engine.
+
+Anthropic path uses adaptive thinking (`thinking={"type": "adaptive"}`) — the
+correct extended-thinking shape for current Claude models — so it genuinely
+"thinks on the fly". Note: temperature must NOT be sent alongside adaptive
+thinking on current models.
 """
+import logging
 from typing import List, Optional
 
 from . import config
 
+logger = logging.getLogger(__name__)
+
 SYSTEM_PROMPT = (
     "You are a careful financial-document assistant. Answer ONLY using the "
-    "provided context excerpts. Do not invent numbers, dates, names, or facts. "
-    "If the answer is not clearly supported by the context, reply with exactly: "
-    f'"{config.ABSTAIN_MESSAGE}". Keep answers short and factual.'
+    "provided context excerpts. Reason step by step, then give a short, factual "
+    "answer. Never invent numbers, dates, names, or company names. If the answer "
+    "is not clearly supported by the context, reply with exactly: "
+    f'"{config.ABSTAIN_MESSAGE}".'
 )
 
 
@@ -23,20 +33,36 @@ def _build_user_prompt(question: str, contexts: List[str]) -> str:
 
 
 def synthesize(question: str, contexts: List[str]) -> Optional[str]:
-    """Return an LLM answer string, or None to signal 'fall back to extractive'."""
-    if config.ANSWER_MODE != "llm":
-        return None
+    """Return an LLM answer string, or None to signal 'fall back to Fast'."""
     try:
         if config.LLM_PROVIDER == "openai":
             return _openai(question, contexts)
         if config.LLM_PROVIDER == "anthropic":
             return _anthropic(question, contexts)
     except Exception as exc:  # pragma: no cover - defensive
-        print(f"[llm] synthesis failed, falling back to extractive: {exc}")
+        logger.warning("LLM synthesis failed, falling back to Fast: %s", exc)
         return None
     return None
 
 
+def _anthropic(question: str, contexts: List[str]) -> Optional[str]:
+    if not config.ANTHROPIC_API_KEY:
+        return None
+    import anthropic  # lazy import
+
+    client = anthropic.Anthropic(api_key=config.ANTHROPIC_API_KEY)
+    resp = client.messages.create(
+        model=config.ANTHROPIC_MODEL,
+        max_tokens=2048,  # generous: must cover adaptive thinking + the answer
+        thinking={"type": "adaptive"},  # reason on the fly (no temperature with this)
+        system=SYSTEM_PROMPT,
+        messages=[{"role": "user", "content": _build_user_prompt(question, contexts)}],
+    )
+    # Take only the visible text blocks; thinking blocks are skipped.
+    parts = [b.text for b in resp.content if getattr(b, "type", "") == "text"]
+    return "".join(parts).strip() or None
+
+
 def _openai(question: str, contexts: List[str]) -> Optional[str]:
     if not config.OPENAI_API_KEY:
         return None
@@ -51,23 +77,4 @@ def _openai(question: str, contexts: List[str]) -> Optional[str]:
             {"role": "user", "content": _build_user_prompt(question, contexts)},
         ],
     )
-    return (resp.choices[0].message.content or "").strip()
-
-
-def _anthropic(question: str, contexts: List[str]) -> Optional[str]:
-    if not config.ANTHROPIC_API_KEY:
-        return None
-    import anthropic  # lazy import
-
-    client = anthropic.Anthropic(api_key=config.ANTHROPIC_API_KEY)
-    resp = client.messages.create(
-        model=config.ANTHROPIC_MODEL,
-        max_tokens=400,
-        temperature=0,
-        system=SYSTEM_PROMPT,
-        messages=[
-            {"role": "user", "content": _build_user_prompt(question, contexts)},
-        ],
-    )
-    parts = [block.text for block in resp.content if getattr(block, "type", "") == "text"]
-    return "".join(parts).strip()
+    return (resp.choices[0].message.content or "").strip() or None

backend/app/main.py

@@ -208,7 +208,7 @@ def ask(req: AskRequest, db: Session = Depends(get_db)):
         )
 
     chunks = db.query(Chunk).filter(Chunk.document_id == document.id).all()
-    answer, abstained, mode, citations = answer_question(question, chunks)
+    answer, abstained, mode, citations = answer_question(question, chunks, req.mode)
     logger.info(
         "Ask doc=%d abstained=%s citations=%d q=%r",
         document.id,

backend/app/schemas.py

@@ -1,6 +1,6 @@
 """Pydantic request/response models for the API."""
 from datetime import datetime
-from typing import List, Optional
+from typing import List, Literal, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -29,12 +29,13 @@ class DocumentOut(BaseModel):
 class AskRequest(BaseModel):
     document_id: int
     question: str = Field(min_length=1, max_length=2000)
+    mode: Literal["fast", "thinking"] = "fast"
 
 
 class AskResponse(BaseModel):
     answer: str
     abstained: bool
-    mode: str  # "extractive" or "llm"
+    mode: str  # what actually answered: "fast" or "thinking"
     citations: List[Citation]
 
 

backend/tests/test_answering.py

@@ -16,7 +16,7 @@ def test_answerable_question_returns_grounded_answer_with_citation():
         "What is the total amount due?", chunks
     )
     assert abstained is False
-    assert mode == "extractive"
+    assert mode == "fast"
     assert "$2,413.98" in answer  # verbatim value from the source
     assert len(citations) >= 1
     assert citations[0].document_name == "doc.txt"
@@ -48,3 +48,16 @@ def test_abstains_on_empty_document():
     answer, abstained, _mode, citations = answer_question("anything?", [])
     assert abstained is True
     assert citations == []
+
+
+def test_thinking_mode_falls_back_to_fast_without_api_key():
+    # With no API key configured, Thinking mode must transparently fall back to
+    # the Fast (extractive) engine — still grounded, still cited, never broken.
+    chunks = [make_chunk(INVOICE)]
+    answer, abstained, mode, citations = answer_question(
+        "What is the total amount due?", chunks, mode="thinking"
+    )
+    assert abstained is False
+    assert mode == "fast"  # fell back because no key
+    assert "$2,413.98" in answer
+    assert len(citations) >= 1

frontend/src/api.js

@@ -32,11 +32,11 @@ export const api = {
   deleteDocument: (id) =>
     fetch(`${BASE}/documents/${id}`, { method: "DELETE" }).then(handle),
 
-  ask: (document_id, question) =>
+  ask: (document_id, question, mode = "fast") =>
     fetch(`${BASE}/ask`, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ document_id, question }),
+      body: JSON.stringify({ document_id, question, mode }),
     }).then(handle),
 
   history: (document_id) => {

frontend/src/components/views/Ask.jsx

@@ -21,6 +21,7 @@ export default function Ask({ documents, demoSignal, onDemoConsumed }) {
   const [history, setHistory] = useState([]);
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState("");
+  const [mode, setMode] = useState("fast"); // "fast" (extractive) | "thinking" (LLM)
   const lastDemo = useRef(0);
 
   // Default to the sample invoice for a deterministic demo, else the first doc.
@@ -49,7 +50,7 @@ export default function Ask({ documents, demoSignal, onDemoConsumed }) {
     setLoading(true);
     setResult(null);
     try {
-      const res = await api.ask(Number(id), q.trim());
+      const res = await api.ask(Number(id), q.trim(), mode);
       setResult(res);
       await loadHistory(id);
     } catch (err) {
@@ -118,6 +119,28 @@ export default function Ask({ documents, demoSignal, onDemoConsumed }) {
             ))}
           </div>
 
+          <span className="field-label">Answer mode</span>
+          <div className="segmented" role="tablist" aria-label="Answer mode">
+            <button
+              type="button"
+              className={mode === "fast" ? "seg active" : "seg"}
+              aria-pressed={mode === "fast"}
+              onClick={() => setMode("fast")}
+            >
+              ⚡ Fast
+              <small>Instant · grounded snippet</small>
+            </button>
+            <button
+              type="button"
+              className={mode === "thinking" ? "seg active" : "seg"}
+              aria-pressed={mode === "thinking"}
+              onClick={() => setMode("thinking")}
+            >
+              🧠 Thinking
+              <small>LLM reasons · needs API key</small>
+            </button>
+          </div>
+
           <form
             onSubmit={(e) => {
               e.preventDefault();
@@ -134,7 +157,7 @@ export default function Ask({ documents, demoSignal, onDemoConsumed }) {
               error={error || undefined}
             />
             <Button variant="primary" type="submit" loading={loading} disabled={!question.trim()}>
-              {loading ? "Retrieving…" : "Ask"}
+              {loading ? (mode === "thinking" ? "Thinking…" : "Retrieving…") : "Ask"}
             </Button>
           </form>
 

frontend/src/styles.css

@@ -588,6 +588,38 @@ code {
   align-items: start;
 }
 
+/* Fast / Thinking segmented control */
+.segmented {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 8px;
+  margin-bottom: 18px;
+}
+.seg {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-start;
+  gap: 2px;
+  padding: 10px 14px;
+  border: 1px solid var(--border);
+  border-radius: var(--r-md);
+  background: var(--surface-1);
+  color: var(--text-dim);
+  font-size: 14px;
+  font-weight: 600;
+  font-family: inherit;
+  cursor: pointer;
+  transition: border-color 0.18s var(--ease), background 0.18s var(--ease), color 0.18s var(--ease);
+}
+.seg:hover { background: var(--surface-2); color: var(--text); }
+.seg small { font-size: 11px; font-weight: 500; color: var(--text-faint); text-transform: none; letter-spacing: 0; }
+.seg.active {
+  color: var(--text);
+  border-color: rgba(124, 139, 255, 0.55);
+  background: var(--accent-soft);
+  box-shadow: 0 0 0 1px rgba(124, 139, 255, 0.35) inset;
+}
+
 .chips { display: flex; flex-wrap: wrap; gap: 10px; }
 .chip {
   display: flex;