Aissugur Kurganbekov Claude Opus 4.8 commited on
Commit
7269eb9
·
1 Parent(s): f2f300a

Harden uploads, report real LLM availability, and support free OpenAI-compatible providers

Browse files

- Bound the upload read in 1 MB blocks so an oversized body can't be
fully materialised in memory before the 10 MB limit returns 413.
- Replace the dead ANSWER_MODE field in /health with default_mode +
llm_available + llm_provider, so clients can tell whether Thinking
mode is actually usable instead of silently falling back to Fast.
- Add OPENAI_BASE_URL so Thinking mode can target any OpenAI-compatible
provider (free tiers such as LLM7 / Cohere / Mistral) with no code
change; document the options in .env.example.
- Add tests for oversized-upload rejection and the health contract;
make the Thinking fall-back test hermetic and add LLM-answer and
LLM-abstain cases (25 passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/.env.example CHANGED
@@ -1,17 +1,29 @@
1
- # Copy this file to ".env" to enable "Thinking" mode. The app works fully
2
- # WITHOUT any key: "Fast" mode (extractive) needs nothing, and "Thinking" mode
3
- # automatically falls back to Fast when no key is present.
 
 
4
 
5
- # Provider for Thinking mode: "anthropic" (Claude) or "openai"
6
- LLM_PROVIDER=anthropic
 
 
 
 
 
7
 
8
- # API keys (only needed for Thinking mode). Leave blank to stay on Fast.
9
- ANTHROPIC_API_KEY=
10
- OPENAI_API_KEY=
 
 
 
11
 
12
- # Models used by Thinking mode (Sonnet 4.6 supports adaptive thinking).
 
 
 
13
  ANTHROPIC_MODEL=claude-sonnet-4-6
14
- OPENAI_MODEL=gpt-4o
15
 
16
  # Retrieval tuning
17
  TOP_K=4
 
1
+ # Copy this file to ".env". The app works fully WITHOUT any key: "Fast" mode
2
+ # (extractive) needs nothing, and "Thinking" mode auto-falls back to Fast when
3
+ # no provider is reachable.
4
+ #
5
+ # "Thinking" mode talks to any OpenAI-compatible provider. Pick ONE option:
6
 
7
+ # ── Option 1: FREE, no signup (good for the demo) ────────────────────────────
8
+ # LLM7.io is a community OpenAI-compatible proxy — no key, no account. Fine for
9
+ # the bundled sample docs; avoid sending real/sensitive documents through it.
10
+ LLM_PROVIDER=openai
11
+ OPENAI_BASE_URL=https://api.llm7.io/v1
12
+ OPENAI_MODEL=default
13
+ OPENAI_API_KEY=unused
14
 
15
+ # ── Option 2: free first-party tiers (better for real documents) ─────────────
16
+ # Cohere — email/Google sign-in, no phone, no card. https://dashboard.cohere.com/api-keys
17
+ # OPENAI_BASE_URL=https://api.cohere.ai/compatibility/v1 OPENAI_MODEL=command-r7b-12-2024
18
+ # Mistral — SMS verify, no card. https://console.mistral.ai (pick "Experiment for free")
19
+ # OPENAI_BASE_URL=https://api.mistral.ai/v1 OPENAI_MODEL=mistral-small-latest
20
+ # For either: keep LLM_PROVIDER=openai and set OPENAI_API_KEY to the issued key.
21
 
22
+ # ── Option 3: paid Anthropic (Claude) or real OpenAI ─────────────────────────
23
+ # Set LLM_PROVIDER=anthropic + an sk-ant-... key, or keep openai with a real
24
+ # sk-... key and OPENAI_BASE_URL empty.
25
+ ANTHROPIC_API_KEY=
26
  ANTHROPIC_MODEL=claude-sonnet-4-6
 
27
 
28
  # Retrieval tuning
29
  TOP_K=4
backend/app/config.py CHANGED
@@ -24,11 +24,11 @@ DATABASE_URL = f"sqlite:///{DB_PATH}"
24
  SAMPLE_DOCS_DIR = BACKEND_DIR.parent / "sample_docs"
25
 
26
  # Answer behaviour.
27
- # Two request-time answer modes:
28
  # "fast" -> extractive engine: instant, offline, free (no API key needed).
29
  # "thinking" -> LLM reasons on the fly with adaptive thinking (needs an API
30
  # key); automatically falls back to "fast" if no key is set.
31
- ANSWER_MODE = os.getenv("ANSWER_MODE", "extractive").strip().lower() # legacy/health only
32
  LLM_PROVIDER = os.getenv("LLM_PROVIDER", "anthropic").strip().lower()
33
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
34
  ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
@@ -37,6 +37,11 @@ ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
37
  ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6").strip()
38
  OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
39
 
 
 
 
 
 
40
  # True if any provider key is configured (so "thinking" mode is actually usable).
41
  LLM_AVAILABLE = bool(
42
  (LLM_PROVIDER == "anthropic" and ANTHROPIC_API_KEY)
 
24
  SAMPLE_DOCS_DIR = BACKEND_DIR.parent / "sample_docs"
25
 
26
  # Answer behaviour.
27
+ # Two request-time answer modes (chosen per /ask call, not globally):
28
  # "fast" -> extractive engine: instant, offline, free (no API key needed).
29
  # "thinking" -> LLM reasons on the fly with adaptive thinking (needs an API
30
  # key); automatically falls back to "fast" if no key is set.
31
+ DEFAULT_ANSWER_MODE = "fast"
32
  LLM_PROVIDER = os.getenv("LLM_PROVIDER", "anthropic").strip().lower()
33
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
34
  ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
 
37
  ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6").strip()
38
  OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
39
 
40
+ # Optional OpenAI-compatible base URL. Set this to point the OpenAI client at any
41
+ # OpenAI-API-compatible provider (e.g. a free tier like Mistral, Cohere, LLM7) so
42
+ # "thinking" mode works without an OpenAI account. Empty = real OpenAI.
43
+ OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "").strip()
44
+
45
  # True if any provider key is configured (so "thinking" mode is actually usable).
46
  LLM_AVAILABLE = bool(
47
  (LLM_PROVIDER == "anthropic" and ANTHROPIC_API_KEY)
backend/app/llm.py CHANGED
@@ -68,7 +68,11 @@ def _openai(question: str, contexts: List[str]) -> Optional[str]:
68
  return None
69
  from openai import OpenAI # lazy import
70
 
71
- client = OpenAI(api_key=config.OPENAI_API_KEY)
 
 
 
 
72
  resp = client.chat.completions.create(
73
  model=config.OPENAI_MODEL,
74
  temperature=0,
 
68
  return None
69
  from openai import OpenAI # lazy import
70
 
71
+ # base_url lets us target any OpenAI-compatible provider (incl. free tiers).
72
+ kwargs = {"api_key": config.OPENAI_API_KEY}
73
+ if config.OPENAI_BASE_URL:
74
+ kwargs["base_url"] = config.OPENAI_BASE_URL
75
+ client = OpenAI(**kwargs)
76
  resp = client.chat.completions.create(
77
  model=config.OPENAI_MODEL,
78
  temperature=0,
backend/app/main.py CHANGED
@@ -80,11 +80,14 @@ async def unhandled_exception_handler(request: Request, exc: Exception):
80
  # --------------------------------------------------------------------------- #
81
  @app.get("/health", tags=["Health"], summary="Health check")
82
  def health():
83
- """Return service status and the active answer mode (extractive or llm)."""
 
 
84
  return {
85
  "status": "ok",
86
- "answer_mode": config.ANSWER_MODE,
87
- "llm_provider": config.LLM_PROVIDER if config.ANSWER_MODE == "llm" else None,
 
88
  }
89
 
90
 
@@ -117,12 +120,23 @@ async def upload_document(
117
  detail=f"Unsupported file type '{ext or 'unknown'}'. Allowed types: PDF, TXT.",
118
  )
119
 
120
- raw = await file.read()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
  if not raw:
122
  raise HTTPException(status_code=400, detail="Uploaded file is empty.")
123
- if len(raw) > config.MAX_UPLOAD_BYTES:
124
- mb = config.MAX_UPLOAD_BYTES // (1024 * 1024)
125
- raise HTTPException(status_code=413, detail=f"File too large (max {mb} MB).")
126
 
127
  logger.info("Upload received: %s (%d bytes)", original_name, len(raw))
128
 
 
80
  # --------------------------------------------------------------------------- #
81
  @app.get("/health", tags=["Health"], summary="Health check")
82
  def health():
83
+ """Return service status and whether the optional 'thinking' (LLM) mode is
84
+ actually usable. 'fast' mode always works; 'thinking' needs a provider key,
85
+ so the frontend uses `llm_available` to avoid offering a silent no-op."""
86
  return {
87
  "status": "ok",
88
+ "default_mode": config.DEFAULT_ANSWER_MODE,
89
+ "llm_available": config.LLM_AVAILABLE,
90
+ "llm_provider": config.LLM_PROVIDER if config.LLM_AVAILABLE else None,
91
  }
92
 
93
 
 
120
  detail=f"Unsupported file type '{ext or 'unknown'}'. Allowed types: PDF, TXT.",
121
  )
122
 
123
+ # Read the upload in bounded blocks and stop the moment it exceeds the limit,
124
+ # so an oversized body can't be fully materialised in memory before we reject
125
+ # it. (Starlette spools large bodies to a temp file, but an unbounded
126
+ # `file.read()` would still load the whole thing into one bytes object.)
127
+ limit = config.MAX_UPLOAD_BYTES
128
+ buf = bytearray()
129
+ while True:
130
+ block = await file.read(1024 * 1024) # 1 MB at a time
131
+ if not block:
132
+ break
133
+ buf.extend(block)
134
+ if len(buf) > limit:
135
+ mb = limit // (1024 * 1024)
136
+ raise HTTPException(status_code=413, detail=f"File too large (max {mb} MB).")
137
+ raw = bytes(buf)
138
  if not raw:
139
  raise HTTPException(status_code=400, detail="Uploaded file is empty.")
 
 
 
140
 
141
  logger.info("Upload received: %s (%d bytes)", original_name, len(raw))
142
 
backend/tests/test_answering.py CHANGED
@@ -50,14 +50,52 @@ def test_abstains_on_empty_document():
50
  assert citations == []
51
 
52
 
53
- def test_thinking_mode_falls_back_to_fast_without_api_key():
54
- # With no API key configured, Thinking mode must transparently fall back to
55
- # the Fast (extractive) engine — still grounded, still cited, never broken.
 
 
 
 
 
56
  chunks = [make_chunk(INVOICE)]
57
  answer, abstained, mode, citations = answer_question(
58
  "What is the total amount due?", chunks, mode="thinking"
59
  )
60
  assert abstained is False
61
- assert mode == "fast" # fell back because no key
62
  assert "$2,413.98" in answer
63
  assert len(citations) >= 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  assert citations == []
51
 
52
 
53
+ def test_thinking_mode_falls_back_to_fast_when_llm_unavailable(monkeypatch):
54
+ # When the LLM is unavailable (no key, or the call fails), Thinking mode must
55
+ # transparently fall back to the Fast (extractive) engine — still grounded,
56
+ # still cited, never broken. Patch synthesize -> None so the test is hermetic
57
+ # regardless of any provider configured in .env.
58
+ from app import llm
59
+
60
+ monkeypatch.setattr(llm, "synthesize", lambda question, contexts: None)
61
  chunks = [make_chunk(INVOICE)]
62
  answer, abstained, mode, citations = answer_question(
63
  "What is the total amount due?", chunks, mode="thinking"
64
  )
65
  assert abstained is False
66
+ assert mode == "fast" # fell back because the LLM returned nothing
67
  assert "$2,413.98" in answer
68
  assert len(citations) >= 1
69
+
70
+
71
+ def test_thinking_mode_uses_llm_answer_when_available(monkeypatch):
72
+ # When the LLM answers, Thinking mode returns that answer, reports
73
+ # mode="thinking", and still attaches the grounded citations.
74
+ from app import llm
75
+
76
+ monkeypatch.setattr(
77
+ llm, "synthesize", lambda q, c: "The total amount due is $2,413.98."
78
+ )
79
+ chunks = [make_chunk(INVOICE)]
80
+ answer, abstained, mode, citations = answer_question(
81
+ "What is the total amount due?", chunks, mode="thinking"
82
+ )
83
+ assert abstained is False
84
+ assert mode == "thinking"
85
+ assert "$2,413.98" in answer
86
+ assert len(citations) >= 1
87
+
88
+
89
+ def test_thinking_mode_abstains_when_llm_says_so(monkeypatch):
90
+ # If the LLM itself returns the abstain message, that must surface as an
91
+ # abstention with no citations — never a confident-looking empty answer.
92
+ from app import llm
93
+
94
+ monkeypatch.setattr(llm, "synthesize", lambda q, c: config.ABSTAIN_MESSAGE)
95
+ chunks = [make_chunk(INVOICE)]
96
+ answer, abstained, mode, citations = answer_question(
97
+ "What is the total amount due?", chunks, mode="thinking"
98
+ )
99
+ assert abstained is True
100
+ assert mode == "thinking"
101
+ assert citations == []
backend/tests/test_api.py CHANGED
@@ -43,6 +43,24 @@ def test_upload_rejects_empty_file(client):
43
  assert res.status_code == 400
44
 
45
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  def _upload_invoice(client):
47
  content = (
48
  b"Invoice Number INV-2025-0473. Due Date April 2 2025. "
 
43
  assert res.status_code == 400
44
 
45
 
46
+ def test_upload_rejects_oversized_file(client):
47
+ big = b"x" * (10 * 1024 * 1024 + 1) # one byte over the 10 MB limit
48
+ res = client.post(
49
+ "/documents/upload", files={"file": ("big.txt", big, "text/plain")}
50
+ )
51
+ assert res.status_code == 413
52
+
53
+
54
+ def test_health_reports_llm_availability(client):
55
+ body = client.get("/health").json()
56
+ assert body["status"] == "ok"
57
+ assert body["default_mode"] == "fast"
58
+ assert isinstance(body["llm_available"], bool)
59
+ # The provider is only surfaced when an LLM is actually usable.
60
+ if not body["llm_available"]:
61
+ assert body["llm_provider"] is None
62
+
63
+
64
  def _upload_invoice(client):
65
  content = (
66
  b"Invoice Number INV-2025-0473. Due Date April 2 2025. "