Spaces:
Sleeping
Sleeping
Aissugur Kurganbekov Claude Opus 4.8 commited on
Commit ·
7269eb9
1
Parent(s): f2f300a
Harden uploads, report real LLM availability, and support free OpenAI-compatible providers
Browse files- Bound the upload read in 1 MB blocks so an oversized body can't be
fully materialised in memory before the 10 MB limit returns 413.
- Replace the dead ANSWER_MODE field in /health with default_mode +
llm_available + llm_provider, so clients can tell whether Thinking
mode is actually usable instead of silently falling back to Fast.
- Add OPENAI_BASE_URL so Thinking mode can target any OpenAI-compatible
provider (free tiers such as LLM7 / Cohere / Mistral) with no code
change; document the options in .env.example.
- Add tests for oversized-upload rejection and the health contract;
make the Thinking fall-back test hermetic and add LLM-answer and
LLM-abstain cases (25 passing).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- backend/.env.example +22 -10
- backend/app/config.py +7 -2
- backend/app/llm.py +5 -1
- backend/app/main.py +21 -7
- backend/tests/test_answering.py +42 -4
- backend/tests/test_api.py +18 -0
backend/.env.example
CHANGED
|
@@ -1,17 +1,29 @@
|
|
| 1 |
-
# Copy this file to ".env"
|
| 2 |
-
#
|
| 3 |
-
#
|
|
|
|
|
|
|
| 4 |
|
| 5 |
-
#
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
-
#
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
-
#
|
|
|
|
|
|
|
|
|
|
| 13 |
ANTHROPIC_MODEL=claude-sonnet-4-6
|
| 14 |
-
OPENAI_MODEL=gpt-4o
|
| 15 |
|
| 16 |
# Retrieval tuning
|
| 17 |
TOP_K=4
|
|
|
|
| 1 |
+
# Copy this file to ".env". The app works fully WITHOUT any key: "Fast" mode
|
| 2 |
+
# (extractive) needs nothing, and "Thinking" mode auto-falls back to Fast when
|
| 3 |
+
# no provider is reachable.
|
| 4 |
+
#
|
| 5 |
+
# "Thinking" mode talks to any OpenAI-compatible provider. Pick ONE option:
|
| 6 |
|
| 7 |
+
# ── Option 1: FREE, no signup (good for the demo) ────────────────────────────
|
| 8 |
+
# LLM7.io is a community OpenAI-compatible proxy — no key, no account. Fine for
|
| 9 |
+
# the bundled sample docs; avoid sending real/sensitive documents through it.
|
| 10 |
+
LLM_PROVIDER=openai
|
| 11 |
+
OPENAI_BASE_URL=https://api.llm7.io/v1
|
| 12 |
+
OPENAI_MODEL=default
|
| 13 |
+
OPENAI_API_KEY=unused
|
| 14 |
|
| 15 |
+
# ── Option 2: free first-party tiers (better for real documents) ─────────────
|
| 16 |
+
# Cohere — email/Google sign-in, no phone, no card. https://dashboard.cohere.com/api-keys
|
| 17 |
+
# OPENAI_BASE_URL=https://api.cohere.ai/compatibility/v1 OPENAI_MODEL=command-r7b-12-2024
|
| 18 |
+
# Mistral — SMS verify, no card. https://console.mistral.ai (pick "Experiment for free")
|
| 19 |
+
# OPENAI_BASE_URL=https://api.mistral.ai/v1 OPENAI_MODEL=mistral-small-latest
|
| 20 |
+
# For either: keep LLM_PROVIDER=openai and set OPENAI_API_KEY to the issued key.
|
| 21 |
|
| 22 |
+
# ── Option 3: paid Anthropic (Claude) or real OpenAI ─────────────────────────
|
| 23 |
+
# Set LLM_PROVIDER=anthropic + an sk-ant-... key, or keep openai with a real
|
| 24 |
+
# sk-... key and OPENAI_BASE_URL empty.
|
| 25 |
+
ANTHROPIC_API_KEY=
|
| 26 |
ANTHROPIC_MODEL=claude-sonnet-4-6
|
|
|
|
| 27 |
|
| 28 |
# Retrieval tuning
|
| 29 |
TOP_K=4
|
backend/app/config.py
CHANGED
|
@@ -24,11 +24,11 @@ DATABASE_URL = f"sqlite:///{DB_PATH}"
|
|
| 24 |
SAMPLE_DOCS_DIR = BACKEND_DIR.parent / "sample_docs"
|
| 25 |
|
| 26 |
# Answer behaviour.
|
| 27 |
-
# Two request-time answer modes:
|
| 28 |
# "fast" -> extractive engine: instant, offline, free (no API key needed).
|
| 29 |
# "thinking" -> LLM reasons on the fly with adaptive thinking (needs an API
|
| 30 |
# key); automatically falls back to "fast" if no key is set.
|
| 31 |
-
|
| 32 |
LLM_PROVIDER = os.getenv("LLM_PROVIDER", "anthropic").strip().lower()
|
| 33 |
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
|
| 34 |
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
|
|
@@ -37,6 +37,11 @@ ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
|
|
| 37 |
ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6").strip()
|
| 38 |
OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
# True if any provider key is configured (so "thinking" mode is actually usable).
|
| 41 |
LLM_AVAILABLE = bool(
|
| 42 |
(LLM_PROVIDER == "anthropic" and ANTHROPIC_API_KEY)
|
|
|
|
| 24 |
SAMPLE_DOCS_DIR = BACKEND_DIR.parent / "sample_docs"
|
| 25 |
|
| 26 |
# Answer behaviour.
|
| 27 |
+
# Two request-time answer modes (chosen per /ask call, not globally):
|
| 28 |
# "fast" -> extractive engine: instant, offline, free (no API key needed).
|
| 29 |
# "thinking" -> LLM reasons on the fly with adaptive thinking (needs an API
|
| 30 |
# key); automatically falls back to "fast" if no key is set.
|
| 31 |
+
DEFAULT_ANSWER_MODE = "fast"
|
| 32 |
LLM_PROVIDER = os.getenv("LLM_PROVIDER", "anthropic").strip().lower()
|
| 33 |
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
|
| 34 |
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "").strip()
|
|
|
|
| 37 |
ANTHROPIC_MODEL = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6").strip()
|
| 38 |
OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
|
| 39 |
|
| 40 |
+
# Optional OpenAI-compatible base URL. Set this to point the OpenAI client at any
|
| 41 |
+
# OpenAI-API-compatible provider (e.g. a free tier like Mistral, Cohere, LLM7) so
|
| 42 |
+
# "thinking" mode works without an OpenAI account. Empty = real OpenAI.
|
| 43 |
+
OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "").strip()
|
| 44 |
+
|
| 45 |
# True if any provider key is configured (so "thinking" mode is actually usable).
|
| 46 |
LLM_AVAILABLE = bool(
|
| 47 |
(LLM_PROVIDER == "anthropic" and ANTHROPIC_API_KEY)
|
backend/app/llm.py
CHANGED
|
@@ -68,7 +68,11 @@ def _openai(question: str, contexts: List[str]) -> Optional[str]:
|
|
| 68 |
return None
|
| 69 |
from openai import OpenAI # lazy import
|
| 70 |
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
resp = client.chat.completions.create(
|
| 73 |
model=config.OPENAI_MODEL,
|
| 74 |
temperature=0,
|
|
|
|
| 68 |
return None
|
| 69 |
from openai import OpenAI # lazy import
|
| 70 |
|
| 71 |
+
# base_url lets us target any OpenAI-compatible provider (incl. free tiers).
|
| 72 |
+
kwargs = {"api_key": config.OPENAI_API_KEY}
|
| 73 |
+
if config.OPENAI_BASE_URL:
|
| 74 |
+
kwargs["base_url"] = config.OPENAI_BASE_URL
|
| 75 |
+
client = OpenAI(**kwargs)
|
| 76 |
resp = client.chat.completions.create(
|
| 77 |
model=config.OPENAI_MODEL,
|
| 78 |
temperature=0,
|
backend/app/main.py
CHANGED
|
@@ -80,11 +80,14 @@ async def unhandled_exception_handler(request: Request, exc: Exception):
|
|
| 80 |
# --------------------------------------------------------------------------- #
|
| 81 |
@app.get("/health", tags=["Health"], summary="Health check")
|
| 82 |
def health():
|
| 83 |
-
"""Return service status and the
|
|
|
|
|
|
|
| 84 |
return {
|
| 85 |
"status": "ok",
|
| 86 |
-
"
|
| 87 |
-
"
|
|
|
|
| 88 |
}
|
| 89 |
|
| 90 |
|
|
@@ -117,12 +120,23 @@ async def upload_document(
|
|
| 117 |
detail=f"Unsupported file type '{ext or 'unknown'}'. Allowed types: PDF, TXT.",
|
| 118 |
)
|
| 119 |
|
| 120 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
if not raw:
|
| 122 |
raise HTTPException(status_code=400, detail="Uploaded file is empty.")
|
| 123 |
-
if len(raw) > config.MAX_UPLOAD_BYTES:
|
| 124 |
-
mb = config.MAX_UPLOAD_BYTES // (1024 * 1024)
|
| 125 |
-
raise HTTPException(status_code=413, detail=f"File too large (max {mb} MB).")
|
| 126 |
|
| 127 |
logger.info("Upload received: %s (%d bytes)", original_name, len(raw))
|
| 128 |
|
|
|
|
| 80 |
# --------------------------------------------------------------------------- #
|
| 81 |
@app.get("/health", tags=["Health"], summary="Health check")
|
| 82 |
def health():
|
| 83 |
+
"""Return service status and whether the optional 'thinking' (LLM) mode is
|
| 84 |
+
actually usable. 'fast' mode always works; 'thinking' needs a provider key,
|
| 85 |
+
so the frontend uses `llm_available` to avoid offering a silent no-op."""
|
| 86 |
return {
|
| 87 |
"status": "ok",
|
| 88 |
+
"default_mode": config.DEFAULT_ANSWER_MODE,
|
| 89 |
+
"llm_available": config.LLM_AVAILABLE,
|
| 90 |
+
"llm_provider": config.LLM_PROVIDER if config.LLM_AVAILABLE else None,
|
| 91 |
}
|
| 92 |
|
| 93 |
|
|
|
|
| 120 |
detail=f"Unsupported file type '{ext or 'unknown'}'. Allowed types: PDF, TXT.",
|
| 121 |
)
|
| 122 |
|
| 123 |
+
# Read the upload in bounded blocks and stop the moment it exceeds the limit,
|
| 124 |
+
# so an oversized body can't be fully materialised in memory before we reject
|
| 125 |
+
# it. (Starlette spools large bodies to a temp file, but an unbounded
|
| 126 |
+
# `file.read()` would still load the whole thing into one bytes object.)
|
| 127 |
+
limit = config.MAX_UPLOAD_BYTES
|
| 128 |
+
buf = bytearray()
|
| 129 |
+
while True:
|
| 130 |
+
block = await file.read(1024 * 1024) # 1 MB at a time
|
| 131 |
+
if not block:
|
| 132 |
+
break
|
| 133 |
+
buf.extend(block)
|
| 134 |
+
if len(buf) > limit:
|
| 135 |
+
mb = limit // (1024 * 1024)
|
| 136 |
+
raise HTTPException(status_code=413, detail=f"File too large (max {mb} MB).")
|
| 137 |
+
raw = bytes(buf)
|
| 138 |
if not raw:
|
| 139 |
raise HTTPException(status_code=400, detail="Uploaded file is empty.")
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
logger.info("Upload received: %s (%d bytes)", original_name, len(raw))
|
| 142 |
|
backend/tests/test_answering.py
CHANGED
|
@@ -50,14 +50,52 @@ def test_abstains_on_empty_document():
|
|
| 50 |
assert citations == []
|
| 51 |
|
| 52 |
|
| 53 |
-
def
|
| 54 |
-
#
|
| 55 |
-
# the Fast (extractive) engine — still grounded,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
chunks = [make_chunk(INVOICE)]
|
| 57 |
answer, abstained, mode, citations = answer_question(
|
| 58 |
"What is the total amount due?", chunks, mode="thinking"
|
| 59 |
)
|
| 60 |
assert abstained is False
|
| 61 |
-
assert mode == "fast" # fell back because
|
| 62 |
assert "$2,413.98" in answer
|
| 63 |
assert len(citations) >= 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
assert citations == []
|
| 51 |
|
| 52 |
|
| 53 |
+
def test_thinking_mode_falls_back_to_fast_when_llm_unavailable(monkeypatch):
|
| 54 |
+
# When the LLM is unavailable (no key, or the call fails), Thinking mode must
|
| 55 |
+
# transparently fall back to the Fast (extractive) engine — still grounded,
|
| 56 |
+
# still cited, never broken. Patch synthesize -> None so the test is hermetic
|
| 57 |
+
# regardless of any provider configured in .env.
|
| 58 |
+
from app import llm
|
| 59 |
+
|
| 60 |
+
monkeypatch.setattr(llm, "synthesize", lambda question, contexts: None)
|
| 61 |
chunks = [make_chunk(INVOICE)]
|
| 62 |
answer, abstained, mode, citations = answer_question(
|
| 63 |
"What is the total amount due?", chunks, mode="thinking"
|
| 64 |
)
|
| 65 |
assert abstained is False
|
| 66 |
+
assert mode == "fast" # fell back because the LLM returned nothing
|
| 67 |
assert "$2,413.98" in answer
|
| 68 |
assert len(citations) >= 1
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_thinking_mode_uses_llm_answer_when_available(monkeypatch):
|
| 72 |
+
# When the LLM answers, Thinking mode returns that answer, reports
|
| 73 |
+
# mode="thinking", and still attaches the grounded citations.
|
| 74 |
+
from app import llm
|
| 75 |
+
|
| 76 |
+
monkeypatch.setattr(
|
| 77 |
+
llm, "synthesize", lambda q, c: "The total amount due is $2,413.98."
|
| 78 |
+
)
|
| 79 |
+
chunks = [make_chunk(INVOICE)]
|
| 80 |
+
answer, abstained, mode, citations = answer_question(
|
| 81 |
+
"What is the total amount due?", chunks, mode="thinking"
|
| 82 |
+
)
|
| 83 |
+
assert abstained is False
|
| 84 |
+
assert mode == "thinking"
|
| 85 |
+
assert "$2,413.98" in answer
|
| 86 |
+
assert len(citations) >= 1
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def test_thinking_mode_abstains_when_llm_says_so(monkeypatch):
|
| 90 |
+
# If the LLM itself returns the abstain message, that must surface as an
|
| 91 |
+
# abstention with no citations — never a confident-looking empty answer.
|
| 92 |
+
from app import llm
|
| 93 |
+
|
| 94 |
+
monkeypatch.setattr(llm, "synthesize", lambda q, c: config.ABSTAIN_MESSAGE)
|
| 95 |
+
chunks = [make_chunk(INVOICE)]
|
| 96 |
+
answer, abstained, mode, citations = answer_question(
|
| 97 |
+
"What is the total amount due?", chunks, mode="thinking"
|
| 98 |
+
)
|
| 99 |
+
assert abstained is True
|
| 100 |
+
assert mode == "thinking"
|
| 101 |
+
assert citations == []
|
backend/tests/test_api.py
CHANGED
|
@@ -43,6 +43,24 @@ def test_upload_rejects_empty_file(client):
|
|
| 43 |
assert res.status_code == 400
|
| 44 |
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
def _upload_invoice(client):
|
| 47 |
content = (
|
| 48 |
b"Invoice Number INV-2025-0473. Due Date April 2 2025. "
|
|
|
|
| 43 |
assert res.status_code == 400
|
| 44 |
|
| 45 |
|
| 46 |
+
def test_upload_rejects_oversized_file(client):
|
| 47 |
+
big = b"x" * (10 * 1024 * 1024 + 1) # one byte over the 10 MB limit
|
| 48 |
+
res = client.post(
|
| 49 |
+
"/documents/upload", files={"file": ("big.txt", big, "text/plain")}
|
| 50 |
+
)
|
| 51 |
+
assert res.status_code == 413
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_health_reports_llm_availability(client):
|
| 55 |
+
body = client.get("/health").json()
|
| 56 |
+
assert body["status"] == "ok"
|
| 57 |
+
assert body["default_mode"] == "fast"
|
| 58 |
+
assert isinstance(body["llm_available"], bool)
|
| 59 |
+
# The provider is only surfaced when an LLM is actually usable.
|
| 60 |
+
if not body["llm_available"]:
|
| 61 |
+
assert body["llm_provider"] is None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
def _upload_invoice(client):
|
| 65 |
content = (
|
| 66 |
b"Invoice Number INV-2025-0473. Due Date April 2 2025. "
|