RAG-Documind-AI / scripts /run_corpus_test.py
Shubham Pawar
Claude Sonnet 5
fix: make ingestion and querying robust; add acceptance-test corpus
c8f7f2e
Raw History Blame Contribute Delete
9.54 kB
"""
Acceptance test for DocuMind AI, driven through the public HTTP API.
It ingests the Nimbus Analytics test corpus (data/test_corpus/), then runs the
Q&A matrix in data/test_corpus/qa_pairs.yaml against POST /api/query and checks
that each answer contains the facts it should (and none of the facts it should
not). It also checks a few behaviours that have nothing to do with any single
answer: rejection of unsupported file types, collection isolation, and the
streaming endpoint.
Usage:
# 1. start the backend in another terminal:
# uvicorn src.api.main:app --port 8000
# 2. run:
./venv/bin/python scripts/run_corpus_test.py
Environment variables:
DOCUMIND_API_URL base URL of the backend (default http://localhost:8000)
NIMBUS_TEST_API_KEY LLM API key to pass as BYOK (default: none -> server key)
NIMBUS_TEST_PROVIDER gemini | openai | anthropic | groq (default gemini)
SKIP_INGEST=1 reuse collections from a previous run
"""
from __future__ import annotations
import os
import sys
import json
import time
from pathlib import Path
import requests
import yaml
ROOT = Path(__file__).resolve().parents[1]
CORPUS_DIR = ROOT / "data" / "test_corpus"
QA_FILE = CORPUS_DIR / "qa_pairs.yaml"
API_URL = os.getenv("DOCUMIND_API_URL", "http://localhost:8000").rstrip("/")
API_KEY = os.getenv("NIMBUS_TEST_API_KEY") or None
PROVIDER = os.getenv("NIMBUS_TEST_PROVIDER", "gemini")
SKIP_INGEST = os.getenv("SKIP_INGEST") == "1"
NIMBUS_COLLECTION = "test_corpus"
ACME_COLLECTION = "tenant_acme"
GREEN, RED, YELLOW, DIM, RESET = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m"
results: list[tuple[str, bool, str]] = []
def record(name: str, passed: bool, detail: str = "") -> None:
results.append((name, passed, detail))
mark = f"{GREEN}PASS{RESET}" if passed else f"{RED}FAIL{RESET}"
line = f" [{mark}] {name}"
if detail:
line += f"\n {DIM}{detail}{RESET}"
print(line)
def section(title: str) -> None:
print(f"\n{title}\n" + "-" * len(title))
# --------------------------------------------------------------------------- #
# Ingestion
# --------------------------------------------------------------------------- #
def ingest(paths: list[Path], collection: str) -> dict:
files = [("files", (p.name, p.read_bytes())) for p in paths]
resp = requests.post(
f"{API_URL}/api/ingest",
files=files,
data={"collection_name": collection},
timeout=300,
)
resp.raise_for_status()
return resp.json()
def do_ingestion() -> None:
section("Ingestion")
# Corpus docs are the numbered files (01_..08_); TEST_PLAN.md etc. are not corpus.
nimbus_files = sorted(
p for p in CORPUS_DIR.iterdir()
if p.is_file()
and p.suffix.lower() in {".md", ".txt", ".pdf"}
and p.name[:1].isdigit()
)
try:
info = ingest(nimbus_files, NIMBUS_COLLECTION)
ok = info.get("success") and info.get("chunks_created", 0) > 0
record(
"ingest test_corpus",
ok,
f"{info.get('documents_processed')} docs -> {info.get('chunks_created')} chunks "
f"from {len(nimbus_files)} files",
)
if info.get("documents_processed", 0) < len(nimbus_files):
record(
"all corpus files ingested",
False,
f"only {info.get('documents_processed')} of {len(nimbus_files)} "
"files produced documents; the loader skipped some",
)
except Exception as e: # noqa: BLE001
record("ingest test_corpus", False, repr(e))
try:
info = ingest([CORPUS_DIR / "tenant_acme" / "acme_handbook.md"], ACME_COLLECTION)
record("ingest tenant_acme", bool(info.get("success")), json.dumps(info))
except Exception as e: # noqa: BLE001
record("ingest tenant_acme", False, repr(e))
# Unsupported file type must be rejected with 400.
csv_path = CORPUS_DIR / "unsupported" / "notes.csv"
try:
resp = requests.post(
f"{API_URL}/api/ingest",
files=[("files", (csv_path.name, csv_path.read_bytes()))],
data={"collection_name": "should_not_exist"},
timeout=60,
)
record(
"reject unsupported file type (.csv -> 400)",
resp.status_code == 400,
f"got HTTP {resp.status_code}",
)
except Exception as e: # noqa: BLE001
record("reject unsupported file type (.csv -> 400)", False, repr(e))
# --------------------------------------------------------------------------- #
# Querying
# --------------------------------------------------------------------------- #
def query(question: str, collection: str, top_k: int = 8) -> dict:
payload = {
"question": question,
"collection_name": collection,
"top_k": top_k,
"include_sources": True,
"provider": PROVIDER,
}
if API_KEY:
payload["api_key"] = API_KEY
# Free provider tiers rate-limit aggressively; retry a few times on 429/503.
last_exc = None
for attempt in range(4):
resp = requests.post(f"{API_URL}/api/query", json=payload, timeout=120)
if resp.status_code in (429, 503):
last_exc = requests.HTTPError(f"{resp.status_code} from /api/query")
time.sleep(8 * (attempt + 1))
continue
resp.raise_for_status()
return resp.json()
raise last_exc
# Normalise typographic dashes and collapse whitespace so keyword checks are not
# defeated by an LLM emitting "AES‑256" instead of "AES-256".
_DASHES = str.maketrans({"‐": "-", "‑": "-", "‒": "-", "–": "-",
"—": "-", "−": "-", "­": "-"})
def _norm(s: str) -> str:
return " ".join(str(s).translate(_DASHES).lower().split())
def _contains(haystack: str, needle: str) -> bool:
return _norm(needle) in _norm(haystack)
def evaluate_case(case: dict) -> None:
name = f"{case['id']} ({case.get('category', '?')})"
try:
data = query(case["question"], case["collection"])
except Exception as e: # noqa: BLE001
record(name, False, f"query error: {e!r}")
return
answer = data.get("answer", "") or ""
problems: list[str] = []
for kw in case.get("keywords_all", []):
if not _contains(answer, str(kw)):
problems.append(f"missing required '{kw}'")
any_kws = case.get("keywords_any", [])
if any_kws and not any(_contains(answer, str(kw)) for kw in any_kws):
problems.append(f"none of {any_kws} present")
for kw in case.get("must_not_contain", []):
if _contains(answer, str(kw)):
problems.append(f"contains forbidden '{kw}'")
behavior = case.get("expected_behavior")
passed = not problems
detail = "; ".join(problems) if problems else f"answer: {answer[:160]}"
if behavior and passed:
detail = f"[manual check: {behavior}] answer: {answer[:140]}"
record(name, passed, detail)
def do_queries() -> None:
spec = yaml.safe_load(QA_FILE.read_text())
cases = spec["cases"]
section(f"Q&A matrix ({len(cases)} cases)")
for case in cases:
evaluate_case(case)
time.sleep(float(os.getenv("NIMBUS_TEST_DELAY", "1")))
# --------------------------------------------------------------------------- #
# Streaming
# --------------------------------------------------------------------------- #
def do_streaming() -> None:
section("Streaming endpoint")
payload = {
"question": "What is Nimbus Analytics?",
"collection_name": NIMBUS_COLLECTION,
"provider": PROVIDER,
}
if API_KEY:
payload["api_key"] = API_KEY
try:
with requests.post(
f"{API_URL}/api/query/stream", json=payload, stream=True, timeout=120
) as resp:
resp.raise_for_status()
chunks = []
for raw in resp.iter_lines():
if raw and raw.startswith(b"data: "):
chunks.append(raw[6:].decode("utf-8", "replace"))
got_done = "[DONE]" in chunks
body = "".join(c for c in chunks if c != "[DONE]")
record(
"stream returns SSE chunks + [DONE]",
got_done and len(body) > 0,
f"{len(chunks)} chunks, {len(body)} chars, done={got_done}",
)
except Exception as e: # noqa: BLE001
record("stream returns SSE chunks + [DONE]", False, repr(e))
# --------------------------------------------------------------------------- #
def main() -> int:
print(f"DocuMind acceptance test -> {API_URL}")
print(f"provider={PROVIDER} byok_key={'set' if API_KEY else 'server default'}")
try:
requests.get(f"{API_URL}/health", timeout=10).raise_for_status()
except Exception as e: # noqa: BLE001
print(f"{RED}Backend not reachable at {API_URL}: {e}{RESET}")
print("Start it with: uvicorn src.api.main:app --port 8000")
return 2
if not SKIP_INGEST:
do_ingestion()
time.sleep(1)
do_queries()
do_streaming()
total = len(results)
failed = [r for r in results if not r[1]]
section("Summary")
print(f" {total - len(failed)}/{total} passed")
if failed:
print(f"\n {RED}Failures:{RESET}")
for n, _, d in failed:
print(f" - {n}: {d}")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())