Spaces:
Build error
Build error
Shubham Pawar
Claude Sonnet 5
fix: make ingestion and querying robust; add acceptance-test corpus
c8f7f2e Download scripts/run_corpus_test.py from AIenthusSP/RAG-Documind-AI: direct link, hf CLI and curl.
- Browser
- Download file 9.54 kB
-
https://huggingface.co/spaces/AIenthusSP/RAG-Documind-AI/resolve/main/scripts/run_corpus_test.py
- Command line
-
hf download hf://spaces/AIenthusSP/RAG-Documind-AI/scripts/run_corpus_test.py
-
curl -L -o run_corpus_test.py https://huggingface.co/spaces/AIenthusSP/RAG-Documind-AI/resolve/main/scripts/run_corpus_test.py
9.54 kB
| """ | |
| Acceptance test for DocuMind AI, driven through the public HTTP API. | |
| It ingests the Nimbus Analytics test corpus (data/test_corpus/), then runs the | |
| Q&A matrix in data/test_corpus/qa_pairs.yaml against POST /api/query and checks | |
| that each answer contains the facts it should (and none of the facts it should | |
| not). It also checks a few behaviours that have nothing to do with any single | |
| answer: rejection of unsupported file types, collection isolation, and the | |
| streaming endpoint. | |
| Usage: | |
| # 1. start the backend in another terminal: | |
| # uvicorn src.api.main:app --port 8000 | |
| # 2. run: | |
| ./venv/bin/python scripts/run_corpus_test.py | |
| Environment variables: | |
| DOCUMIND_API_URL base URL of the backend (default http://localhost:8000) | |
| NIMBUS_TEST_API_KEY LLM API key to pass as BYOK (default: none -> server key) | |
| NIMBUS_TEST_PROVIDER gemini | openai | anthropic | groq (default gemini) | |
| SKIP_INGEST=1 reuse collections from a previous run | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| import json | |
| import time | |
| from pathlib import Path | |
| import requests | |
| import yaml | |
| ROOT = Path(__file__).resolve().parents[1] | |
| CORPUS_DIR = ROOT / "data" / "test_corpus" | |
| QA_FILE = CORPUS_DIR / "qa_pairs.yaml" | |
| API_URL = os.getenv("DOCUMIND_API_URL", "http://localhost:8000").rstrip("/") | |
| API_KEY = os.getenv("NIMBUS_TEST_API_KEY") or None | |
| PROVIDER = os.getenv("NIMBUS_TEST_PROVIDER", "gemini") | |
| SKIP_INGEST = os.getenv("SKIP_INGEST") == "1" | |
| NIMBUS_COLLECTION = "test_corpus" | |
| ACME_COLLECTION = "tenant_acme" | |
| GREEN, RED, YELLOW, DIM, RESET = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m" | |
| results: list[tuple[str, bool, str]] = [] | |
| def record(name: str, passed: bool, detail: str = "") -> None: | |
| results.append((name, passed, detail)) | |
| mark = f"{GREEN}PASS{RESET}" if passed else f"{RED}FAIL{RESET}" | |
| line = f" [{mark}] {name}" | |
| if detail: | |
| line += f"\n {DIM}{detail}{RESET}" | |
| print(line) | |
| def section(title: str) -> None: | |
| print(f"\n{title}\n" + "-" * len(title)) | |
| # --------------------------------------------------------------------------- # | |
| # Ingestion | |
| # --------------------------------------------------------------------------- # | |
| def ingest(paths: list[Path], collection: str) -> dict: | |
| files = [("files", (p.name, p.read_bytes())) for p in paths] | |
| resp = requests.post( | |
| f"{API_URL}/api/ingest", | |
| files=files, | |
| data={"collection_name": collection}, | |
| timeout=300, | |
| ) | |
| resp.raise_for_status() | |
| return resp.json() | |
| def do_ingestion() -> None: | |
| section("Ingestion") | |
| # Corpus docs are the numbered files (01_..08_); TEST_PLAN.md etc. are not corpus. | |
| nimbus_files = sorted( | |
| p for p in CORPUS_DIR.iterdir() | |
| if p.is_file() | |
| and p.suffix.lower() in {".md", ".txt", ".pdf"} | |
| and p.name[:1].isdigit() | |
| ) | |
| try: | |
| info = ingest(nimbus_files, NIMBUS_COLLECTION) | |
| ok = info.get("success") and info.get("chunks_created", 0) > 0 | |
| record( | |
| "ingest test_corpus", | |
| ok, | |
| f"{info.get('documents_processed')} docs -> {info.get('chunks_created')} chunks " | |
| f"from {len(nimbus_files)} files", | |
| ) | |
| if info.get("documents_processed", 0) < len(nimbus_files): | |
| record( | |
| "all corpus files ingested", | |
| False, | |
| f"only {info.get('documents_processed')} of {len(nimbus_files)} " | |
| "files produced documents; the loader skipped some", | |
| ) | |
| except Exception as e: # noqa: BLE001 | |
| record("ingest test_corpus", False, repr(e)) | |
| try: | |
| info = ingest([CORPUS_DIR / "tenant_acme" / "acme_handbook.md"], ACME_COLLECTION) | |
| record("ingest tenant_acme", bool(info.get("success")), json.dumps(info)) | |
| except Exception as e: # noqa: BLE001 | |
| record("ingest tenant_acme", False, repr(e)) | |
| # Unsupported file type must be rejected with 400. | |
| csv_path = CORPUS_DIR / "unsupported" / "notes.csv" | |
| try: | |
| resp = requests.post( | |
| f"{API_URL}/api/ingest", | |
| files=[("files", (csv_path.name, csv_path.read_bytes()))], | |
| data={"collection_name": "should_not_exist"}, | |
| timeout=60, | |
| ) | |
| record( | |
| "reject unsupported file type (.csv -> 400)", | |
| resp.status_code == 400, | |
| f"got HTTP {resp.status_code}", | |
| ) | |
| except Exception as e: # noqa: BLE001 | |
| record("reject unsupported file type (.csv -> 400)", False, repr(e)) | |
| # --------------------------------------------------------------------------- # | |
| # Querying | |
| # --------------------------------------------------------------------------- # | |
| def query(question: str, collection: str, top_k: int = 8) -> dict: | |
| payload = { | |
| "question": question, | |
| "collection_name": collection, | |
| "top_k": top_k, | |
| "include_sources": True, | |
| "provider": PROVIDER, | |
| } | |
| if API_KEY: | |
| payload["api_key"] = API_KEY | |
| # Free provider tiers rate-limit aggressively; retry a few times on 429/503. | |
| last_exc = None | |
| for attempt in range(4): | |
| resp = requests.post(f"{API_URL}/api/query", json=payload, timeout=120) | |
| if resp.status_code in (429, 503): | |
| last_exc = requests.HTTPError(f"{resp.status_code} from /api/query") | |
| time.sleep(8 * (attempt + 1)) | |
| continue | |
| resp.raise_for_status() | |
| return resp.json() | |
| raise last_exc | |
| # Normalise typographic dashes and collapse whitespace so keyword checks are not | |
| # defeated by an LLM emitting "AES‑256" instead of "AES-256". | |
| _DASHES = str.maketrans({"‐": "-", "‑": "-", "‒": "-", "–": "-", | |
| "—": "-", "−": "-", "": "-"}) | |
| def _norm(s: str) -> str: | |
| return " ".join(str(s).translate(_DASHES).lower().split()) | |
| def _contains(haystack: str, needle: str) -> bool: | |
| return _norm(needle) in _norm(haystack) | |
| def evaluate_case(case: dict) -> None: | |
| name = f"{case['id']} ({case.get('category', '?')})" | |
| try: | |
| data = query(case["question"], case["collection"]) | |
| except Exception as e: # noqa: BLE001 | |
| record(name, False, f"query error: {e!r}") | |
| return | |
| answer = data.get("answer", "") or "" | |
| problems: list[str] = [] | |
| for kw in case.get("keywords_all", []): | |
| if not _contains(answer, str(kw)): | |
| problems.append(f"missing required '{kw}'") | |
| any_kws = case.get("keywords_any", []) | |
| if any_kws and not any(_contains(answer, str(kw)) for kw in any_kws): | |
| problems.append(f"none of {any_kws} present") | |
| for kw in case.get("must_not_contain", []): | |
| if _contains(answer, str(kw)): | |
| problems.append(f"contains forbidden '{kw}'") | |
| behavior = case.get("expected_behavior") | |
| passed = not problems | |
| detail = "; ".join(problems) if problems else f"answer: {answer[:160]}" | |
| if behavior and passed: | |
| detail = f"[manual check: {behavior}] answer: {answer[:140]}" | |
| record(name, passed, detail) | |
| def do_queries() -> None: | |
| spec = yaml.safe_load(QA_FILE.read_text()) | |
| cases = spec["cases"] | |
| section(f"Q&A matrix ({len(cases)} cases)") | |
| for case in cases: | |
| evaluate_case(case) | |
| time.sleep(float(os.getenv("NIMBUS_TEST_DELAY", "1"))) | |
| # --------------------------------------------------------------------------- # | |
| # Streaming | |
| # --------------------------------------------------------------------------- # | |
| def do_streaming() -> None: | |
| section("Streaming endpoint") | |
| payload = { | |
| "question": "What is Nimbus Analytics?", | |
| "collection_name": NIMBUS_COLLECTION, | |
| "provider": PROVIDER, | |
| } | |
| if API_KEY: | |
| payload["api_key"] = API_KEY | |
| try: | |
| with requests.post( | |
| f"{API_URL}/api/query/stream", json=payload, stream=True, timeout=120 | |
| ) as resp: | |
| resp.raise_for_status() | |
| chunks = [] | |
| for raw in resp.iter_lines(): | |
| if raw and raw.startswith(b"data: "): | |
| chunks.append(raw[6:].decode("utf-8", "replace")) | |
| got_done = "[DONE]" in chunks | |
| body = "".join(c for c in chunks if c != "[DONE]") | |
| record( | |
| "stream returns SSE chunks + [DONE]", | |
| got_done and len(body) > 0, | |
| f"{len(chunks)} chunks, {len(body)} chars, done={got_done}", | |
| ) | |
| except Exception as e: # noqa: BLE001 | |
| record("stream returns SSE chunks + [DONE]", False, repr(e)) | |
| # --------------------------------------------------------------------------- # | |
| def main() -> int: | |
| print(f"DocuMind acceptance test -> {API_URL}") | |
| print(f"provider={PROVIDER} byok_key={'set' if API_KEY else 'server default'}") | |
| try: | |
| requests.get(f"{API_URL}/health", timeout=10).raise_for_status() | |
| except Exception as e: # noqa: BLE001 | |
| print(f"{RED}Backend not reachable at {API_URL}: {e}{RESET}") | |
| print("Start it with: uvicorn src.api.main:app --port 8000") | |
| return 2 | |
| if not SKIP_INGEST: | |
| do_ingestion() | |
| time.sleep(1) | |
| do_queries() | |
| do_streaming() | |
| total = len(results) | |
| failed = [r for r in results if not r[1]] | |
| section("Summary") | |
| print(f" {total - len(failed)}/{total} passed") | |
| if failed: | |
| print(f"\n {RED}Failures:{RESET}") | |
| for n, _, d in failed: | |
| print(f" - {n}: {d}") | |
| return 1 if failed else 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |