Spaces:
Sleeping
Sleeping
Download api/rag.py from NeuroCheckAI/neurocheck: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/spaces/NeuroCheckAI/neurocheck/resolve/main/api/rag.py
- Command line
-
hf download hf://spaces/NeuroCheckAI/neurocheck/api/rag.py
-
curl -L -o rag.py https://huggingface.co/spaces/NeuroCheckAI/neurocheck/resolve/main/api/rag.py
10.3 kB
| """ | |
| NeuroCheck Agentic AI β RAG Knowledge Base | |
| Embeds the 150 MS cases from the docx + MS clinical guidelines into ChromaDB. | |
| Uses local sentence-transformers (no API key needed). | |
| """ | |
| from __future__ import annotations | |
| import os, zipfile, re | |
| from pathlib import Path | |
| from typing import List | |
| from loguru import logger | |
| import chromadb | |
| from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction | |
| # ββ Paths βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| BASE_DIR = Path(__file__).parent.parent | |
| DOCX_PATH = BASE_DIR / "AGENTIC AI chatbots.docx" | |
| CHROMA_DIR = Path(os.environ.get("CHROMA_DB_PATH", str(BASE_DIR / "chroma_db"))) | |
| # ββ Embedding model (local, no API key) ββββββββββββββββββββββββββββββββββββββ | |
| EMBED_MODEL = "all-MiniLM-L6-v2" # ~90MB, fast, good for clinical text | |
| _client: chromadb.PersistentClient | None = None | |
| _collection = None | |
| def get_collection(): | |
| global _client, _collection | |
| if _collection is not None: | |
| return _collection | |
| ef = SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL) | |
| _client = chromadb.PersistentClient(path=str(CHROMA_DIR)) | |
| _collection = _client.get_or_create_collection( | |
| name="ms_knowledge", | |
| embedding_function=ef, | |
| metadata={"hnsw:space": "cosine"}, | |
| ) | |
| return _collection | |
| def build_knowledge_base(force: bool = False) -> None: | |
| """ | |
| Build the ChromaDB vector store from the docx file. | |
| Only runs if the collection is empty (or force=True). | |
| """ | |
| col = get_collection() | |
| if col.count() > 0 and not force: | |
| logger.info(f"[RAG] Knowledge base already built ({col.count()} chunks). Skipping.") | |
| return | |
| logger.info("[RAG] Building knowledge base from docx...") | |
| docs = _extract_docx(DOCX_PATH) | |
| docs += _ms_guidelines() # inject hard-coded clinical knowledge | |
| chunks = _chunk_documents(docs) | |
| ids = [f"doc_{i}" for i in range(len(chunks))] | |
| texts = [c["text"] for c in chunks] | |
| metas = [c["meta"] for c in chunks] | |
| # Upsert in batches of 100 | |
| for i in range(0, len(texts), 100): | |
| col.upsert( | |
| ids=ids[i:i+100], | |
| documents=texts[i:i+100], | |
| metadatas=metas[i:i+100], | |
| ) | |
| logger.info(f"[RAG] Knowledge base ready: {col.count()} chunks indexed.") | |
| def query(text: str, n_results: int = 5) -> List[dict]: | |
| """Return top-n relevant knowledge chunks for a given query.""" | |
| col = get_collection() | |
| if col.count() == 0: | |
| return [] | |
| results = col.query(query_texts=[text], n_results=min(n_results, col.count())) | |
| output = [] | |
| for doc, meta, dist in zip( | |
| results["documents"][0], | |
| results["metadatas"][0], | |
| results["distances"][0], | |
| ): | |
| output.append({"text": doc, "source": meta.get("source", ""), "score": round(1 - dist, 3)}) | |
| return output | |
| # ββ Document extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _extract_docx(path: Path) -> List[dict]: | |
| if not path.exists(): | |
| logger.warning(f"[RAG] Docx not found at {path}") | |
| return [] | |
| import xml.etree.ElementTree as ET | |
| docs = [] | |
| with zipfile.ZipFile(str(path)) as z: | |
| xml = z.read("word/document.xml") | |
| root = ET.fromstring(xml) | |
| ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" | |
| paras = [] | |
| for para in root.iter(f"{ns}p"): | |
| line = "".join(node.text or "" for node in para.iter(f"{ns}t")).strip() | |
| if line: | |
| paras.append(line) | |
| # Group into logical blocks of 3 paragraphs for context | |
| for i in range(0, len(paras), 3): | |
| block = " ".join(paras[i:i+3]) | |
| docs.append({"text": block, "meta": {"source": "AGENTIC_AI_chatbots.docx"}}) | |
| return docs | |
| def _ms_guidelines() -> List[dict]: | |
| """Hard-coded authoritative MS clinical guidelines as RAG knowledge.""" | |
| guidelines = [ | |
| { | |
| "text": ( | |
| "McDonald Criteria 2024: MS diagnosis requires dissemination in space (DIS) " | |
| "and dissemination in time (DIT). DIS: lesions in β₯2 CNS regions (periventricular, " | |
| "cortical/juxtacortical, infratentorial, spinal cord, optic nerve). " | |
| "DIT: new T2/gadolinium lesion on follow-up MRI, or simultaneous enhancing + non-enhancing lesions." | |
| ), | |
| "meta": {"source": "McDonald_Criteria_2024"}, | |
| }, | |
| { | |
| "text": ( | |
| "MS Workup Standard: MRI orbits with/without contrast for optic nerve involvement. " | |
| "MRI brain and spinal cord with/without contrast. Lumbar puncture: CSF oligoclonal bands " | |
| ">2 exclusive to CSF, elevated IgG index, kappa free light chains. " | |
| "Serum cell-based assay for NMO (AQP4-IgG) and MOGAD (MOG-IgG). " | |
| "Exclude mimics: sarcoidosis, lupus, vasculitis, B12 deficiency, cervical myelopathy." | |
| ), | |
| "meta": {"source": "MS_Workup_Protocol"}, | |
| }, | |
| { | |
| "text": ( | |
| "Optic Neuritis classic triad: monocular visual loss + pain on eye movement + dyschromatopsia " | |
| "(red colour desaturation). Peak age 20-45. Female predominance. " | |
| "Uhthoff phenomenon: vision worsens with heat/exercise. " | |
| "Recovery in 85% within 3 months. Risk of MS after ON: 50% at 15 years." | |
| ), | |
| "meta": {"source": "NINDS_Optic_Neuritis"}, | |
| }, | |
| { | |
| "text": ( | |
| "NMOSD (Neuromyelitis Optica Spectrum Disorder): AQP4-IgG positive. " | |
| "Area postrema syndrome: intractable hiccups + nausea/vomiting. " | |
| "LETM: longitudinally extensive transverse myelitis >3 vertebral segments. " | |
| "Bilateral/severe optic neuritis. Periependymal brain lesions. " | |
| "Distinguished from MS by AQP4-IgG serum test (cell-based assay)." | |
| ), | |
| "meta": {"source": "NMOSD_Guidelines"}, | |
| }, | |
| { | |
| "text": ( | |
| "MOGAD (MOG-Antibody Associated Disease): MOG-IgG positive. " | |
| "ADEM-like presentation with large hazy confluent white matter lesions. " | |
| "Optic perineuritis (sheath enhancement, not nerve). Conus medullaris involvement. " | |
| "Often affects younger patients. Differentiated by MOG-IgG cell-based assay." | |
| ), | |
| "meta": {"source": "MOGAD_Guidelines"}, | |
| }, | |
| { | |
| "text": ( | |
| "Lhermitte's Sign: Electric shock sensation down spine on neck flexion. " | |
| "Pathognomonic for cervical cord demyelination. " | |
| "Uhthoff Phenomenon: worsening of any MS symptom with heat, exercise, fever, hot bath. " | |
| "MS Hug: tight band sensation around trunk/chest β thoracic cord lesion. " | |
| "These are anchor features requiring urgent neurology referral." | |
| ), | |
| "meta": {"source": "MS_Anchor_Symptoms"}, | |
| }, | |
| { | |
| "text": ( | |
| "IIH vs MS in obese patients: IIH (idiopathic intracranial hypertension) " | |
| "typically presents with papilledema, headache, pulsatile tinnitus in obese females. " | |
| "However, MS can coexist. If colour vision is impaired (dyschromatopsia) in IIH, " | |
| "suspect concurrent optic neuritis. Heavy legs, motor symptoms, or prior transient " | |
| "neurological episodes in obese patients must trigger MS workup." | |
| ), | |
| "meta": {"source": "IIH_MS_Overlap"}, | |
| }, | |
| { | |
| "text": ( | |
| "CSF Analysis in MS: Oligoclonal bands (OCBs) >2 exclusive to CSF (not serum) β " | |
| "present in 85-95% of MS. IgG index >0.7 (elevated). IgG synthesis rate elevated. " | |
| "Kappa free light chains substitute for OCBs where OCB testing unavailable. " | |
| "Cell count <50 lymphocytes. Glucose and protein usually normal. " | |
| "Paired serum-CSF sample essential for OCB interpretation." | |
| ), | |
| "meta": {"source": "CSF_MS_Protocol"}, | |
| }, | |
| { | |
| "text": ( | |
| "Internuclear Ophthalmoplegia (INO): failure of ipsilateral eye adduction on lateral gaze " | |
| "with contralateral eye nystagmus. Caused by MLF lesion. Bilateral INO pathognomonic for MS. " | |
| "Sixth nerve palsy: lateral gaze palsy. Skew deviation: vertical misalignment of eyes. " | |
| "Any new diplopia in young adult β MS must be excluded." | |
| ), | |
| "meta": {"source": "MS_Eye_Movement_Disorders"}, | |
| }, | |
| { | |
| "text": ( | |
| "Trigeminal Neuralgia in young adults (<45): electric shock face/jaw pain. " | |
| "When occurring in young patients without vascular loop compression, MS pons lesion " | |
| "must be excluded. MRI brain with specific pons protocol required. " | |
| "Facial numbness + diplopia combination suggests brainstem MS lesion." | |
| ), | |
| "meta": {"source": "TN_MS_Brainstem"}, | |
| }, | |
| ] | |
| return guidelines | |
| def _chunk_documents(docs: List[dict], max_len: int = 400) -> List[dict]: | |
| """Split long texts into overlapping chunks.""" | |
| out = [] | |
| for d in docs: | |
| text = d["text"] | |
| if len(text) <= max_len: | |
| out.append(d) | |
| else: | |
| # Split by sentence boundaries | |
| sentences = re.split(r"(?<=[.!?])\s+", text) | |
| chunk, chunk_len = [], 0 | |
| for s in sentences: | |
| if chunk_len + len(s) > max_len and chunk: | |
| out.append({"text": " ".join(chunk), "meta": d["meta"]}) | |
| chunk, chunk_len = [], 0 | |
| chunk.append(s) | |
| chunk_len += len(s) | |
| if chunk: | |
| out.append({"text": " ".join(chunk), "meta": d["meta"]}) | |
| return out | |