""" NeuroCheck Agentic AI — RAG Knowledge Base Embeds the 150 MS cases from the docx + MS clinical guidelines into ChromaDB. Uses local sentence-transformers (no API key needed). """ from __future__ import annotations import os, zipfile, re from pathlib import Path from typing import List from loguru import logger import chromadb from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction # ── Paths ───────────────────────────────────────────────────────────────────── BASE_DIR = Path(__file__).parent.parent DOCX_PATH = BASE_DIR / "AGENTIC AI chatbots.docx" CHROMA_DIR = Path(os.environ.get("CHROMA_DB_PATH", str(BASE_DIR / "chroma_db"))) # ── Embedding model (local, no API key) ────────────────────────────────────── EMBED_MODEL = "all-MiniLM-L6-v2" # ~90MB, fast, good for clinical text _client: chromadb.PersistentClient | None = None _collection = None def get_collection(): global _client, _collection if _collection is not None: return _collection ef = SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL) _client = chromadb.PersistentClient(path=str(CHROMA_DIR)) _collection = _client.get_or_create_collection( name="ms_knowledge", embedding_function=ef, metadata={"hnsw:space": "cosine"}, ) return _collection def build_knowledge_base(force: bool = False) -> None: """ Build the ChromaDB vector store from the docx file. Only runs if the collection is empty (or force=True). """ col = get_collection() if col.count() > 0 and not force: logger.info(f"[RAG] Knowledge base already built ({col.count()} chunks). Skipping.") return logger.info("[RAG] Building knowledge base from docx...") docs = _extract_docx(DOCX_PATH) docs += _ms_guidelines() # inject hard-coded clinical knowledge chunks = _chunk_documents(docs) ids = [f"doc_{i}" for i in range(len(chunks))] texts = [c["text"] for c in chunks] metas = [c["meta"] for c in chunks] # Upsert in batches of 100 for i in range(0, len(texts), 100): col.upsert( ids=ids[i:i+100], documents=texts[i:i+100], metadatas=metas[i:i+100], ) logger.info(f"[RAG] Knowledge base ready: {col.count()} chunks indexed.") def query(text: str, n_results: int = 5) -> List[dict]: """Return top-n relevant knowledge chunks for a given query.""" col = get_collection() if col.count() == 0: return [] results = col.query(query_texts=[text], n_results=min(n_results, col.count())) output = [] for doc, meta, dist in zip( results["documents"][0], results["metadatas"][0], results["distances"][0], ): output.append({"text": doc, "source": meta.get("source", ""), "score": round(1 - dist, 3)}) return output # ── Document extraction ─────────────────────────────────────────────────────── def _extract_docx(path: Path) -> List[dict]: if not path.exists(): logger.warning(f"[RAG] Docx not found at {path}") return [] import xml.etree.ElementTree as ET docs = [] with zipfile.ZipFile(str(path)) as z: xml = z.read("word/document.xml") root = ET.fromstring(xml) ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" paras = [] for para in root.iter(f"{ns}p"): line = "".join(node.text or "" for node in para.iter(f"{ns}t")).strip() if line: paras.append(line) # Group into logical blocks of 3 paragraphs for context for i in range(0, len(paras), 3): block = " ".join(paras[i:i+3]) docs.append({"text": block, "meta": {"source": "AGENTIC_AI_chatbots.docx"}}) return docs def _ms_guidelines() -> List[dict]: """Hard-coded authoritative MS clinical guidelines as RAG knowledge.""" guidelines = [ { "text": ( "McDonald Criteria 2024: MS diagnosis requires dissemination in space (DIS) " "and dissemination in time (DIT). DIS: lesions in ≥2 CNS regions (periventricular, " "cortical/juxtacortical, infratentorial, spinal cord, optic nerve). " "DIT: new T2/gadolinium lesion on follow-up MRI, or simultaneous enhancing + non-enhancing lesions." ), "meta": {"source": "McDonald_Criteria_2024"}, }, { "text": ( "MS Workup Standard: MRI orbits with/without contrast for optic nerve involvement. " "MRI brain and spinal cord with/without contrast. Lumbar puncture: CSF oligoclonal bands " ">2 exclusive to CSF, elevated IgG index, kappa free light chains. " "Serum cell-based assay for NMO (AQP4-IgG) and MOGAD (MOG-IgG). " "Exclude mimics: sarcoidosis, lupus, vasculitis, B12 deficiency, cervical myelopathy." ), "meta": {"source": "MS_Workup_Protocol"}, }, { "text": ( "Optic Neuritis classic triad: monocular visual loss + pain on eye movement + dyschromatopsia " "(red colour desaturation). Peak age 20-45. Female predominance. " "Uhthoff phenomenon: vision worsens with heat/exercise. " "Recovery in 85% within 3 months. Risk of MS after ON: 50% at 15 years." ), "meta": {"source": "NINDS_Optic_Neuritis"}, }, { "text": ( "NMOSD (Neuromyelitis Optica Spectrum Disorder): AQP4-IgG positive. " "Area postrema syndrome: intractable hiccups + nausea/vomiting. " "LETM: longitudinally extensive transverse myelitis >3 vertebral segments. " "Bilateral/severe optic neuritis. Periependymal brain lesions. " "Distinguished from MS by AQP4-IgG serum test (cell-based assay)." ), "meta": {"source": "NMOSD_Guidelines"}, }, { "text": ( "MOGAD (MOG-Antibody Associated Disease): MOG-IgG positive. " "ADEM-like presentation with large hazy confluent white matter lesions. " "Optic perineuritis (sheath enhancement, not nerve). Conus medullaris involvement. " "Often affects younger patients. Differentiated by MOG-IgG cell-based assay." ), "meta": {"source": "MOGAD_Guidelines"}, }, { "text": ( "Lhermitte's Sign: Electric shock sensation down spine on neck flexion. " "Pathognomonic for cervical cord demyelination. " "Uhthoff Phenomenon: worsening of any MS symptom with heat, exercise, fever, hot bath. " "MS Hug: tight band sensation around trunk/chest — thoracic cord lesion. " "These are anchor features requiring urgent neurology referral." ), "meta": {"source": "MS_Anchor_Symptoms"}, }, { "text": ( "IIH vs MS in obese patients: IIH (idiopathic intracranial hypertension) " "typically presents with papilledema, headache, pulsatile tinnitus in obese females. " "However, MS can coexist. If colour vision is impaired (dyschromatopsia) in IIH, " "suspect concurrent optic neuritis. Heavy legs, motor symptoms, or prior transient " "neurological episodes in obese patients must trigger MS workup." ), "meta": {"source": "IIH_MS_Overlap"}, }, { "text": ( "CSF Analysis in MS: Oligoclonal bands (OCBs) >2 exclusive to CSF (not serum) — " "present in 85-95% of MS. IgG index >0.7 (elevated). IgG synthesis rate elevated. " "Kappa free light chains substitute for OCBs where OCB testing unavailable. " "Cell count <50 lymphocytes. Glucose and protein usually normal. " "Paired serum-CSF sample essential for OCB interpretation." ), "meta": {"source": "CSF_MS_Protocol"}, }, { "text": ( "Internuclear Ophthalmoplegia (INO): failure of ipsilateral eye adduction on lateral gaze " "with contralateral eye nystagmus. Caused by MLF lesion. Bilateral INO pathognomonic for MS. " "Sixth nerve palsy: lateral gaze palsy. Skew deviation: vertical misalignment of eyes. " "Any new diplopia in young adult — MS must be excluded." ), "meta": {"source": "MS_Eye_Movement_Disorders"}, }, { "text": ( "Trigeminal Neuralgia in young adults (<45): electric shock face/jaw pain. " "When occurring in young patients without vascular loop compression, MS pons lesion " "must be excluded. MRI brain with specific pons protocol required. " "Facial numbness + diplopia combination suggests brainstem MS lesion." ), "meta": {"source": "TN_MS_Brainstem"}, }, ] return guidelines def _chunk_documents(docs: List[dict], max_len: int = 400) -> List[dict]: """Split long texts into overlapping chunks.""" out = [] for d in docs: text = d["text"] if len(text) <= max_len: out.append(d) else: # Split by sentence boundaries sentences = re.split(r"(?<=[.!?])\s+", text) chunk, chunk_len = [], 0 for s in sentences: if chunk_len + len(s) > max_len and chunk: out.append({"text": " ".join(chunk), "meta": d["meta"]}) chunk, chunk_len = [], 0 chunk.append(s) chunk_len += len(s) if chunk: out.append({"text": " ".join(chunk), "meta": d["meta"]}) return out