Spaces:
Sleeping
Sleeping
File size: 10,328 Bytes
e803935 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | """
NeuroCheck Agentic AI β RAG Knowledge Base
Embeds the 150 MS cases from the docx + MS clinical guidelines into ChromaDB.
Uses local sentence-transformers (no API key needed).
"""
from __future__ import annotations
import os, zipfile, re
from pathlib import Path
from typing import List
from loguru import logger
import chromadb
from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
# ββ Paths βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
BASE_DIR = Path(__file__).parent.parent
DOCX_PATH = BASE_DIR / "AGENTIC AI chatbots.docx"
CHROMA_DIR = Path(os.environ.get("CHROMA_DB_PATH", str(BASE_DIR / "chroma_db")))
# ββ Embedding model (local, no API key) ββββββββββββββββββββββββββββββββββββββ
EMBED_MODEL = "all-MiniLM-L6-v2" # ~90MB, fast, good for clinical text
_client: chromadb.PersistentClient | None = None
_collection = None
def get_collection():
global _client, _collection
if _collection is not None:
return _collection
ef = SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL)
_client = chromadb.PersistentClient(path=str(CHROMA_DIR))
_collection = _client.get_or_create_collection(
name="ms_knowledge",
embedding_function=ef,
metadata={"hnsw:space": "cosine"},
)
return _collection
def build_knowledge_base(force: bool = False) -> None:
"""
Build the ChromaDB vector store from the docx file.
Only runs if the collection is empty (or force=True).
"""
col = get_collection()
if col.count() > 0 and not force:
logger.info(f"[RAG] Knowledge base already built ({col.count()} chunks). Skipping.")
return
logger.info("[RAG] Building knowledge base from docx...")
docs = _extract_docx(DOCX_PATH)
docs += _ms_guidelines() # inject hard-coded clinical knowledge
chunks = _chunk_documents(docs)
ids = [f"doc_{i}" for i in range(len(chunks))]
texts = [c["text"] for c in chunks]
metas = [c["meta"] for c in chunks]
# Upsert in batches of 100
for i in range(0, len(texts), 100):
col.upsert(
ids=ids[i:i+100],
documents=texts[i:i+100],
metadatas=metas[i:i+100],
)
logger.info(f"[RAG] Knowledge base ready: {col.count()} chunks indexed.")
def query(text: str, n_results: int = 5) -> List[dict]:
"""Return top-n relevant knowledge chunks for a given query."""
col = get_collection()
if col.count() == 0:
return []
results = col.query(query_texts=[text], n_results=min(n_results, col.count()))
output = []
for doc, meta, dist in zip(
results["documents"][0],
results["metadatas"][0],
results["distances"][0],
):
output.append({"text": doc, "source": meta.get("source", ""), "score": round(1 - dist, 3)})
return output
# ββ Document extraction βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def _extract_docx(path: Path) -> List[dict]:
if not path.exists():
logger.warning(f"[RAG] Docx not found at {path}")
return []
import xml.etree.ElementTree as ET
docs = []
with zipfile.ZipFile(str(path)) as z:
xml = z.read("word/document.xml")
root = ET.fromstring(xml)
ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
paras = []
for para in root.iter(f"{ns}p"):
line = "".join(node.text or "" for node in para.iter(f"{ns}t")).strip()
if line:
paras.append(line)
# Group into logical blocks of 3 paragraphs for context
for i in range(0, len(paras), 3):
block = " ".join(paras[i:i+3])
docs.append({"text": block, "meta": {"source": "AGENTIC_AI_chatbots.docx"}})
return docs
def _ms_guidelines() -> List[dict]:
"""Hard-coded authoritative MS clinical guidelines as RAG knowledge."""
guidelines = [
{
"text": (
"McDonald Criteria 2024: MS diagnosis requires dissemination in space (DIS) "
"and dissemination in time (DIT). DIS: lesions in β₯2 CNS regions (periventricular, "
"cortical/juxtacortical, infratentorial, spinal cord, optic nerve). "
"DIT: new T2/gadolinium lesion on follow-up MRI, or simultaneous enhancing + non-enhancing lesions."
),
"meta": {"source": "McDonald_Criteria_2024"},
},
{
"text": (
"MS Workup Standard: MRI orbits with/without contrast for optic nerve involvement. "
"MRI brain and spinal cord with/without contrast. Lumbar puncture: CSF oligoclonal bands "
">2 exclusive to CSF, elevated IgG index, kappa free light chains. "
"Serum cell-based assay for NMO (AQP4-IgG) and MOGAD (MOG-IgG). "
"Exclude mimics: sarcoidosis, lupus, vasculitis, B12 deficiency, cervical myelopathy."
),
"meta": {"source": "MS_Workup_Protocol"},
},
{
"text": (
"Optic Neuritis classic triad: monocular visual loss + pain on eye movement + dyschromatopsia "
"(red colour desaturation). Peak age 20-45. Female predominance. "
"Uhthoff phenomenon: vision worsens with heat/exercise. "
"Recovery in 85% within 3 months. Risk of MS after ON: 50% at 15 years."
),
"meta": {"source": "NINDS_Optic_Neuritis"},
},
{
"text": (
"NMOSD (Neuromyelitis Optica Spectrum Disorder): AQP4-IgG positive. "
"Area postrema syndrome: intractable hiccups + nausea/vomiting. "
"LETM: longitudinally extensive transverse myelitis >3 vertebral segments. "
"Bilateral/severe optic neuritis. Periependymal brain lesions. "
"Distinguished from MS by AQP4-IgG serum test (cell-based assay)."
),
"meta": {"source": "NMOSD_Guidelines"},
},
{
"text": (
"MOGAD (MOG-Antibody Associated Disease): MOG-IgG positive. "
"ADEM-like presentation with large hazy confluent white matter lesions. "
"Optic perineuritis (sheath enhancement, not nerve). Conus medullaris involvement. "
"Often affects younger patients. Differentiated by MOG-IgG cell-based assay."
),
"meta": {"source": "MOGAD_Guidelines"},
},
{
"text": (
"Lhermitte's Sign: Electric shock sensation down spine on neck flexion. "
"Pathognomonic for cervical cord demyelination. "
"Uhthoff Phenomenon: worsening of any MS symptom with heat, exercise, fever, hot bath. "
"MS Hug: tight band sensation around trunk/chest β thoracic cord lesion. "
"These are anchor features requiring urgent neurology referral."
),
"meta": {"source": "MS_Anchor_Symptoms"},
},
{
"text": (
"IIH vs MS in obese patients: IIH (idiopathic intracranial hypertension) "
"typically presents with papilledema, headache, pulsatile tinnitus in obese females. "
"However, MS can coexist. If colour vision is impaired (dyschromatopsia) in IIH, "
"suspect concurrent optic neuritis. Heavy legs, motor symptoms, or prior transient "
"neurological episodes in obese patients must trigger MS workup."
),
"meta": {"source": "IIH_MS_Overlap"},
},
{
"text": (
"CSF Analysis in MS: Oligoclonal bands (OCBs) >2 exclusive to CSF (not serum) β "
"present in 85-95% of MS. IgG index >0.7 (elevated). IgG synthesis rate elevated. "
"Kappa free light chains substitute for OCBs where OCB testing unavailable. "
"Cell count <50 lymphocytes. Glucose and protein usually normal. "
"Paired serum-CSF sample essential for OCB interpretation."
),
"meta": {"source": "CSF_MS_Protocol"},
},
{
"text": (
"Internuclear Ophthalmoplegia (INO): failure of ipsilateral eye adduction on lateral gaze "
"with contralateral eye nystagmus. Caused by MLF lesion. Bilateral INO pathognomonic for MS. "
"Sixth nerve palsy: lateral gaze palsy. Skew deviation: vertical misalignment of eyes. "
"Any new diplopia in young adult β MS must be excluded."
),
"meta": {"source": "MS_Eye_Movement_Disorders"},
},
{
"text": (
"Trigeminal Neuralgia in young adults (<45): electric shock face/jaw pain. "
"When occurring in young patients without vascular loop compression, MS pons lesion "
"must be excluded. MRI brain with specific pons protocol required. "
"Facial numbness + diplopia combination suggests brainstem MS lesion."
),
"meta": {"source": "TN_MS_Brainstem"},
},
]
return guidelines
def _chunk_documents(docs: List[dict], max_len: int = 400) -> List[dict]:
"""Split long texts into overlapping chunks."""
out = []
for d in docs:
text = d["text"]
if len(text) <= max_len:
out.append(d)
else:
# Split by sentence boundaries
sentences = re.split(r"(?<=[.!?])\s+", text)
chunk, chunk_len = [], 0
for s in sentences:
if chunk_len + len(s) > max_len and chunk:
out.append({"text": " ".join(chunk), "meta": d["meta"]})
chunk, chunk_len = [], 0
chunk.append(s)
chunk_len += len(s)
if chunk:
out.append({"text": " ".join(chunk), "meta": d["meta"]})
return out
|