AxonIQ-Testing / api /rag.py
ZeroNinerOne's picture
Upload 12 files
e803935 verified
Raw History Blame Contribute Delete
10.3 kB
"""
NeuroCheck Agentic AI β€” RAG Knowledge Base
Embeds the 150 MS cases from the docx + MS clinical guidelines into ChromaDB.
Uses local sentence-transformers (no API key needed).
"""
from __future__ import annotations
import os, zipfile, re
from pathlib import Path
from typing import List
from loguru import logger
import chromadb
from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
# ── Paths ─────────────────────────────────────────────────────────────────────
BASE_DIR = Path(__file__).parent.parent
DOCX_PATH = BASE_DIR / "AGENTIC AI chatbots.docx"
CHROMA_DIR = Path(os.environ.get("CHROMA_DB_PATH", str(BASE_DIR / "chroma_db")))
# ── Embedding model (local, no API key) ──────────────────────────────────────
EMBED_MODEL = "all-MiniLM-L6-v2" # ~90MB, fast, good for clinical text
_client: chromadb.PersistentClient | None = None
_collection = None
def get_collection():
global _client, _collection
if _collection is not None:
return _collection
ef = SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL)
_client = chromadb.PersistentClient(path=str(CHROMA_DIR))
_collection = _client.get_or_create_collection(
name="ms_knowledge",
embedding_function=ef,
metadata={"hnsw:space": "cosine"},
)
return _collection
def build_knowledge_base(force: bool = False) -> None:
"""
Build the ChromaDB vector store from the docx file.
Only runs if the collection is empty (or force=True).
"""
col = get_collection()
if col.count() > 0 and not force:
logger.info(f"[RAG] Knowledge base already built ({col.count()} chunks). Skipping.")
return
logger.info("[RAG] Building knowledge base from docx...")
docs = _extract_docx(DOCX_PATH)
docs += _ms_guidelines() # inject hard-coded clinical knowledge
chunks = _chunk_documents(docs)
ids = [f"doc_{i}" for i in range(len(chunks))]
texts = [c["text"] for c in chunks]
metas = [c["meta"] for c in chunks]
# Upsert in batches of 100
for i in range(0, len(texts), 100):
col.upsert(
ids=ids[i:i+100],
documents=texts[i:i+100],
metadatas=metas[i:i+100],
)
logger.info(f"[RAG] Knowledge base ready: {col.count()} chunks indexed.")
def query(text: str, n_results: int = 5) -> List[dict]:
"""Return top-n relevant knowledge chunks for a given query."""
col = get_collection()
if col.count() == 0:
return []
results = col.query(query_texts=[text], n_results=min(n_results, col.count()))
output = []
for doc, meta, dist in zip(
results["documents"][0],
results["metadatas"][0],
results["distances"][0],
):
output.append({"text": doc, "source": meta.get("source", ""), "score": round(1 - dist, 3)})
return output
# ── Document extraction ───────────────────────────────────────────────────────
def _extract_docx(path: Path) -> List[dict]:
if not path.exists():
logger.warning(f"[RAG] Docx not found at {path}")
return []
import xml.etree.ElementTree as ET
docs = []
with zipfile.ZipFile(str(path)) as z:
xml = z.read("word/document.xml")
root = ET.fromstring(xml)
ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
paras = []
for para in root.iter(f"{ns}p"):
line = "".join(node.text or "" for node in para.iter(f"{ns}t")).strip()
if line:
paras.append(line)
# Group into logical blocks of 3 paragraphs for context
for i in range(0, len(paras), 3):
block = " ".join(paras[i:i+3])
docs.append({"text": block, "meta": {"source": "AGENTIC_AI_chatbots.docx"}})
return docs
def _ms_guidelines() -> List[dict]:
"""Hard-coded authoritative MS clinical guidelines as RAG knowledge."""
guidelines = [
{
"text": (
"McDonald Criteria 2024: MS diagnosis requires dissemination in space (DIS) "
"and dissemination in time (DIT). DIS: lesions in β‰₯2 CNS regions (periventricular, "
"cortical/juxtacortical, infratentorial, spinal cord, optic nerve). "
"DIT: new T2/gadolinium lesion on follow-up MRI, or simultaneous enhancing + non-enhancing lesions."
),
"meta": {"source": "McDonald_Criteria_2024"},
},
{
"text": (
"MS Workup Standard: MRI orbits with/without contrast for optic nerve involvement. "
"MRI brain and spinal cord with/without contrast. Lumbar puncture: CSF oligoclonal bands "
">2 exclusive to CSF, elevated IgG index, kappa free light chains. "
"Serum cell-based assay for NMO (AQP4-IgG) and MOGAD (MOG-IgG). "
"Exclude mimics: sarcoidosis, lupus, vasculitis, B12 deficiency, cervical myelopathy."
),
"meta": {"source": "MS_Workup_Protocol"},
},
{
"text": (
"Optic Neuritis classic triad: monocular visual loss + pain on eye movement + dyschromatopsia "
"(red colour desaturation). Peak age 20-45. Female predominance. "
"Uhthoff phenomenon: vision worsens with heat/exercise. "
"Recovery in 85% within 3 months. Risk of MS after ON: 50% at 15 years."
),
"meta": {"source": "NINDS_Optic_Neuritis"},
},
{
"text": (
"NMOSD (Neuromyelitis Optica Spectrum Disorder): AQP4-IgG positive. "
"Area postrema syndrome: intractable hiccups + nausea/vomiting. "
"LETM: longitudinally extensive transverse myelitis >3 vertebral segments. "
"Bilateral/severe optic neuritis. Periependymal brain lesions. "
"Distinguished from MS by AQP4-IgG serum test (cell-based assay)."
),
"meta": {"source": "NMOSD_Guidelines"},
},
{
"text": (
"MOGAD (MOG-Antibody Associated Disease): MOG-IgG positive. "
"ADEM-like presentation with large hazy confluent white matter lesions. "
"Optic perineuritis (sheath enhancement, not nerve). Conus medullaris involvement. "
"Often affects younger patients. Differentiated by MOG-IgG cell-based assay."
),
"meta": {"source": "MOGAD_Guidelines"},
},
{
"text": (
"Lhermitte's Sign: Electric shock sensation down spine on neck flexion. "
"Pathognomonic for cervical cord demyelination. "
"Uhthoff Phenomenon: worsening of any MS symptom with heat, exercise, fever, hot bath. "
"MS Hug: tight band sensation around trunk/chest β€” thoracic cord lesion. "
"These are anchor features requiring urgent neurology referral."
),
"meta": {"source": "MS_Anchor_Symptoms"},
},
{
"text": (
"IIH vs MS in obese patients: IIH (idiopathic intracranial hypertension) "
"typically presents with papilledema, headache, pulsatile tinnitus in obese females. "
"However, MS can coexist. If colour vision is impaired (dyschromatopsia) in IIH, "
"suspect concurrent optic neuritis. Heavy legs, motor symptoms, or prior transient "
"neurological episodes in obese patients must trigger MS workup."
),
"meta": {"source": "IIH_MS_Overlap"},
},
{
"text": (
"CSF Analysis in MS: Oligoclonal bands (OCBs) >2 exclusive to CSF (not serum) β€” "
"present in 85-95% of MS. IgG index >0.7 (elevated). IgG synthesis rate elevated. "
"Kappa free light chains substitute for OCBs where OCB testing unavailable. "
"Cell count <50 lymphocytes. Glucose and protein usually normal. "
"Paired serum-CSF sample essential for OCB interpretation."
),
"meta": {"source": "CSF_MS_Protocol"},
},
{
"text": (
"Internuclear Ophthalmoplegia (INO): failure of ipsilateral eye adduction on lateral gaze "
"with contralateral eye nystagmus. Caused by MLF lesion. Bilateral INO pathognomonic for MS. "
"Sixth nerve palsy: lateral gaze palsy. Skew deviation: vertical misalignment of eyes. "
"Any new diplopia in young adult β€” MS must be excluded."
),
"meta": {"source": "MS_Eye_Movement_Disorders"},
},
{
"text": (
"Trigeminal Neuralgia in young adults (<45): electric shock face/jaw pain. "
"When occurring in young patients without vascular loop compression, MS pons lesion "
"must be excluded. MRI brain with specific pons protocol required. "
"Facial numbness + diplopia combination suggests brainstem MS lesion."
),
"meta": {"source": "TN_MS_Brainstem"},
},
]
return guidelines
def _chunk_documents(docs: List[dict], max_len: int = 400) -> List[dict]:
"""Split long texts into overlapping chunks."""
out = []
for d in docs:
text = d["text"]
if len(text) <= max_len:
out.append(d)
else:
# Split by sentence boundaries
sentences = re.split(r"(?<=[.!?])\s+", text)
chunk, chunk_len = [], 0
for s in sentences:
if chunk_len + len(s) > max_len and chunk:
out.append({"text": " ".join(chunk), "meta": d["meta"]})
chunk, chunk_len = [], 0
chunk.append(s)
chunk_len += len(s)
if chunk:
out.append({"text": " ".join(chunk), "meta": d["meta"]})
return out