File size: 10,328 Bytes
e803935
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
"""
NeuroCheck Agentic AI β€” RAG Knowledge Base
Embeds the 150 MS cases from the docx + MS clinical guidelines into ChromaDB.
Uses local sentence-transformers (no API key needed).
"""
from __future__ import annotations
import os, zipfile, re
from pathlib import Path
from typing import List
from loguru import logger

import chromadb
from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction

# ── Paths ─────────────────────────────────────────────────────────────────────
BASE_DIR   = Path(__file__).parent.parent
DOCX_PATH  = BASE_DIR / "AGENTIC AI chatbots.docx"
CHROMA_DIR = Path(os.environ.get("CHROMA_DB_PATH", str(BASE_DIR / "chroma_db")))

# ── Embedding model (local, no API key) ──────────────────────────────────────
EMBED_MODEL = "all-MiniLM-L6-v2"   # ~90MB, fast, good for clinical text

_client: chromadb.PersistentClient | None = None
_collection = None


def get_collection():
    global _client, _collection
    if _collection is not None:
        return _collection
    ef = SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL)
    _client = chromadb.PersistentClient(path=str(CHROMA_DIR))
    _collection = _client.get_or_create_collection(
        name="ms_knowledge",
        embedding_function=ef,
        metadata={"hnsw:space": "cosine"},
    )
    return _collection


def build_knowledge_base(force: bool = False) -> None:
    """
    Build the ChromaDB vector store from the docx file.
    Only runs if the collection is empty (or force=True).
    """
    col = get_collection()
    if col.count() > 0 and not force:
        logger.info(f"[RAG] Knowledge base already built ({col.count()} chunks). Skipping.")
        return

    logger.info("[RAG] Building knowledge base from docx...")
    docs = _extract_docx(DOCX_PATH)
    docs += _ms_guidelines()          # inject hard-coded clinical knowledge
    chunks = _chunk_documents(docs)

    ids      = [f"doc_{i}" for i in range(len(chunks))]
    texts    = [c["text"] for c in chunks]
    metas    = [c["meta"] for c in chunks]

    # Upsert in batches of 100
    for i in range(0, len(texts), 100):
        col.upsert(
            ids=ids[i:i+100],
            documents=texts[i:i+100],
            metadatas=metas[i:i+100],
        )

    logger.info(f"[RAG] Knowledge base ready: {col.count()} chunks indexed.")


def query(text: str, n_results: int = 5) -> List[dict]:
    """Return top-n relevant knowledge chunks for a given query."""
    col = get_collection()
    if col.count() == 0:
        return []
    results = col.query(query_texts=[text], n_results=min(n_results, col.count()))
    output = []
    for doc, meta, dist in zip(
        results["documents"][0],
        results["metadatas"][0],
        results["distances"][0],
    ):
        output.append({"text": doc, "source": meta.get("source", ""), "score": round(1 - dist, 3)})
    return output


# ── Document extraction ───────────────────────────────────────────────────────
def _extract_docx(path: Path) -> List[dict]:
    if not path.exists():
        logger.warning(f"[RAG] Docx not found at {path}")
        return []
    import xml.etree.ElementTree as ET
    docs = []
    with zipfile.ZipFile(str(path)) as z:
        xml = z.read("word/document.xml")
    root = ET.fromstring(xml)
    ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
    paras = []
    for para in root.iter(f"{ns}p"):
        line = "".join(node.text or "" for node in para.iter(f"{ns}t")).strip()
        if line:
            paras.append(line)

    # Group into logical blocks of 3 paragraphs for context
    for i in range(0, len(paras), 3):
        block = " ".join(paras[i:i+3])
        docs.append({"text": block, "meta": {"source": "AGENTIC_AI_chatbots.docx"}})
    return docs


def _ms_guidelines() -> List[dict]:
    """Hard-coded authoritative MS clinical guidelines as RAG knowledge."""
    guidelines = [
        {
            "text": (
                "McDonald Criteria 2024: MS diagnosis requires dissemination in space (DIS) "
                "and dissemination in time (DIT). DIS: lesions in β‰₯2 CNS regions (periventricular, "
                "cortical/juxtacortical, infratentorial, spinal cord, optic nerve). "
                "DIT: new T2/gadolinium lesion on follow-up MRI, or simultaneous enhancing + non-enhancing lesions."
            ),
            "meta": {"source": "McDonald_Criteria_2024"},
        },
        {
            "text": (
                "MS Workup Standard: MRI orbits with/without contrast for optic nerve involvement. "
                "MRI brain and spinal cord with/without contrast. Lumbar puncture: CSF oligoclonal bands "
                ">2 exclusive to CSF, elevated IgG index, kappa free light chains. "
                "Serum cell-based assay for NMO (AQP4-IgG) and MOGAD (MOG-IgG). "
                "Exclude mimics: sarcoidosis, lupus, vasculitis, B12 deficiency, cervical myelopathy."
            ),
            "meta": {"source": "MS_Workup_Protocol"},
        },
        {
            "text": (
                "Optic Neuritis classic triad: monocular visual loss + pain on eye movement + dyschromatopsia "
                "(red colour desaturation). Peak age 20-45. Female predominance. "
                "Uhthoff phenomenon: vision worsens with heat/exercise. "
                "Recovery in 85% within 3 months. Risk of MS after ON: 50% at 15 years."
            ),
            "meta": {"source": "NINDS_Optic_Neuritis"},
        },
        {
            "text": (
                "NMOSD (Neuromyelitis Optica Spectrum Disorder): AQP4-IgG positive. "
                "Area postrema syndrome: intractable hiccups + nausea/vomiting. "
                "LETM: longitudinally extensive transverse myelitis >3 vertebral segments. "
                "Bilateral/severe optic neuritis. Periependymal brain lesions. "
                "Distinguished from MS by AQP4-IgG serum test (cell-based assay)."
            ),
            "meta": {"source": "NMOSD_Guidelines"},
        },
        {
            "text": (
                "MOGAD (MOG-Antibody Associated Disease): MOG-IgG positive. "
                "ADEM-like presentation with large hazy confluent white matter lesions. "
                "Optic perineuritis (sheath enhancement, not nerve). Conus medullaris involvement. "
                "Often affects younger patients. Differentiated by MOG-IgG cell-based assay."
            ),
            "meta": {"source": "MOGAD_Guidelines"},
        },
        {
            "text": (
                "Lhermitte's Sign: Electric shock sensation down spine on neck flexion. "
                "Pathognomonic for cervical cord demyelination. "
                "Uhthoff Phenomenon: worsening of any MS symptom with heat, exercise, fever, hot bath. "
                "MS Hug: tight band sensation around trunk/chest β€” thoracic cord lesion. "
                "These are anchor features requiring urgent neurology referral."
            ),
            "meta": {"source": "MS_Anchor_Symptoms"},
        },
        {
            "text": (
                "IIH vs MS in obese patients: IIH (idiopathic intracranial hypertension) "
                "typically presents with papilledema, headache, pulsatile tinnitus in obese females. "
                "However, MS can coexist. If colour vision is impaired (dyschromatopsia) in IIH, "
                "suspect concurrent optic neuritis. Heavy legs, motor symptoms, or prior transient "
                "neurological episodes in obese patients must trigger MS workup."
            ),
            "meta": {"source": "IIH_MS_Overlap"},
        },
        {
            "text": (
                "CSF Analysis in MS: Oligoclonal bands (OCBs) >2 exclusive to CSF (not serum) β€” "
                "present in 85-95% of MS. IgG index >0.7 (elevated). IgG synthesis rate elevated. "
                "Kappa free light chains substitute for OCBs where OCB testing unavailable. "
                "Cell count <50 lymphocytes. Glucose and protein usually normal. "
                "Paired serum-CSF sample essential for OCB interpretation."
            ),
            "meta": {"source": "CSF_MS_Protocol"},
        },
        {
            "text": (
                "Internuclear Ophthalmoplegia (INO): failure of ipsilateral eye adduction on lateral gaze "
                "with contralateral eye nystagmus. Caused by MLF lesion. Bilateral INO pathognomonic for MS. "
                "Sixth nerve palsy: lateral gaze palsy. Skew deviation: vertical misalignment of eyes. "
                "Any new diplopia in young adult β€” MS must be excluded."
            ),
            "meta": {"source": "MS_Eye_Movement_Disorders"},
        },
        {
            "text": (
                "Trigeminal Neuralgia in young adults (<45): electric shock face/jaw pain. "
                "When occurring in young patients without vascular loop compression, MS pons lesion "
                "must be excluded. MRI brain with specific pons protocol required. "
                "Facial numbness + diplopia combination suggests brainstem MS lesion."
            ),
            "meta": {"source": "TN_MS_Brainstem"},
        },
    ]
    return guidelines


def _chunk_documents(docs: List[dict], max_len: int = 400) -> List[dict]:
    """Split long texts into overlapping chunks."""
    out = []
    for d in docs:
        text = d["text"]
        if len(text) <= max_len:
            out.append(d)
        else:
            # Split by sentence boundaries
            sentences = re.split(r"(?<=[.!?])\s+", text)
            chunk, chunk_len = [], 0
            for s in sentences:
                if chunk_len + len(s) > max_len and chunk:
                    out.append({"text": " ".join(chunk), "meta": d["meta"]})
                    chunk, chunk_len = [], 0
                chunk.append(s)
                chunk_len += len(s)
            if chunk:
                out.append({"text": " ".join(chunk), "meta": d["meta"]})
    return out