File size: 6,913 Bytes
2379054
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
"""Unified RAG retrieve + provenance logging."""

from __future__ import annotations

import re
from typing import Any, Dict, List, Optional

from core.audit_logger import get_logger
from core.config import ProductionConfig
from core.database import insert_provenance

logger = get_logger("core.rag")

CLAUSE_RE = re.compile(
    r"(?:(?:Section|Sec\.?|Clause|Cl\.?|Table)\s+\d+(?:\.\d+)*)",
    re.IGNORECASE,
)

GOLD_DOCUMENTS = [
    {
        "id": "gold_lift_2500",
        "text": (
            "maximum lift for brownfield per NR/L2/TRK/2500 Section 4.2.1 is 30 mm. "
            "Clause 4.2.1 applies to plain line brownfield renewal tamping."
        ),
        "meta": {"source": "NR_L2_TRK_2500.pdf", "page": "12", "clause": "4.2.1", "standard": "NR/L2/TRK/2500"},
    },
    {
        "id": "gold_radius_2102",
        "text": (
            "min radius formula per NR/L2/TRK/2102: R = 30.6 * V^2 / (applied cant + cant deficiency) "
            "with V in mph and cant in mm. Section 5 geometry."
        ),
        "meta": {"source": "NR_L2_TRK_2102.pdf", "page": "18", "clause": "5", "standard": "NR/L2/TRK/2102"},
    },
    {
        "id": "gold_repw4220a",
        "text": (
            "What does REPW4220A show? REPW4220A.pdf is an NR60 turnout and crossover PEPW drawing. "
            "It shows NR60 switch and crossing geometry, stock rail, and crossing angles."
        ),
        "meta": {"source": "REPW4220A.pdf", "page": "1", "clause": "drawing", "standard": "NR60"},
    },
    {
        "id": "gold_sop",
        "text": (
            "Steps in complete_sop.txt: PHASE 1 SURVEY, DESIGN, CANT, VERTICAL. "
            "1.0 design mindset before using software. 1.1 survey review and responsibility. "
            "Getting the survey correct is the number one starting point of a design."
        ),
        "meta": {"source": "complete_sop.txt", "page": "1", "clause": "1.0", "standard": "SOP"},
    },
    {
        "id": "gold_cant_def_2049",
        "text": (
            "cant deficiency limit per NR/L2/TRK/2049 Issue 9 Module 02 is 110 mm. "
            "Clause / Table cant deficiency 110mm."
        ),
        "meta": {"source": "NR_L2_TRK_2049_mod02.pdf", "page": "4", "clause": "mod02", "standard": "NR/L2/TRK/2049"},
    },
]


def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> List[str]:
    """Split text while keeping clause headers on the chunk that follows them."""
    chunk_size = chunk_size or ProductionConfig.CHUNK_SIZE
    overlap = overlap or ProductionConfig.CHUNK_OVERLAP
    if not text:
        return []
    parts = re.split(r"(?=(?:(?:Section|Sec\.?|Clause|Table)\s+\d+(?:\.\d+)*))", text)
    chunks: List[str] = []
    buf = ""
    for part in parts:
        if len(buf) + len(part) <= chunk_size:
            buf += part
            continue
        if buf:
            chunks.append(buf.strip())
        if len(part) <= chunk_size:
            buf = (chunks[-1][-overlap:] if chunks and overlap else "") + part
        else:
            start = 0
            while start < len(part):
                end = start + chunk_size
                chunks.append(part[start:end].strip())
                start = end - overlap
            buf = ""
    if buf.strip():
        chunks.append(buf.strip())
    return [c for c in chunks if c]


def _cosine_sim(distances) -> float:
    if distances is None:
        return 0.0
    d = float(distances)
    return max(0.0, 1.0 - d)


def retrieve(query: str, n_results: int = None, threshold: float = None, collection_name: str = None) -> List[Dict[str, Any]]:
    """Query Chroma with the project embedding model; filter by similarity threshold."""
    n_results = n_results or ProductionConfig.SEARCH_TOP_K
    threshold = threshold if threshold is not None else ProductionConfig.SIMILARITY_THRESHOLD
    collection_name = collection_name or ProductionConfig.CHROMA_COLLECTION
    try:
        import chromadb
        from sentence_transformers import SentenceTransformer
    except ImportError as e:
        logger.exception("RAG dependencies missing: %s", e)
        return []

    client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
    try:
        collection = client.get_collection(collection_name)
    except Exception as e:
        logger.warning("Chroma collection missing: %s", e)
        return []

    model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
    q_emb = model.encode([query])
    raw = collection.query(query_embeddings=q_emb.tolist(), n_results=n_results)
    hits: List[Dict[str, Any]] = []
    docs = (raw.get("documents") or [[]])[0]
    metas = (raw.get("metadatas") or [[]])[0]
    if docs:
        import numpy as np

        d_emb = model.encode(docs)
        q = np.array(q_emb[0])
        qn = np.linalg.norm(q) or 1.0
        for i, doc in enumerate(docs):
            v = np.array(d_emb[i])
            sim = float(np.dot(q, v) / (qn * (np.linalg.norm(v) or 1.0)))
            meta = metas[i] if i < len(metas) else {}
            logger.info("RAG hit sim=%.3f source=%s", sim, meta.get("source"))
            if sim < threshold:
                continue
            hits.append({"content": doc, "metadata": meta, "score": sim})
    return hits


def answer_from_context(query: str, hits: List[Dict[str, Any]]) -> str:
    if not hits:
        return "Not found in knowledge base — do not hallucinate"
    parts = []
    for h in hits:
        meta = h.get("metadata") or {}
        src = meta.get("source", "unknown")
        page = meta.get("page", "?")
        clause = meta.get("clause") or "n/a"
        parts.append(f"[{src} p{page} cl {clause}] {h.get('content', '')[:600]}")
    return "\n".join(parts)


def log_llm_answer(source_file: str, standard_clause: str, chainage: float = None):
    insert_provenance(
        entity_type="llm_answer",
        entity_id="rag",
        source_file=source_file,
        chainage=chainage,
        standard_clause=standard_clause,
    )


def index_gold_documents(collection=None):
    """Index the five gold facts used by RAG correctness tests (isolated cosine collection)."""
    try:
        import chromadb
        from sentence_transformers import SentenceTransformer
    except ImportError:
        return None
    client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
    name = "rag_gold_test"
    try:
        client.delete_collection(name)
    except Exception as e:
        logger.debug("gold collection delete: %s", e)
    col = collection or client.create_collection(name, metadata={"hnsw:space": "cosine"})
    model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
    texts = [d["text"] for d in GOLD_DOCUMENTS]
    col.upsert(
        ids=[d["id"] for d in GOLD_DOCUMENTS],
        documents=texts,
        embeddings=model.encode(texts).tolist(),
        metadatas=[d["meta"] for d in GOLD_DOCUMENTS],
    )
    return col