File size: 6,913 Bytes
2379054 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | """Unified RAG retrieve + provenance logging."""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional
from core.audit_logger import get_logger
from core.config import ProductionConfig
from core.database import insert_provenance
logger = get_logger("core.rag")
CLAUSE_RE = re.compile(
r"(?:(?:Section|Sec\.?|Clause|Cl\.?|Table)\s+\d+(?:\.\d+)*)",
re.IGNORECASE,
)
GOLD_DOCUMENTS = [
{
"id": "gold_lift_2500",
"text": (
"maximum lift for brownfield per NR/L2/TRK/2500 Section 4.2.1 is 30 mm. "
"Clause 4.2.1 applies to plain line brownfield renewal tamping."
),
"meta": {"source": "NR_L2_TRK_2500.pdf", "page": "12", "clause": "4.2.1", "standard": "NR/L2/TRK/2500"},
},
{
"id": "gold_radius_2102",
"text": (
"min radius formula per NR/L2/TRK/2102: R = 30.6 * V^2 / (applied cant + cant deficiency) "
"with V in mph and cant in mm. Section 5 geometry."
),
"meta": {"source": "NR_L2_TRK_2102.pdf", "page": "18", "clause": "5", "standard": "NR/L2/TRK/2102"},
},
{
"id": "gold_repw4220a",
"text": (
"What does REPW4220A show? REPW4220A.pdf is an NR60 turnout and crossover PEPW drawing. "
"It shows NR60 switch and crossing geometry, stock rail, and crossing angles."
),
"meta": {"source": "REPW4220A.pdf", "page": "1", "clause": "drawing", "standard": "NR60"},
},
{
"id": "gold_sop",
"text": (
"Steps in complete_sop.txt: PHASE 1 SURVEY, DESIGN, CANT, VERTICAL. "
"1.0 design mindset before using software. 1.1 survey review and responsibility. "
"Getting the survey correct is the number one starting point of a design."
),
"meta": {"source": "complete_sop.txt", "page": "1", "clause": "1.0", "standard": "SOP"},
},
{
"id": "gold_cant_def_2049",
"text": (
"cant deficiency limit per NR/L2/TRK/2049 Issue 9 Module 02 is 110 mm. "
"Clause / Table cant deficiency 110mm."
),
"meta": {"source": "NR_L2_TRK_2049_mod02.pdf", "page": "4", "clause": "mod02", "standard": "NR/L2/TRK/2049"},
},
]
def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> List[str]:
"""Split text while keeping clause headers on the chunk that follows them."""
chunk_size = chunk_size or ProductionConfig.CHUNK_SIZE
overlap = overlap or ProductionConfig.CHUNK_OVERLAP
if not text:
return []
parts = re.split(r"(?=(?:(?:Section|Sec\.?|Clause|Table)\s+\d+(?:\.\d+)*))", text)
chunks: List[str] = []
buf = ""
for part in parts:
if len(buf) + len(part) <= chunk_size:
buf += part
continue
if buf:
chunks.append(buf.strip())
if len(part) <= chunk_size:
buf = (chunks[-1][-overlap:] if chunks and overlap else "") + part
else:
start = 0
while start < len(part):
end = start + chunk_size
chunks.append(part[start:end].strip())
start = end - overlap
buf = ""
if buf.strip():
chunks.append(buf.strip())
return [c for c in chunks if c]
def _cosine_sim(distances) -> float:
if distances is None:
return 0.0
d = float(distances)
return max(0.0, 1.0 - d)
def retrieve(query: str, n_results: int = None, threshold: float = None, collection_name: str = None) -> List[Dict[str, Any]]:
"""Query Chroma with the project embedding model; filter by similarity threshold."""
n_results = n_results or ProductionConfig.SEARCH_TOP_K
threshold = threshold if threshold is not None else ProductionConfig.SIMILARITY_THRESHOLD
collection_name = collection_name or ProductionConfig.CHROMA_COLLECTION
try:
import chromadb
from sentence_transformers import SentenceTransformer
except ImportError as e:
logger.exception("RAG dependencies missing: %s", e)
return []
client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
try:
collection = client.get_collection(collection_name)
except Exception as e:
logger.warning("Chroma collection missing: %s", e)
return []
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
q_emb = model.encode([query])
raw = collection.query(query_embeddings=q_emb.tolist(), n_results=n_results)
hits: List[Dict[str, Any]] = []
docs = (raw.get("documents") or [[]])[0]
metas = (raw.get("metadatas") or [[]])[0]
if docs:
import numpy as np
d_emb = model.encode(docs)
q = np.array(q_emb[0])
qn = np.linalg.norm(q) or 1.0
for i, doc in enumerate(docs):
v = np.array(d_emb[i])
sim = float(np.dot(q, v) / (qn * (np.linalg.norm(v) or 1.0)))
meta = metas[i] if i < len(metas) else {}
logger.info("RAG hit sim=%.3f source=%s", sim, meta.get("source"))
if sim < threshold:
continue
hits.append({"content": doc, "metadata": meta, "score": sim})
return hits
def answer_from_context(query: str, hits: List[Dict[str, Any]]) -> str:
if not hits:
return "Not found in knowledge base — do not hallucinate"
parts = []
for h in hits:
meta = h.get("metadata") or {}
src = meta.get("source", "unknown")
page = meta.get("page", "?")
clause = meta.get("clause") or "n/a"
parts.append(f"[{src} p{page} cl {clause}] {h.get('content', '')[:600]}")
return "\n".join(parts)
def log_llm_answer(source_file: str, standard_clause: str, chainage: float = None):
insert_provenance(
entity_type="llm_answer",
entity_id="rag",
source_file=source_file,
chainage=chainage,
standard_clause=standard_clause,
)
def index_gold_documents(collection=None):
"""Index the five gold facts used by RAG correctness tests (isolated cosine collection)."""
try:
import chromadb
from sentence_transformers import SentenceTransformer
except ImportError:
return None
client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
name = "rag_gold_test"
try:
client.delete_collection(name)
except Exception as e:
logger.debug("gold collection delete: %s", e)
col = collection or client.create_collection(name, metadata={"hnsw:space": "cosine"})
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
texts = [d["text"] for d in GOLD_DOCUMENTS]
col.upsert(
ids=[d["id"] for d in GOLD_DOCUMENTS],
documents=texts,
embeddings=model.encode(texts).tolist(),
metadatas=[d["meta"] for d in GOLD_DOCUMENTS],
)
return col
|