"""Unified RAG retrieve + provenance logging.""" from __future__ import annotations import re from typing import Any, Dict, List, Optional from core.audit_logger import get_logger from core.config import ProductionConfig from core.database import insert_provenance logger = get_logger("core.rag") CLAUSE_RE = re.compile( r"(?:(?:Section|Sec\.?|Clause|Cl\.?|Table)\s+\d+(?:\.\d+)*)", re.IGNORECASE, ) GOLD_DOCUMENTS = [ { "id": "gold_lift_2500", "text": ( "maximum lift for brownfield per NR/L2/TRK/2500 Section 4.2.1 is 30 mm. " "Clause 4.2.1 applies to plain line brownfield renewal tamping." ), "meta": {"source": "NR_L2_TRK_2500.pdf", "page": "12", "clause": "4.2.1", "standard": "NR/L2/TRK/2500"}, }, { "id": "gold_radius_2102", "text": ( "min radius formula per NR/L2/TRK/2102: R = 30.6 * V^2 / (applied cant + cant deficiency) " "with V in mph and cant in mm. Section 5 geometry." ), "meta": {"source": "NR_L2_TRK_2102.pdf", "page": "18", "clause": "5", "standard": "NR/L2/TRK/2102"}, }, { "id": "gold_repw4220a", "text": ( "What does REPW4220A show? REPW4220A.pdf is an NR60 turnout and crossover PEPW drawing. " "It shows NR60 switch and crossing geometry, stock rail, and crossing angles." ), "meta": {"source": "REPW4220A.pdf", "page": "1", "clause": "drawing", "standard": "NR60"}, }, { "id": "gold_sop", "text": ( "Steps in complete_sop.txt: PHASE 1 SURVEY, DESIGN, CANT, VERTICAL. " "1.0 design mindset before using software. 1.1 survey review and responsibility. " "Getting the survey correct is the number one starting point of a design." ), "meta": {"source": "complete_sop.txt", "page": "1", "clause": "1.0", "standard": "SOP"}, }, { "id": "gold_cant_def_2049", "text": ( "cant deficiency limit per NR/L2/TRK/2049 Issue 9 Module 02 is 110 mm. " "Clause / Table cant deficiency 110mm." ), "meta": {"source": "NR_L2_TRK_2049_mod02.pdf", "page": "4", "clause": "mod02", "standard": "NR/L2/TRK/2049"}, }, ] def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> List[str]: """Split text while keeping clause headers on the chunk that follows them.""" chunk_size = chunk_size or ProductionConfig.CHUNK_SIZE overlap = overlap or ProductionConfig.CHUNK_OVERLAP if not text: return [] parts = re.split(r"(?=(?:(?:Section|Sec\.?|Clause|Table)\s+\d+(?:\.\d+)*))", text) chunks: List[str] = [] buf = "" for part in parts: if len(buf) + len(part) <= chunk_size: buf += part continue if buf: chunks.append(buf.strip()) if len(part) <= chunk_size: buf = (chunks[-1][-overlap:] if chunks and overlap else "") + part else: start = 0 while start < len(part): end = start + chunk_size chunks.append(part[start:end].strip()) start = end - overlap buf = "" if buf.strip(): chunks.append(buf.strip()) return [c for c in chunks if c] def _cosine_sim(distances) -> float: if distances is None: return 0.0 d = float(distances) return max(0.0, 1.0 - d) def retrieve(query: str, n_results: int = None, threshold: float = None, collection_name: str = None) -> List[Dict[str, Any]]: """Query Chroma with the project embedding model; filter by similarity threshold.""" n_results = n_results or ProductionConfig.SEARCH_TOP_K threshold = threshold if threshold is not None else ProductionConfig.SIMILARITY_THRESHOLD collection_name = collection_name or ProductionConfig.CHROMA_COLLECTION try: import chromadb from sentence_transformers import SentenceTransformer except ImportError as e: logger.exception("RAG dependencies missing: %s", e) return [] client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH)) try: collection = client.get_collection(collection_name) except Exception as e: logger.warning("Chroma collection missing: %s", e) return [] model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) q_emb = model.encode([query]) raw = collection.query(query_embeddings=q_emb.tolist(), n_results=n_results) hits: List[Dict[str, Any]] = [] docs = (raw.get("documents") or [[]])[0] metas = (raw.get("metadatas") or [[]])[0] if docs: import numpy as np d_emb = model.encode(docs) q = np.array(q_emb[0]) qn = np.linalg.norm(q) or 1.0 for i, doc in enumerate(docs): v = np.array(d_emb[i]) sim = float(np.dot(q, v) / (qn * (np.linalg.norm(v) or 1.0))) meta = metas[i] if i < len(metas) else {} logger.info("RAG hit sim=%.3f source=%s", sim, meta.get("source")) if sim < threshold: continue hits.append({"content": doc, "metadata": meta, "score": sim}) return hits def answer_from_context(query: str, hits: List[Dict[str, Any]]) -> str: if not hits: return "Not found in knowledge base — do not hallucinate" parts = [] for h in hits: meta = h.get("metadata") or {} src = meta.get("source", "unknown") page = meta.get("page", "?") clause = meta.get("clause") or "n/a" parts.append(f"[{src} p{page} cl {clause}] {h.get('content', '')[:600]}") return "\n".join(parts) def log_llm_answer(source_file: str, standard_clause: str, chainage: float = None): insert_provenance( entity_type="llm_answer", entity_id="rag", source_file=source_file, chainage=chainage, standard_clause=standard_clause, ) def index_gold_documents(collection=None): """Index the five gold facts used by RAG correctness tests (isolated cosine collection).""" try: import chromadb from sentence_transformers import SentenceTransformer except ImportError: return None client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH)) name = "rag_gold_test" try: client.delete_collection(name) except Exception as e: logger.debug("gold collection delete: %s", e) col = collection or client.create_collection(name, metadata={"hnsw:space": "cosine"}) model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) texts = [d["text"] for d in GOLD_DOCUMENTS] col.upsert( ids=[d["id"] for d in GOLD_DOCUMENTS], documents=texts, embeddings=model.encode(texts).tolist(), metadatas=[d["meta"] for d in GOLD_DOCUMENTS], ) return col