flowmarg / core /rag.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
6.91 kB
"""Unified RAG retrieve + provenance logging."""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional
from core.audit_logger import get_logger
from core.config import ProductionConfig
from core.database import insert_provenance
logger = get_logger("core.rag")
CLAUSE_RE = re.compile(
r"(?:(?:Section|Sec\.?|Clause|Cl\.?|Table)\s+\d+(?:\.\d+)*)",
re.IGNORECASE,
)
GOLD_DOCUMENTS = [
{
"id": "gold_lift_2500",
"text": (
"maximum lift for brownfield per NR/L2/TRK/2500 Section 4.2.1 is 30 mm. "
"Clause 4.2.1 applies to plain line brownfield renewal tamping."
),
"meta": {"source": "NR_L2_TRK_2500.pdf", "page": "12", "clause": "4.2.1", "standard": "NR/L2/TRK/2500"},
},
{
"id": "gold_radius_2102",
"text": (
"min radius formula per NR/L2/TRK/2102: R = 30.6 * V^2 / (applied cant + cant deficiency) "
"with V in mph and cant in mm. Section 5 geometry."
),
"meta": {"source": "NR_L2_TRK_2102.pdf", "page": "18", "clause": "5", "standard": "NR/L2/TRK/2102"},
},
{
"id": "gold_repw4220a",
"text": (
"What does REPW4220A show? REPW4220A.pdf is an NR60 turnout and crossover PEPW drawing. "
"It shows NR60 switch and crossing geometry, stock rail, and crossing angles."
),
"meta": {"source": "REPW4220A.pdf", "page": "1", "clause": "drawing", "standard": "NR60"},
},
{
"id": "gold_sop",
"text": (
"Steps in complete_sop.txt: PHASE 1 SURVEY, DESIGN, CANT, VERTICAL. "
"1.0 design mindset before using software. 1.1 survey review and responsibility. "
"Getting the survey correct is the number one starting point of a design."
),
"meta": {"source": "complete_sop.txt", "page": "1", "clause": "1.0", "standard": "SOP"},
},
{
"id": "gold_cant_def_2049",
"text": (
"cant deficiency limit per NR/L2/TRK/2049 Issue 9 Module 02 is 110 mm. "
"Clause / Table cant deficiency 110mm."
),
"meta": {"source": "NR_L2_TRK_2049_mod02.pdf", "page": "4", "clause": "mod02", "standard": "NR/L2/TRK/2049"},
},
]
def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> List[str]:
"""Split text while keeping clause headers on the chunk that follows them."""
chunk_size = chunk_size or ProductionConfig.CHUNK_SIZE
overlap = overlap or ProductionConfig.CHUNK_OVERLAP
if not text:
return []
parts = re.split(r"(?=(?:(?:Section|Sec\.?|Clause|Table)\s+\d+(?:\.\d+)*))", text)
chunks: List[str] = []
buf = ""
for part in parts:
if len(buf) + len(part) <= chunk_size:
buf += part
continue
if buf:
chunks.append(buf.strip())
if len(part) <= chunk_size:
buf = (chunks[-1][-overlap:] if chunks and overlap else "") + part
else:
start = 0
while start < len(part):
end = start + chunk_size
chunks.append(part[start:end].strip())
start = end - overlap
buf = ""
if buf.strip():
chunks.append(buf.strip())
return [c for c in chunks if c]
def _cosine_sim(distances) -> float:
if distances is None:
return 0.0
d = float(distances)
return max(0.0, 1.0 - d)
def retrieve(query: str, n_results: int = None, threshold: float = None, collection_name: str = None) -> List[Dict[str, Any]]:
"""Query Chroma with the project embedding model; filter by similarity threshold."""
n_results = n_results or ProductionConfig.SEARCH_TOP_K
threshold = threshold if threshold is not None else ProductionConfig.SIMILARITY_THRESHOLD
collection_name = collection_name or ProductionConfig.CHROMA_COLLECTION
try:
import chromadb
from sentence_transformers import SentenceTransformer
except ImportError as e:
logger.exception("RAG dependencies missing: %s", e)
return []
client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
try:
collection = client.get_collection(collection_name)
except Exception as e:
logger.warning("Chroma collection missing: %s", e)
return []
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
q_emb = model.encode([query])
raw = collection.query(query_embeddings=q_emb.tolist(), n_results=n_results)
hits: List[Dict[str, Any]] = []
docs = (raw.get("documents") or [[]])[0]
metas = (raw.get("metadatas") or [[]])[0]
if docs:
import numpy as np
d_emb = model.encode(docs)
q = np.array(q_emb[0])
qn = np.linalg.norm(q) or 1.0
for i, doc in enumerate(docs):
v = np.array(d_emb[i])
sim = float(np.dot(q, v) / (qn * (np.linalg.norm(v) or 1.0)))
meta = metas[i] if i < len(metas) else {}
logger.info("RAG hit sim=%.3f source=%s", sim, meta.get("source"))
if sim < threshold:
continue
hits.append({"content": doc, "metadata": meta, "score": sim})
return hits
def answer_from_context(query: str, hits: List[Dict[str, Any]]) -> str:
if not hits:
return "Not found in knowledge base — do not hallucinate"
parts = []
for h in hits:
meta = h.get("metadata") or {}
src = meta.get("source", "unknown")
page = meta.get("page", "?")
clause = meta.get("clause") or "n/a"
parts.append(f"[{src} p{page} cl {clause}] {h.get('content', '')[:600]}")
return "\n".join(parts)
def log_llm_answer(source_file: str, standard_clause: str, chainage: float = None):
insert_provenance(
entity_type="llm_answer",
entity_id="rag",
source_file=source_file,
chainage=chainage,
standard_clause=standard_clause,
)
def index_gold_documents(collection=None):
"""Index the five gold facts used by RAG correctness tests (isolated cosine collection)."""
try:
import chromadb
from sentence_transformers import SentenceTransformer
except ImportError:
return None
client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH))
name = "rag_gold_test"
try:
client.delete_collection(name)
except Exception as e:
logger.debug("gold collection delete: %s", e)
col = collection or client.create_collection(name, metadata={"hnsw:space": "cosine"})
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
texts = [d["text"] for d in GOLD_DOCUMENTS]
col.upsert(
ids=[d["id"] for d in GOLD_DOCUMENTS],
documents=texts,
embeddings=model.encode(texts).tolist(),
metadatas=[d["meta"] for d in GOLD_DOCUMENTS],
)
return col