Download core/rag.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 6.91 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/rag.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/core/rag.py
-
curl -L -o rag.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/rag.py
6.91 kB
| """Unified RAG retrieve + provenance logging.""" | |
| from __future__ import annotations | |
| import re | |
| from typing import Any, Dict, List, Optional | |
| from core.audit_logger import get_logger | |
| from core.config import ProductionConfig | |
| from core.database import insert_provenance | |
| logger = get_logger("core.rag") | |
| CLAUSE_RE = re.compile( | |
| r"(?:(?:Section|Sec\.?|Clause|Cl\.?|Table)\s+\d+(?:\.\d+)*)", | |
| re.IGNORECASE, | |
| ) | |
| GOLD_DOCUMENTS = [ | |
| { | |
| "id": "gold_lift_2500", | |
| "text": ( | |
| "maximum lift for brownfield per NR/L2/TRK/2500 Section 4.2.1 is 30 mm. " | |
| "Clause 4.2.1 applies to plain line brownfield renewal tamping." | |
| ), | |
| "meta": {"source": "NR_L2_TRK_2500.pdf", "page": "12", "clause": "4.2.1", "standard": "NR/L2/TRK/2500"}, | |
| }, | |
| { | |
| "id": "gold_radius_2102", | |
| "text": ( | |
| "min radius formula per NR/L2/TRK/2102: R = 30.6 * V^2 / (applied cant + cant deficiency) " | |
| "with V in mph and cant in mm. Section 5 geometry." | |
| ), | |
| "meta": {"source": "NR_L2_TRK_2102.pdf", "page": "18", "clause": "5", "standard": "NR/L2/TRK/2102"}, | |
| }, | |
| { | |
| "id": "gold_repw4220a", | |
| "text": ( | |
| "What does REPW4220A show? REPW4220A.pdf is an NR60 turnout and crossover PEPW drawing. " | |
| "It shows NR60 switch and crossing geometry, stock rail, and crossing angles." | |
| ), | |
| "meta": {"source": "REPW4220A.pdf", "page": "1", "clause": "drawing", "standard": "NR60"}, | |
| }, | |
| { | |
| "id": "gold_sop", | |
| "text": ( | |
| "Steps in complete_sop.txt: PHASE 1 SURVEY, DESIGN, CANT, VERTICAL. " | |
| "1.0 design mindset before using software. 1.1 survey review and responsibility. " | |
| "Getting the survey correct is the number one starting point of a design." | |
| ), | |
| "meta": {"source": "complete_sop.txt", "page": "1", "clause": "1.0", "standard": "SOP"}, | |
| }, | |
| { | |
| "id": "gold_cant_def_2049", | |
| "text": ( | |
| "cant deficiency limit per NR/L2/TRK/2049 Issue 9 Module 02 is 110 mm. " | |
| "Clause / Table cant deficiency 110mm." | |
| ), | |
| "meta": {"source": "NR_L2_TRK_2049_mod02.pdf", "page": "4", "clause": "mod02", "standard": "NR/L2/TRK/2049"}, | |
| }, | |
| ] | |
| def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> List[str]: | |
| """Split text while keeping clause headers on the chunk that follows them.""" | |
| chunk_size = chunk_size or ProductionConfig.CHUNK_SIZE | |
| overlap = overlap or ProductionConfig.CHUNK_OVERLAP | |
| if not text: | |
| return [] | |
| parts = re.split(r"(?=(?:(?:Section|Sec\.?|Clause|Table)\s+\d+(?:\.\d+)*))", text) | |
| chunks: List[str] = [] | |
| buf = "" | |
| for part in parts: | |
| if len(buf) + len(part) <= chunk_size: | |
| buf += part | |
| continue | |
| if buf: | |
| chunks.append(buf.strip()) | |
| if len(part) <= chunk_size: | |
| buf = (chunks[-1][-overlap:] if chunks and overlap else "") + part | |
| else: | |
| start = 0 | |
| while start < len(part): | |
| end = start + chunk_size | |
| chunks.append(part[start:end].strip()) | |
| start = end - overlap | |
| buf = "" | |
| if buf.strip(): | |
| chunks.append(buf.strip()) | |
| return [c for c in chunks if c] | |
| def _cosine_sim(distances) -> float: | |
| if distances is None: | |
| return 0.0 | |
| d = float(distances) | |
| return max(0.0, 1.0 - d) | |
| def retrieve(query: str, n_results: int = None, threshold: float = None, collection_name: str = None) -> List[Dict[str, Any]]: | |
| """Query Chroma with the project embedding model; filter by similarity threshold.""" | |
| n_results = n_results or ProductionConfig.SEARCH_TOP_K | |
| threshold = threshold if threshold is not None else ProductionConfig.SIMILARITY_THRESHOLD | |
| collection_name = collection_name or ProductionConfig.CHROMA_COLLECTION | |
| try: | |
| import chromadb | |
| from sentence_transformers import SentenceTransformer | |
| except ImportError as e: | |
| logger.exception("RAG dependencies missing: %s", e) | |
| return [] | |
| client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH)) | |
| try: | |
| collection = client.get_collection(collection_name) | |
| except Exception as e: | |
| logger.warning("Chroma collection missing: %s", e) | |
| return [] | |
| model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) | |
| q_emb = model.encode([query]) | |
| raw = collection.query(query_embeddings=q_emb.tolist(), n_results=n_results) | |
| hits: List[Dict[str, Any]] = [] | |
| docs = (raw.get("documents") or [[]])[0] | |
| metas = (raw.get("metadatas") or [[]])[0] | |
| if docs: | |
| import numpy as np | |
| d_emb = model.encode(docs) | |
| q = np.array(q_emb[0]) | |
| qn = np.linalg.norm(q) or 1.0 | |
| for i, doc in enumerate(docs): | |
| v = np.array(d_emb[i]) | |
| sim = float(np.dot(q, v) / (qn * (np.linalg.norm(v) or 1.0))) | |
| meta = metas[i] if i < len(metas) else {} | |
| logger.info("RAG hit sim=%.3f source=%s", sim, meta.get("source")) | |
| if sim < threshold: | |
| continue | |
| hits.append({"content": doc, "metadata": meta, "score": sim}) | |
| return hits | |
| def answer_from_context(query: str, hits: List[Dict[str, Any]]) -> str: | |
| if not hits: | |
| return "Not found in knowledge base — do not hallucinate" | |
| parts = [] | |
| for h in hits: | |
| meta = h.get("metadata") or {} | |
| src = meta.get("source", "unknown") | |
| page = meta.get("page", "?") | |
| clause = meta.get("clause") or "n/a" | |
| parts.append(f"[{src} p{page} cl {clause}] {h.get('content', '')[:600]}") | |
| return "\n".join(parts) | |
| def log_llm_answer(source_file: str, standard_clause: str, chainage: float = None): | |
| insert_provenance( | |
| entity_type="llm_answer", | |
| entity_id="rag", | |
| source_file=source_file, | |
| chainage=chainage, | |
| standard_clause=standard_clause, | |
| ) | |
| def index_gold_documents(collection=None): | |
| """Index the five gold facts used by RAG correctness tests (isolated cosine collection).""" | |
| try: | |
| import chromadb | |
| from sentence_transformers import SentenceTransformer | |
| except ImportError: | |
| return None | |
| client = chromadb.PersistentClient(path=str(ProductionConfig.CHROMA_PATH)) | |
| name = "rag_gold_test" | |
| try: | |
| client.delete_collection(name) | |
| except Exception as e: | |
| logger.debug("gold collection delete: %s", e) | |
| col = collection or client.create_collection(name, metadata={"hnsw:space": "cosine"}) | |
| model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) | |
| texts = [d["text"] for d in GOLD_DOCUMENTS] | |
| col.upsert( | |
| ids=[d["id"] for d in GOLD_DOCUMENTS], | |
| documents=texts, | |
| embeddings=model.encode(texts).tolist(), | |
| metadatas=[d["meta"] for d in GOLD_DOCUMENTS], | |
| ) | |
| return col | |