""" core/chroma_project.py Project-specific ChromaDB integration. """ import os import json import pandas as pd from pathlib import Path from core.database import query_chainages, query_survey_features from core.standards_loader import load_standards try: import chromadb from sentence_transformers import SentenceTransformer CHROMA_AVAILABLE = True except ImportError: CHROMA_AVAILABLE = False from core.config import ProductionConfig from core.audit_logger import get_logger logger = get_logger("core.chroma_project") CHROMA_PATH = ProductionConfig.CHROMA_PATH def get_collection(collection_name="track_design_knowledge"): """Get or create a ChromaDB collection.""" if not CHROMA_AVAILABLE: return None client = chromadb.PersistentClient(path=str(CHROMA_PATH)) try: collection = client.get_collection(collection_name) except Exception: collection = client.create_collection(collection_name) return collection def index_project_data(project_name: str): """ Index current project data (chainages, survey, standards) into ChromaDB. This makes the project data searchable via RAG. """ if not CHROMA_AVAILABLE: logger.info("ChromaDB not available - skipping indexing.") return collection = get_collection() if collection is None: return # Load embedding model try: model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) except Exception as e: logger.warning("Embedding model load failed: %s", e) return # Query chainages chainages_df = query_chainages() documents = [] metadatas = [] ids = [] if not chainages_df.empty: for _, row in chainages_df.iterrows(): ch = row.get("chainage") align = row.get("alignment", "Unknown") level = row.get("level") radius = row.get("radius") cant = row.get("cant") grade = row.get("grade") text = f"Project: {project_name}. Alignment: {align}. Chainage: {ch:.3f}. Level: {level}. Radius: {radius}. Cant: {cant}. Grade: {grade}." documents.append(text) metadatas.append({"source": "chainage", "alignment": align, "chainage": ch}) ids.append(f"chainage_{align}_{ch}") # Query survey features survey_df = query_survey_features() if not survey_df.empty: for _, row in survey_df.iterrows(): cat = row.get("category", "Unknown") loc = row.get("chainage") desc = row.get("description", "") text = f"Project: {project_name}. Survey feature: {cat} at CH {loc}. Description: {desc}." documents.append(text) metadatas.append({"source": "survey", "category": cat, "chainage": loc}) ids.append(f"survey_{cat}_{loc}") # Add standards standards = load_standards() for std_name, std_data in standards.items(): for key, value in std_data.get("limits", {}).items(): if isinstance(value, dict): limit = value.get("value") unit = value.get("unit", "") desc = value.get("description", "") text = f"Standard: {std_name}. Parameter: {key}. Limit: {limit} {unit}. Description: {desc}." documents.append(text) metadatas.append({"source": "standard", "standard": std_name, "parameter": key}) ids.append(f"standard_{std_name}_{key}") if documents: # Delete existing project data from collection try: collection.delete(ids=ids) except Exception: pass # Add in batches batch_size = 100 for i in range(0, len(documents), batch_size): end = min(i + batch_size, len(documents)) try: embeddings = model.encode(documents[i:end]) collection.add( ids=ids[i:end], documents=documents[i:end], embeddings=embeddings.tolist(), metadatas=metadatas[i:end] ) except Exception as e: logger.warning("Batch %s failed: %s", i, e) logger.info("Indexed %s documents into ChromaDB for project %s.", len(documents), project_name) else: logger.info("No project data to index.") def query_project_rag(query: str, project_name: str, n_results=None): """Query ChromaDB for project-specific data.""" if not CHROMA_AVAILABLE: return [] collection = get_collection() if collection is None: return [] n_results = n_results or ProductionConfig.SEARCH_TOP_K try: model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) embedding = model.encode([query]) results = collection.query( query_embeddings=embedding.tolist(), n_results=n_results, ) return results except Exception as e: logger.warning("RAG query failed: %s", e) return []