flowmarg / core /chroma_project.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
5.08 kB
"""
core/chroma_project.py
Project-specific ChromaDB integration.
"""
import os
import json
import pandas as pd
from pathlib import Path
from core.database import query_chainages, query_survey_features
from core.standards_loader import load_standards
try:
import chromadb
from sentence_transformers import SentenceTransformer
CHROMA_AVAILABLE = True
except ImportError:
CHROMA_AVAILABLE = False
from core.config import ProductionConfig
from core.audit_logger import get_logger
logger = get_logger("core.chroma_project")
CHROMA_PATH = ProductionConfig.CHROMA_PATH
def get_collection(collection_name="track_design_knowledge"):
"""Get or create a ChromaDB collection."""
if not CHROMA_AVAILABLE:
return None
client = chromadb.PersistentClient(path=str(CHROMA_PATH))
try:
collection = client.get_collection(collection_name)
except Exception:
collection = client.create_collection(collection_name)
return collection
def index_project_data(project_name: str):
"""
Index current project data (chainages, survey, standards) into ChromaDB.
This makes the project data searchable via RAG.
"""
if not CHROMA_AVAILABLE:
logger.info("ChromaDB not available - skipping indexing.")
return
collection = get_collection()
if collection is None:
return
# Load embedding model
try:
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
except Exception as e:
logger.warning("Embedding model load failed: %s", e)
return
# Query chainages
chainages_df = query_chainages()
documents = []
metadatas = []
ids = []
if not chainages_df.empty:
for _, row in chainages_df.iterrows():
ch = row.get("chainage")
align = row.get("alignment", "Unknown")
level = row.get("level")
radius = row.get("radius")
cant = row.get("cant")
grade = row.get("grade")
text = f"Project: {project_name}. Alignment: {align}. Chainage: {ch:.3f}. Level: {level}. Radius: {radius}. Cant: {cant}. Grade: {grade}."
documents.append(text)
metadatas.append({"source": "chainage", "alignment": align, "chainage": ch})
ids.append(f"chainage_{align}_{ch}")
# Query survey features
survey_df = query_survey_features()
if not survey_df.empty:
for _, row in survey_df.iterrows():
cat = row.get("category", "Unknown")
loc = row.get("chainage")
desc = row.get("description", "")
text = f"Project: {project_name}. Survey feature: {cat} at CH {loc}. Description: {desc}."
documents.append(text)
metadatas.append({"source": "survey", "category": cat, "chainage": loc})
ids.append(f"survey_{cat}_{loc}")
# Add standards
standards = load_standards()
for std_name, std_data in standards.items():
for key, value in std_data.get("limits", {}).items():
if isinstance(value, dict):
limit = value.get("value")
unit = value.get("unit", "")
desc = value.get("description", "")
text = f"Standard: {std_name}. Parameter: {key}. Limit: {limit} {unit}. Description: {desc}."
documents.append(text)
metadatas.append({"source": "standard", "standard": std_name, "parameter": key})
ids.append(f"standard_{std_name}_{key}")
if documents:
# Delete existing project data from collection
try:
collection.delete(ids=ids)
except Exception:
pass
# Add in batches
batch_size = 100
for i in range(0, len(documents), batch_size):
end = min(i + batch_size, len(documents))
try:
embeddings = model.encode(documents[i:end])
collection.add(
ids=ids[i:end],
documents=documents[i:end],
embeddings=embeddings.tolist(),
metadatas=metadatas[i:end]
)
except Exception as e:
logger.warning("Batch %s failed: %s", i, e)
logger.info("Indexed %s documents into ChromaDB for project %s.", len(documents), project_name)
else:
logger.info("No project data to index.")
def query_project_rag(query: str, project_name: str, n_results=None):
"""Query ChromaDB for project-specific data."""
if not CHROMA_AVAILABLE:
return []
collection = get_collection()
if collection is None:
return []
n_results = n_results or ProductionConfig.SEARCH_TOP_K
try:
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
embedding = model.encode([query])
results = collection.query(
query_embeddings=embedding.tolist(),
n_results=n_results,
)
return results
except Exception as e:
logger.warning("RAG query failed: %s", e)
return []