Download core/chroma_project.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 5.08 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/chroma_project.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/core/chroma_project.py
-
curl -L -o chroma_project.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/core/chroma_project.py
5.08 kB
| """ | |
| core/chroma_project.py | |
| Project-specific ChromaDB integration. | |
| """ | |
| import os | |
| import json | |
| import pandas as pd | |
| from pathlib import Path | |
| from core.database import query_chainages, query_survey_features | |
| from core.standards_loader import load_standards | |
| try: | |
| import chromadb | |
| from sentence_transformers import SentenceTransformer | |
| CHROMA_AVAILABLE = True | |
| except ImportError: | |
| CHROMA_AVAILABLE = False | |
| from core.config import ProductionConfig | |
| from core.audit_logger import get_logger | |
| logger = get_logger("core.chroma_project") | |
| CHROMA_PATH = ProductionConfig.CHROMA_PATH | |
| def get_collection(collection_name="track_design_knowledge"): | |
| """Get or create a ChromaDB collection.""" | |
| if not CHROMA_AVAILABLE: | |
| return None | |
| client = chromadb.PersistentClient(path=str(CHROMA_PATH)) | |
| try: | |
| collection = client.get_collection(collection_name) | |
| except Exception: | |
| collection = client.create_collection(collection_name) | |
| return collection | |
| def index_project_data(project_name: str): | |
| """ | |
| Index current project data (chainages, survey, standards) into ChromaDB. | |
| This makes the project data searchable via RAG. | |
| """ | |
| if not CHROMA_AVAILABLE: | |
| logger.info("ChromaDB not available - skipping indexing.") | |
| return | |
| collection = get_collection() | |
| if collection is None: | |
| return | |
| # Load embedding model | |
| try: | |
| model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) | |
| except Exception as e: | |
| logger.warning("Embedding model load failed: %s", e) | |
| return | |
| # Query chainages | |
| chainages_df = query_chainages() | |
| documents = [] | |
| metadatas = [] | |
| ids = [] | |
| if not chainages_df.empty: | |
| for _, row in chainages_df.iterrows(): | |
| ch = row.get("chainage") | |
| align = row.get("alignment", "Unknown") | |
| level = row.get("level") | |
| radius = row.get("radius") | |
| cant = row.get("cant") | |
| grade = row.get("grade") | |
| text = f"Project: {project_name}. Alignment: {align}. Chainage: {ch:.3f}. Level: {level}. Radius: {radius}. Cant: {cant}. Grade: {grade}." | |
| documents.append(text) | |
| metadatas.append({"source": "chainage", "alignment": align, "chainage": ch}) | |
| ids.append(f"chainage_{align}_{ch}") | |
| # Query survey features | |
| survey_df = query_survey_features() | |
| if not survey_df.empty: | |
| for _, row in survey_df.iterrows(): | |
| cat = row.get("category", "Unknown") | |
| loc = row.get("chainage") | |
| desc = row.get("description", "") | |
| text = f"Project: {project_name}. Survey feature: {cat} at CH {loc}. Description: {desc}." | |
| documents.append(text) | |
| metadatas.append({"source": "survey", "category": cat, "chainage": loc}) | |
| ids.append(f"survey_{cat}_{loc}") | |
| # Add standards | |
| standards = load_standards() | |
| for std_name, std_data in standards.items(): | |
| for key, value in std_data.get("limits", {}).items(): | |
| if isinstance(value, dict): | |
| limit = value.get("value") | |
| unit = value.get("unit", "") | |
| desc = value.get("description", "") | |
| text = f"Standard: {std_name}. Parameter: {key}. Limit: {limit} {unit}. Description: {desc}." | |
| documents.append(text) | |
| metadatas.append({"source": "standard", "standard": std_name, "parameter": key}) | |
| ids.append(f"standard_{std_name}_{key}") | |
| if documents: | |
| # Delete existing project data from collection | |
| try: | |
| collection.delete(ids=ids) | |
| except Exception: | |
| pass | |
| # Add in batches | |
| batch_size = 100 | |
| for i in range(0, len(documents), batch_size): | |
| end = min(i + batch_size, len(documents)) | |
| try: | |
| embeddings = model.encode(documents[i:end]) | |
| collection.add( | |
| ids=ids[i:end], | |
| documents=documents[i:end], | |
| embeddings=embeddings.tolist(), | |
| metadatas=metadatas[i:end] | |
| ) | |
| except Exception as e: | |
| logger.warning("Batch %s failed: %s", i, e) | |
| logger.info("Indexed %s documents into ChromaDB for project %s.", len(documents), project_name) | |
| else: | |
| logger.info("No project data to index.") | |
| def query_project_rag(query: str, project_name: str, n_results=None): | |
| """Query ChromaDB for project-specific data.""" | |
| if not CHROMA_AVAILABLE: | |
| return [] | |
| collection = get_collection() | |
| if collection is None: | |
| return [] | |
| n_results = n_results or ProductionConfig.SEARCH_TOP_K | |
| try: | |
| model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) | |
| embedding = model.encode([query]) | |
| results = collection.query( | |
| query_embeddings=embedding.tolist(), | |
| n_results=n_results, | |
| ) | |
| return results | |
| except Exception as e: | |
| logger.warning("RAG query failed: %s", e) | |
| return [] | |