import os import json from pathlib import Path from typing import List, Dict, Any, Optional import numpy as np # For vector storage and retrieval import chromadb from chromadb.utils import embedding_functions from sentence_transformers import SentenceTransformer # Check for lightweight mode LIGHTWEIGHT_MODE = os.environ.get("USE_LIGHTWEIGHT_MODE", "False").lower() == "true" # Paths come from paths.py so every module that touches the data directory # agrees on where it is — see that module's docstring for what went wrong # when they each decided independently. # # This previously branched on DEPLOYMENT_ENV=huggingface, honouring DATA_DIR # only on that one host and hardcoding "./data" everywhere else. The # non-Hugging-Face branch also preferred "./backend/data" when it existed; # it does not exist in this repo, so that path never ran. from paths import ( # noqa: E402 ALU_BRAIN_DIR, DATA_DIR, DOCUMENTS_DIR, METADATA_FILE, VECTOR_DB_DIR, ensure_dirs, ) ensure_dirs() # Maximum chunk size for document splitting MAX_CHUNK_SIZE = 1000 # characters MAX_CHUNK_OVERLAP = 200 # characters # Create directories if they don't exist DOCUMENTS_DIR.mkdir(parents=True, exist_ok=True) class Document: """Simple document class to store text and metadata""" def __init__(self, text: str, metadata: Dict[str, Any], score: Optional[float] = None): self.text = text self.metadata = metadata self.score = score class RetrievalEngine: """ Handles document retrieval using vector embeddings: - Document chunking and embedding - Semantic search - Context retrieval for the prompt engine """ def __init__(self, collection_name: str = "alu_documents"): """ `collection_name` selects which ChromaDB collection this engine reads and writes. Defaults to "alu_documents" — ALU's existing collection — so building an engine with no arguments behaves exactly as before multi-tenancy existed. A second organization gets its own engine constructed with its own collection name (see org_registry.py), which ChromaDB creates fresh and empty the first time it's used: a query against it can only ever return that organization's documents. """ self.collection_name = collection_name # Initialize the embedding model self.embedding_model = self.load_embedding_model() # Initialize ChromaDB self.client = chromadb.PersistentClient(path=str(VECTOR_DB_DIR)) # Create or get the collection self.embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction( model_name='all-MiniLM-L6-v2' ) # get_or_create_collection (rather than get_collection + a try/except # around the "missing" case) is used deliberately: which exception # type ChromaDB raises for "collection does not exist" has changed # across major versions (ValueError historically, a dedicated # NotFoundError more recently), so catching one specific type here # would silently stop creating new collections on a different # ChromaDB version. get_or_create_collection is the version-stable # way to express "connect if it exists, create it if it doesn't" — # exactly what a brand-new organization's first request needs. self.collection = self.client.get_or_create_collection( name=self.collection_name, embedding_function=self.embedding_function ) print(f"Vector collection ready: '{self.collection_name}'") # Initialize document processor reference from document_processor import DocumentProcessor self.document_processor = DocumentProcessor() def load_embedding_model(self): if LIGHTWEIGHT_MODE: # Use a smaller model for HF deployment return SentenceTransformer('paraphrase-MiniLM-L3-v2') else: # Your regular model return SentenceTransformer('all-MiniLM-L6-v2') def _chunk_text(self, text: str, chunk_size: int = MAX_CHUNK_SIZE, chunk_overlap: int = MAX_CHUNK_OVERLAP) -> List[str]: """Split text into chunks with overlap""" if not text: return [] chunks = [] start = 0 while start < len(text): end = min(start + chunk_size, len(text)) # If we're not at the end, try to find a good breakpoint if end < len(text): # Look for a period, newline, or space to break at breakpoints = [text.rfind('.', start, end), text.rfind('\n', start, end), text.rfind(' ', start, end)] # Filter valid breakpoints and find the latest one valid_breakpoints = [bp for bp in breakpoints if bp != -1] if valid_breakpoints: end = max(valid_breakpoints) + 1 # Add this chunk to the list chunks.append(text[start:end]) # Move the start point, accounting for overlap start = end - chunk_overlap if end - chunk_overlap > start else end return chunks def update_vector_store(self, doc_id: str): """Process and add a document to the vector store""" try: # Get document text doc_text = self.document_processor.get_document_text(doc_id) if not doc_text: print(f"Document text not found for ID: {doc_id}") return False # Get document metadata with open(METADATA_FILE, "r") as f: all_metadata = json.load(f) if doc_id not in all_metadata: print(f"Document metadata not found for ID: {doc_id}") return False metadata = all_metadata[doc_id] # Chunk the document chunks = self._chunk_text(doc_text) # Prepare for batch add doc_ids = [] metadatas = [] for i, chunk in enumerate(chunks): chunk_id = f"{doc_id}_chunk_{i}" doc_ids.append(chunk_id) # Create metadata for the chunk chunk_metadata = { "doc_id": doc_id, "chunk_id": i, "title": metadata.get("title", "Untitled"), "source": metadata.get("source", "Unknown"), "chunk_index": i, "total_chunks": len(chunks), } metadatas.append(chunk_metadata) # Add to the collection if chunks: self.collection.add( documents=chunks, ids=doc_ids, metadatas=metadatas ) print(f"Added {len(chunks)} chunks from document {doc_id}") return True return False except Exception as e: print(f"Error updating vector store: {e}") return False def remove_document(self, doc_id: str): """Remove a document's chunks from the vector store""" try: # Query to find all chunks for this document results = self.collection.get( where={"doc_id": doc_id} ) # Delete the chunks if results and results.get("ids"): for chunk_id in results["ids"]: self.collection.delete(chunk_id) print(f"Removed {len(results['ids'])} chunks for document {doc_id}") return True return False except Exception as e: print(f"Error removing document from vector store: {e}") return False def rebuild_index(self): """Rebuild the entire vector index from scratch""" try: # Clear the collection self.collection.delete(where={}) print("Cleared vector collection") # Get all document IDs with open(METADATA_FILE, "r") as f: all_metadata = json.load(f) # Add each document for doc_id in all_metadata.keys(): self.update_vector_store(doc_id) print(f"Rebuilt vector index with {len(all_metadata)} documents") return True except Exception as e: print(f"Error rebuilding index: {e}") return False def retrieve_context(self, query: str, role: str = "student", top_k: int = 8) -> List[Document]: """ Retrieve relevant context for a query: 1. Perform semantic search against the vector store 2. Return top matches as Document objects top_k=8 (was 5): "list all X" questions — ML faculty, clubs — spread their answer across several Sheet rows, and top-5 often missed the siblings, so the model truthfully said it lacked a full list. 8 covers most such lists without bloating prompt cost. """ try: # Query the collection results = self.collection.query( query_texts=[query], n_results=top_k ) # Create Document objects documents = [] if results and results.get("documents") and results.get("documents")[0]: for i, doc_text in enumerate(results["documents"][0]): metadata = results["metadatas"][0][i] if results.get("metadatas") and results["metadatas"][0] else {} # Add distance/score if available score = None if results.get("distances") and results["distances"][0]: score = results["distances"][0][i] documents.append(Document( text=doc_text, metadata=metadata, score=score )) # Role-based filtering (could be expanded) if role != "admin" and role != "faculty": # For regular students, filter out admin-only content if needed # (This is just a placeholder for role-based access logic) pass return documents except Exception as e: print(f"Error retrieving context: {e}") return []