alu-chatbot / retrieval_engine.py
Donne120
Claude Opus 5
Give every module one definition of the data directory
c8527f1
Raw History Blame Contribute Delete
10.9 kB
import os
import json
from pathlib import Path
from typing import List, Dict, Any, Optional
import numpy as np
# For vector storage and retrieval
import chromadb
from chromadb.utils import embedding_functions
from sentence_transformers import SentenceTransformer
# Check for lightweight mode
LIGHTWEIGHT_MODE = os.environ.get("USE_LIGHTWEIGHT_MODE", "False").lower() == "true"
# Paths come from paths.py so every module that touches the data directory
# agrees on where it is β€” see that module's docstring for what went wrong
# when they each decided independently.
#
# This previously branched on DEPLOYMENT_ENV=huggingface, honouring DATA_DIR
# only on that one host and hardcoding "./data" everywhere else. The
# non-Hugging-Face branch also preferred "./backend/data" when it existed;
# it does not exist in this repo, so that path never ran.
from paths import ( # noqa: E402
ALU_BRAIN_DIR,
DATA_DIR,
DOCUMENTS_DIR,
METADATA_FILE,
VECTOR_DB_DIR,
ensure_dirs,
)
ensure_dirs()
# Maximum chunk size for document splitting
MAX_CHUNK_SIZE = 1000 # characters
MAX_CHUNK_OVERLAP = 200 # characters
# Create directories if they don't exist
DOCUMENTS_DIR.mkdir(parents=True, exist_ok=True)
class Document:
"""Simple document class to store text and metadata"""
def __init__(self, text: str, metadata: Dict[str, Any], score: Optional[float] = None):
self.text = text
self.metadata = metadata
self.score = score
class RetrievalEngine:
"""
Handles document retrieval using vector embeddings:
- Document chunking and embedding
- Semantic search
- Context retrieval for the prompt engine
"""
def __init__(self, collection_name: str = "alu_documents"):
"""
`collection_name` selects which ChromaDB collection this engine reads
and writes. Defaults to "alu_documents" β€” ALU's existing collection β€”
so building an engine with no arguments behaves exactly as before
multi-tenancy existed. A second organization gets its own engine
constructed with its own collection name (see org_registry.py), which
ChromaDB creates fresh and empty the first time it's used: a query
against it can only ever return that organization's documents.
"""
self.collection_name = collection_name
# Initialize the embedding model
self.embedding_model = self.load_embedding_model()
# Initialize ChromaDB
self.client = chromadb.PersistentClient(path=str(VECTOR_DB_DIR))
# Create or get the collection
self.embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(
model_name='all-MiniLM-L6-v2'
)
# get_or_create_collection (rather than get_collection + a try/except
# around the "missing" case) is used deliberately: which exception
# type ChromaDB raises for "collection does not exist" has changed
# across major versions (ValueError historically, a dedicated
# NotFoundError more recently), so catching one specific type here
# would silently stop creating new collections on a different
# ChromaDB version. get_or_create_collection is the version-stable
# way to express "connect if it exists, create it if it doesn't" β€”
# exactly what a brand-new organization's first request needs.
self.collection = self.client.get_or_create_collection(
name=self.collection_name,
embedding_function=self.embedding_function
)
print(f"Vector collection ready: '{self.collection_name}'")
# Initialize document processor reference
from document_processor import DocumentProcessor
self.document_processor = DocumentProcessor()
def load_embedding_model(self):
if LIGHTWEIGHT_MODE:
# Use a smaller model for HF deployment
return SentenceTransformer('paraphrase-MiniLM-L3-v2')
else:
# Your regular model
return SentenceTransformer('all-MiniLM-L6-v2')
def _chunk_text(self, text: str, chunk_size: int = MAX_CHUNK_SIZE, chunk_overlap: int = MAX_CHUNK_OVERLAP) -> List[str]:
"""Split text into chunks with overlap"""
if not text:
return []
chunks = []
start = 0
while start < len(text):
end = min(start + chunk_size, len(text))
# If we're not at the end, try to find a good breakpoint
if end < len(text):
# Look for a period, newline, or space to break at
breakpoints = [text.rfind('.', start, end),
text.rfind('\n', start, end),
text.rfind(' ', start, end)]
# Filter valid breakpoints and find the latest one
valid_breakpoints = [bp for bp in breakpoints if bp != -1]
if valid_breakpoints:
end = max(valid_breakpoints) + 1
# Add this chunk to the list
chunks.append(text[start:end])
# Move the start point, accounting for overlap
start = end - chunk_overlap if end - chunk_overlap > start else end
return chunks
def update_vector_store(self, doc_id: str):
"""Process and add a document to the vector store"""
try:
# Get document text
doc_text = self.document_processor.get_document_text(doc_id)
if not doc_text:
print(f"Document text not found for ID: {doc_id}")
return False
# Get document metadata
with open(METADATA_FILE, "r") as f:
all_metadata = json.load(f)
if doc_id not in all_metadata:
print(f"Document metadata not found for ID: {doc_id}")
return False
metadata = all_metadata[doc_id]
# Chunk the document
chunks = self._chunk_text(doc_text)
# Prepare for batch add
doc_ids = []
metadatas = []
for i, chunk in enumerate(chunks):
chunk_id = f"{doc_id}_chunk_{i}"
doc_ids.append(chunk_id)
# Create metadata for the chunk
chunk_metadata = {
"doc_id": doc_id,
"chunk_id": i,
"title": metadata.get("title", "Untitled"),
"source": metadata.get("source", "Unknown"),
"chunk_index": i,
"total_chunks": len(chunks),
}
metadatas.append(chunk_metadata)
# Add to the collection
if chunks:
self.collection.add(
documents=chunks,
ids=doc_ids,
metadatas=metadatas
)
print(f"Added {len(chunks)} chunks from document {doc_id}")
return True
return False
except Exception as e:
print(f"Error updating vector store: {e}")
return False
def remove_document(self, doc_id: str):
"""Remove a document's chunks from the vector store"""
try:
# Query to find all chunks for this document
results = self.collection.get(
where={"doc_id": doc_id}
)
# Delete the chunks
if results and results.get("ids"):
for chunk_id in results["ids"]:
self.collection.delete(chunk_id)
print(f"Removed {len(results['ids'])} chunks for document {doc_id}")
return True
return False
except Exception as e:
print(f"Error removing document from vector store: {e}")
return False
def rebuild_index(self):
"""Rebuild the entire vector index from scratch"""
try:
# Clear the collection
self.collection.delete(where={})
print("Cleared vector collection")
# Get all document IDs
with open(METADATA_FILE, "r") as f:
all_metadata = json.load(f)
# Add each document
for doc_id in all_metadata.keys():
self.update_vector_store(doc_id)
print(f"Rebuilt vector index with {len(all_metadata)} documents")
return True
except Exception as e:
print(f"Error rebuilding index: {e}")
return False
def retrieve_context(self, query: str, role: str = "student", top_k: int = 8) -> List[Document]:
"""
Retrieve relevant context for a query:
1. Perform semantic search against the vector store
2. Return top matches as Document objects
top_k=8 (was 5): "list all X" questions β€” ML faculty, clubs β€” spread
their answer across several Sheet rows, and top-5 often missed the
siblings, so the model truthfully said it lacked a full list. 8 covers
most such lists without bloating prompt cost.
"""
try:
# Query the collection
results = self.collection.query(
query_texts=[query],
n_results=top_k
)
# Create Document objects
documents = []
if results and results.get("documents") and results.get("documents")[0]:
for i, doc_text in enumerate(results["documents"][0]):
metadata = results["metadatas"][0][i] if results.get("metadatas") and results["metadatas"][0] else {}
# Add distance/score if available
score = None
if results.get("distances") and results["distances"][0]:
score = results["distances"][0][i]
documents.append(Document(
text=doc_text,
metadata=metadata,
score=score
))
# Role-based filtering (could be expanded)
if role != "admin" and role != "faculty":
# For regular students, filter out admin-only content if needed
# (This is just a placeholder for role-based access logic)
pass
return documents
except Exception as e:
print(f"Error retrieving context: {e}")
return []