import os import shutil import streamlit as st from langchain_chroma import Chroma from langchain.text_splitter import Language, RecursiveCharacterTextSplitter from langchain_community.document_loaders import NotebookLoader, TextLoader # File extensions to include for indexing SUPPORTED_EXTENSIONS = { ".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h", ".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala", ".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb", ".html", ".css", ".sql", } SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"} SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"} def vector_db(embeddings, documents): """Create (or recreate) a Chroma vector store from the given documents.""" db_name = st.session_state.get("db_name", "default") persist_dir = os.path.join(os.getcwd(), f"db_{db_name}") if os.path.exists(persist_dir): shutil.rmtree(persist_dir) vec_db = Chroma.from_documents( documents=documents, embedding=embeddings, collection_name="codebot_collection", persist_directory=persist_dir, ) return vec_db def load_to_db(repo_path): """ Walk the cloned repo, load every supported file, and split into chunks. Uses language-aware splitting for Python and JS files. """ docs = [] skipped = 0 for root, dirs, files in os.walk(repo_path): dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")] for filename in files: if filename.startswith(".") or filename in SKIP_FILES: continue ext = os.path.splitext(filename)[1].lower() if ext not in SUPPORTED_EXTENSIONS: continue file_path = os.path.join(root, filename) try: if ext == ".ipynb": loader = NotebookLoader(file_path, include_outputs=False) else: loader = TextLoader(file_path, encoding="utf-8") docs.extend(loader.load()) except Exception: skipped += 1 if not docs: st.error("No supported files found in the repository.") st.stop() # Language-aware splitter for better code chunking splitter = RecursiveCharacterTextSplitter.from_language( language=Language.PYTHON, chunk_size=1500, chunk_overlap=150, ) # Fallback generic splitter for non-Python files generic_splitter = RecursiveCharacterTextSplitter( chunk_size=1500, chunk_overlap=150, ) chunks = [] for doc in docs: ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower() try: if ext == ".py": chunks.extend(splitter.split_documents([doc])) else: chunks.extend(generic_splitter.split_documents([doc])) except Exception: chunks.extend(generic_splitter.split_documents([doc])) return chunks