File size: 3,093 Bytes
462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d 462062c 88cd04d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | import os
import shutil
import streamlit as st
from langchain_chroma import Chroma
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
from langchain_community.document_loaders import NotebookLoader, TextLoader
# File extensions to include for indexing
SUPPORTED_EXTENSIONS = {
".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h",
".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala",
".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb",
".html", ".css", ".sql",
}
SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"}
SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"}
def vector_db(embeddings, documents):
"""Create (or recreate) a Chroma vector store from the given documents."""
db_name = st.session_state.get("db_name", "default")
persist_dir = os.path.join(os.getcwd(), f"db_{db_name}")
if os.path.exists(persist_dir):
shutil.rmtree(persist_dir)
vec_db = Chroma.from_documents(
documents=documents,
embedding=embeddings,
collection_name="codebot_collection",
persist_directory=persist_dir,
)
return vec_db
def load_to_db(repo_path):
"""
Walk the cloned repo, load every supported file, and split into chunks.
Uses language-aware splitting for Python and JS files.
"""
docs = []
skipped = 0
for root, dirs, files in os.walk(repo_path):
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]
for filename in files:
if filename.startswith(".") or filename in SKIP_FILES:
continue
ext = os.path.splitext(filename)[1].lower()
if ext not in SUPPORTED_EXTENSIONS:
continue
file_path = os.path.join(root, filename)
try:
if ext == ".ipynb":
loader = NotebookLoader(file_path, include_outputs=False)
else:
loader = TextLoader(file_path, encoding="utf-8")
docs.extend(loader.load())
except Exception:
skipped += 1
if not docs:
st.error("No supported files found in the repository.")
st.stop()
# Language-aware splitter for better code chunking
splitter = RecursiveCharacterTextSplitter.from_language(
language=Language.PYTHON,
chunk_size=1500,
chunk_overlap=150,
)
# Fallback generic splitter for non-Python files
generic_splitter = RecursiveCharacterTextSplitter(
chunk_size=1500,
chunk_overlap=150,
)
chunks = []
for doc in docs:
ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower()
try:
if ext == ".py":
chunks.extend(splitter.split_documents([doc]))
else:
chunks.extend(generic_splitter.split_documents([doc]))
except Exception:
chunks.extend(generic_splitter.split_documents([doc]))
return chunks |