Update db.py
Browse files- repochat/db.py +74 -29
repochat/db.py
CHANGED
|
@@ -1,48 +1,93 @@
|
|
| 1 |
import os
|
| 2 |
import shutil
|
| 3 |
import streamlit as st
|
| 4 |
-
from
|
| 5 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 6 |
-
from
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
vec_db = Chroma.from_documents(
|
| 17 |
-
documents=
|
| 18 |
embedding=embeddings,
|
| 19 |
-
collection_name=
|
| 20 |
-
persist_directory=
|
| 21 |
)
|
| 22 |
-
|
| 23 |
-
vec_db.persist()
|
| 24 |
-
|
| 25 |
return vec_db
|
| 26 |
|
|
|
|
| 27 |
def load_to_db(repo_path):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
docs = []
|
|
|
|
|
|
|
| 29 |
for root, dirs, files in os.walk(repo_path):
|
| 30 |
-
dirs[:] = [d for d in dirs if not d.startswith(
|
|
|
|
| 31 |
for filename in files:
|
| 32 |
-
if filename.startswith(
|
| 33 |
continue
|
| 34 |
-
|
|
|
|
|
|
|
| 35 |
continue
|
|
|
|
| 36 |
file_path = os.path.join(root, filename)
|
| 37 |
try:
|
| 38 |
-
if
|
| 39 |
-
loader = NotebookLoader(file_path)
|
| 40 |
else:
|
| 41 |
loader = TextLoader(file_path, encoding="utf-8")
|
| 42 |
-
|
| 43 |
-
except Exception
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
import shutil
|
| 3 |
import streamlit as st
|
| 4 |
+
from langchain_chroma import Chroma
|
| 5 |
+
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
|
| 6 |
+
from langchain_community.document_loaders import NotebookLoader, TextLoader
|
| 7 |
+
|
| 8 |
+
# File extensions to include for indexing
|
| 9 |
+
SUPPORTED_EXTENSIONS = {
|
| 10 |
+
".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h",
|
| 11 |
+
".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala",
|
| 12 |
+
".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb",
|
| 13 |
+
".html", ".css", ".sql",
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"}
|
| 17 |
+
SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def vector_db(embeddings, documents):
|
| 21 |
+
"""Create (or recreate) a Chroma vector store from the given documents."""
|
| 22 |
+
db_name = st.session_state.get("db_name", "default")
|
| 23 |
+
persist_dir = os.path.join(os.getcwd(), f"db_{db_name}")
|
| 24 |
+
|
| 25 |
+
if os.path.exists(persist_dir):
|
| 26 |
+
shutil.rmtree(persist_dir)
|
| 27 |
|
| 28 |
vec_db = Chroma.from_documents(
|
| 29 |
+
documents=documents,
|
| 30 |
embedding=embeddings,
|
| 31 |
+
collection_name="codebot_collection",
|
| 32 |
+
persist_directory=persist_dir,
|
| 33 |
)
|
|
|
|
|
|
|
|
|
|
| 34 |
return vec_db
|
| 35 |
|
| 36 |
+
|
| 37 |
def load_to_db(repo_path):
|
| 38 |
+
"""
|
| 39 |
+
Walk the cloned repo, load every supported file, and split into chunks.
|
| 40 |
+
Uses language-aware splitting for Python and JS files.
|
| 41 |
+
"""
|
| 42 |
docs = []
|
| 43 |
+
skipped = 0
|
| 44 |
+
|
| 45 |
for root, dirs, files in os.walk(repo_path):
|
| 46 |
+
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]
|
| 47 |
+
|
| 48 |
for filename in files:
|
| 49 |
+
if filename.startswith(".") or filename in SKIP_FILES:
|
| 50 |
continue
|
| 51 |
+
|
| 52 |
+
ext = os.path.splitext(filename)[1].lower()
|
| 53 |
+
if ext not in SUPPORTED_EXTENSIONS:
|
| 54 |
continue
|
| 55 |
+
|
| 56 |
file_path = os.path.join(root, filename)
|
| 57 |
try:
|
| 58 |
+
if ext == ".ipynb":
|
| 59 |
+
loader = NotebookLoader(file_path, include_outputs=False)
|
| 60 |
else:
|
| 61 |
loader = TextLoader(file_path, encoding="utf-8")
|
| 62 |
+
docs.extend(loader.load())
|
| 63 |
+
except Exception:
|
| 64 |
+
skipped += 1
|
| 65 |
+
|
| 66 |
+
if not docs:
|
| 67 |
+
st.error("No supported files found in the repository.")
|
| 68 |
+
st.stop()
|
| 69 |
+
|
| 70 |
+
# Language-aware splitter for better code chunking
|
| 71 |
+
splitter = RecursiveCharacterTextSplitter.from_language(
|
| 72 |
+
language=Language.PYTHON,
|
| 73 |
+
chunk_size=1500,
|
| 74 |
+
chunk_overlap=150,
|
| 75 |
+
)
|
| 76 |
+
# Fallback generic splitter for non-Python files
|
| 77 |
+
generic_splitter = RecursiveCharacterTextSplitter(
|
| 78 |
+
chunk_size=1500,
|
| 79 |
+
chunk_overlap=150,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
chunks = []
|
| 83 |
+
for doc in docs:
|
| 84 |
+
ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower()
|
| 85 |
+
try:
|
| 86 |
+
if ext == ".py":
|
| 87 |
+
chunks.extend(splitter.split_documents([doc]))
|
| 88 |
+
else:
|
| 89 |
+
chunks.extend(generic_splitter.split_documents([doc]))
|
| 90 |
+
except Exception:
|
| 91 |
+
chunks.extend(generic_splitter.split_documents([doc]))
|
| 92 |
+
|
| 93 |
+
return chunks
|