File size: 3,093 Bytes
462062c
 
 
88cd04d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
462062c
 
88cd04d
462062c
88cd04d
 
462062c
 
 
88cd04d
462062c
88cd04d
 
 
 
462062c
88cd04d
 
462062c
88cd04d
 
462062c
88cd04d
462062c
88cd04d
 
 
462062c
88cd04d
462062c
 
88cd04d
 
462062c
 
88cd04d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
import os
import shutil
import streamlit as st
from langchain_chroma import Chroma
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
from langchain_community.document_loaders import NotebookLoader, TextLoader

# File extensions to include for indexing
SUPPORTED_EXTENSIONS = {
    ".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h",
    ".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala",
    ".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb",
    ".html", ".css", ".sql",
}

SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"}
SKIP_DIRS  = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"}


def vector_db(embeddings, documents):
    """Create (or recreate) a Chroma vector store from the given documents."""
    db_name  = st.session_state.get("db_name", "default")
    persist_dir = os.path.join(os.getcwd(), f"db_{db_name}")

    if os.path.exists(persist_dir):
        shutil.rmtree(persist_dir)

    vec_db = Chroma.from_documents(
        documents=documents,
        embedding=embeddings,
        collection_name="codebot_collection",
        persist_directory=persist_dir,
    )
    return vec_db


def load_to_db(repo_path):
    """
    Walk the cloned repo, load every supported file, and split into chunks.
    Uses language-aware splitting for Python and JS files.
    """
    docs = []
    skipped = 0

    for root, dirs, files in os.walk(repo_path):
        dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]

        for filename in files:
            if filename.startswith(".") or filename in SKIP_FILES:
                continue

            ext = os.path.splitext(filename)[1].lower()
            if ext not in SUPPORTED_EXTENSIONS:
                continue

            file_path = os.path.join(root, filename)
            try:
                if ext == ".ipynb":
                    loader = NotebookLoader(file_path, include_outputs=False)
                else:
                    loader = TextLoader(file_path, encoding="utf-8")
                docs.extend(loader.load())
            except Exception:
                skipped += 1

    if not docs:
        st.error("No supported files found in the repository.")
        st.stop()

    # Language-aware splitter for better code chunking
    splitter = RecursiveCharacterTextSplitter.from_language(
        language=Language.PYTHON,
        chunk_size=1500,
        chunk_overlap=150,
    )
    # Fallback generic splitter for non-Python files
    generic_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1500,
        chunk_overlap=150,
    )

    chunks = []
    for doc in docs:
        ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower()
        try:
            if ext == ".py":
                chunks.extend(splitter.split_documents([doc]))
            else:
                chunks.extend(generic_splitter.split_documents([doc]))
        except Exception:
            chunks.extend(generic_splitter.split_documents([doc]))

    return chunks