rcs3 commited on
Commit
88cd04d
·
verified ·
1 Parent(s): 2920714

Update db.py

Browse files
Files changed (1) hide show
  1. repochat/db.py +74 -29
repochat/db.py CHANGED
@@ -1,48 +1,93 @@
1
  import os
2
  import shutil
3
  import streamlit as st
4
- from langchain.vectorstores import Chroma
5
- from langchain.text_splitter import RecursiveCharacterTextSplitter
6
- from langchain.document_loaders import NotebookLoader, TextLoader
7
-
8
- def vector_db(embeddings, code):
9
- collection_name = "db_collection"
10
- local_directory = "db_" + st.session_state["db_name"]
11
- persist_directory = os.path.join(os.getcwd(), local_directory)
12
-
13
- if os.path.exists(persist_directory):
14
- shutil.rmtree(persist_directory)
 
 
 
 
 
 
 
 
 
 
 
 
15
 
16
  vec_db = Chroma.from_documents(
17
- documents=code,
18
  embedding=embeddings,
19
- collection_name=collection_name,
20
- persist_directory=persist_directory
21
  )
22
-
23
- vec_db.persist()
24
-
25
  return vec_db
26
 
 
27
  def load_to_db(repo_path):
 
 
 
 
28
  docs = []
 
 
29
  for root, dirs, files in os.walk(repo_path):
30
- dirs[:] = [d for d in dirs if not d.startswith('.')]
 
31
  for filename in files:
32
- if filename.startswith('.'):
33
  continue
34
- if filename == 'package-lock.json':
 
 
35
  continue
 
36
  file_path = os.path.join(root, filename)
37
  try:
38
- if file_path.endswith('.ipynb'):
39
- loader = NotebookLoader(file_path)
40
  else:
41
  loader = TextLoader(file_path, encoding="utf-8")
42
- docs.extend(loader.load_and_split())
43
- except Exception as e:
44
- pass
45
-
46
- code_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
47
- code = code_splitter.split_documents(docs)
48
- return code
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  import shutil
3
  import streamlit as st
4
+ from langchain_chroma import Chroma
5
+ from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
6
+ from langchain_community.document_loaders import NotebookLoader, TextLoader
7
+
8
+ # File extensions to include for indexing
9
+ SUPPORTED_EXTENSIONS = {
10
+ ".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h",
11
+ ".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala",
12
+ ".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb",
13
+ ".html", ".css", ".sql",
14
+ }
15
+
16
+ SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"}
17
+ SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"}
18
+
19
+
20
+ def vector_db(embeddings, documents):
21
+ """Create (or recreate) a Chroma vector store from the given documents."""
22
+ db_name = st.session_state.get("db_name", "default")
23
+ persist_dir = os.path.join(os.getcwd(), f"db_{db_name}")
24
+
25
+ if os.path.exists(persist_dir):
26
+ shutil.rmtree(persist_dir)
27
 
28
  vec_db = Chroma.from_documents(
29
+ documents=documents,
30
  embedding=embeddings,
31
+ collection_name="codebot_collection",
32
+ persist_directory=persist_dir,
33
  )
 
 
 
34
  return vec_db
35
 
36
+
37
  def load_to_db(repo_path):
38
+ """
39
+ Walk the cloned repo, load every supported file, and split into chunks.
40
+ Uses language-aware splitting for Python and JS files.
41
+ """
42
  docs = []
43
+ skipped = 0
44
+
45
  for root, dirs, files in os.walk(repo_path):
46
+ dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")]
47
+
48
  for filename in files:
49
+ if filename.startswith(".") or filename in SKIP_FILES:
50
  continue
51
+
52
+ ext = os.path.splitext(filename)[1].lower()
53
+ if ext not in SUPPORTED_EXTENSIONS:
54
  continue
55
+
56
  file_path = os.path.join(root, filename)
57
  try:
58
+ if ext == ".ipynb":
59
+ loader = NotebookLoader(file_path, include_outputs=False)
60
  else:
61
  loader = TextLoader(file_path, encoding="utf-8")
62
+ docs.extend(loader.load())
63
+ except Exception:
64
+ skipped += 1
65
+
66
+ if not docs:
67
+ st.error("No supported files found in the repository.")
68
+ st.stop()
69
+
70
+ # Language-aware splitter for better code chunking
71
+ splitter = RecursiveCharacterTextSplitter.from_language(
72
+ language=Language.PYTHON,
73
+ chunk_size=1500,
74
+ chunk_overlap=150,
75
+ )
76
+ # Fallback generic splitter for non-Python files
77
+ generic_splitter = RecursiveCharacterTextSplitter(
78
+ chunk_size=1500,
79
+ chunk_overlap=150,
80
+ )
81
+
82
+ chunks = []
83
+ for doc in docs:
84
+ ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower()
85
+ try:
86
+ if ext == ".py":
87
+ chunks.extend(splitter.split_documents([doc]))
88
+ else:
89
+ chunks.extend(generic_splitter.split_documents([doc]))
90
+ except Exception:
91
+ chunks.extend(generic_splitter.split_documents([doc]))
92
+
93
+ return chunks