Download repochat/db.py from rcs3/Codebot: direct link, hf CLI and curl.
- Browser
- Download file 3.09 kB
-
https://huggingface.co/spaces/rcs3/Codebot/resolve/main/repochat/db.py
- Command line
-
hf download hf://spaces/rcs3/Codebot/repochat/db.py
-
curl -L -o db.py https://huggingface.co/spaces/rcs3/Codebot/resolve/main/repochat/db.py
3.09 kB
| import os | |
| import shutil | |
| import streamlit as st | |
| from langchain_chroma import Chroma | |
| from langchain.text_splitter import Language, RecursiveCharacterTextSplitter | |
| from langchain_community.document_loaders import NotebookLoader, TextLoader | |
| # File extensions to include for indexing | |
| SUPPORTED_EXTENSIONS = { | |
| ".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h", | |
| ".cs", ".go", ".rs", ".rb", ".php", ".swift", ".kt", ".scala", | |
| ".md", ".txt", ".yaml", ".yml", ".toml", ".json", ".sh", ".ipynb", | |
| ".html", ".css", ".sql", | |
| } | |
| SKIP_FILES = {"package-lock.json", "yarn.lock", "poetry.lock", "Pipfile.lock"} | |
| SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"} | |
| def vector_db(embeddings, documents): | |
| """Create (or recreate) a Chroma vector store from the given documents.""" | |
| db_name = st.session_state.get("db_name", "default") | |
| persist_dir = os.path.join(os.getcwd(), f"db_{db_name}") | |
| if os.path.exists(persist_dir): | |
| shutil.rmtree(persist_dir) | |
| vec_db = Chroma.from_documents( | |
| documents=documents, | |
| embedding=embeddings, | |
| collection_name="codebot_collection", | |
| persist_directory=persist_dir, | |
| ) | |
| return vec_db | |
| def load_to_db(repo_path): | |
| """ | |
| Walk the cloned repo, load every supported file, and split into chunks. | |
| Uses language-aware splitting for Python and JS files. | |
| """ | |
| docs = [] | |
| skipped = 0 | |
| for root, dirs, files in os.walk(repo_path): | |
| dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")] | |
| for filename in files: | |
| if filename.startswith(".") or filename in SKIP_FILES: | |
| continue | |
| ext = os.path.splitext(filename)[1].lower() | |
| if ext not in SUPPORTED_EXTENSIONS: | |
| continue | |
| file_path = os.path.join(root, filename) | |
| try: | |
| if ext == ".ipynb": | |
| loader = NotebookLoader(file_path, include_outputs=False) | |
| else: | |
| loader = TextLoader(file_path, encoding="utf-8") | |
| docs.extend(loader.load()) | |
| except Exception: | |
| skipped += 1 | |
| if not docs: | |
| st.error("No supported files found in the repository.") | |
| st.stop() | |
| # Language-aware splitter for better code chunking | |
| splitter = RecursiveCharacterTextSplitter.from_language( | |
| language=Language.PYTHON, | |
| chunk_size=1500, | |
| chunk_overlap=150, | |
| ) | |
| # Fallback generic splitter for non-Python files | |
| generic_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1500, | |
| chunk_overlap=150, | |
| ) | |
| chunks = [] | |
| for doc in docs: | |
| ext = os.path.splitext(doc.metadata.get("source", ""))[1].lower() | |
| try: | |
| if ext == ".py": | |
| chunks.extend(splitter.split_documents([doc])) | |
| else: | |
| chunks.extend(generic_splitter.split_documents([doc])) | |
| except Exception: | |
| chunks.extend(generic_splitter.split_documents([doc])) | |
| return chunks |