| |
| """Bootstrap script: populates the `code` LanceDB collection from the |
| local repo tree, so the vector store isn't empty on first deploy. |
| |
| Usage: python bootstrap_vector_store.py [repo_root] |
| |
| Chunking is deliberately simple (fixed line-count windows) rather than |
| AST-aware — good enough to make code semantically searchable without |
| pulling in a language-specific parser for this first pass. |
| """ |
| import sys |
| from pathlib import Path |
|
|
| import vector_store |
|
|
| INCLUDE_SUFFIXES = {".rs", ".py", ".toml", ".md"} |
| EXCLUDE_DIRS = {"target", "__pycache__", ".git", "node_modules"} |
| CHUNK_LINES = 60 |
| CHUNK_OVERLAP = 10 |
|
|
|
|
| def iter_source_files(root: Path): |
| for path in root.rglob("*"): |
| if path.is_dir(): |
| continue |
| if any(part in EXCLUDE_DIRS for part in path.parts): |
| continue |
| if path.suffix in INCLUDE_SUFFIXES: |
| yield path |
|
|
|
|
| def chunk_lines(lines: list[str]) -> list[str]: |
| if not lines: |
| return [] |
| chunks = [] |
| step = CHUNK_LINES - CHUNK_OVERLAP |
| for start in range(0, len(lines), step): |
| chunk = "".join(lines[start : start + CHUNK_LINES]) |
| if chunk.strip(): |
| chunks.append(chunk) |
| if start + CHUNK_LINES >= len(lines): |
| break |
| return chunks |
|
|
|
|
| def main(): |
| root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parents[1] |
| if not root.exists(): |
| print(f"error: {root} does not exist") |
| sys.exit(1) |
|
|
| total_files = 0 |
| total_chunks = 0 |
| for path in iter_source_files(root): |
| try: |
| lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) |
| except OSError as exc: |
| print(f"skip {path}: {exc}") |
| continue |
|
|
| rel_path = str(path.relative_to(root)) |
| for i, chunk in enumerate(chunk_lines(lines)): |
| vector_store.ingest_code_chunk(rel_path, i, chunk) |
| total_chunks += 1 |
| total_files += 1 |
| print(f"indexed {rel_path} ({len(lines)} lines)") |
|
|
| print(f"done: {total_files} files, {total_chunks} chunks indexed into 'code' collection") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
