Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /contextfs_contextfs /tests /integration /test_codebase.py
| """ | |
| Integration tests for indexing and processing codebases. | |
| Uses fixture-generated test repositories to ensure reproducible tests | |
| without external dependencies. | |
| """ | |
| from pathlib import Path | |
| import pytest | |
| def sample_project(temp_dir: Path) -> Path: | |
| """Create a sample multi-language project for testing.""" | |
| # Create project structure | |
| src_dir = temp_dir / "src" | |
| src_dir.mkdir() | |
| # Python files | |
| (src_dir / "main.py").write_text('''"""Main application module.""" | |
| import os | |
| from typing import Optional | |
| from utils import helper_function | |
| class Application: | |
| """Main application class.""" | |
| def __init__(self, name: str): | |
| self.name = name | |
| self.config = {} | |
| def run(self) -> None: | |
| """Run the application.""" | |
| print(f"Running {self.name}") | |
| helper_function() | |
| def main(): | |
| """Entry point.""" | |
| app = Application("TestApp") | |
| app.run() | |
| if __name__ == "__main__": | |
| main() | |
| ''') | |
| (src_dir / "utils.py").write_text('''"""Utility functions.""" | |
| from typing import Any, List | |
| def helper_function() -> None: | |
| """A helper function.""" | |
| print("Helper called") | |
| def process_data(items: List[Any]) -> List[Any]: | |
| """Process a list of items.""" | |
| return [item for item in items if item is not None] | |
| class DataProcessor: | |
| """Process data with various strategies.""" | |
| def __init__(self, strategy: str = "default"): | |
| self.strategy = strategy | |
| def process(self, data: Any) -> Any: | |
| """Process data using configured strategy.""" | |
| if self.strategy == "default": | |
| return data | |
| return None | |
| ''') | |
| # TypeScript files | |
| components_dir = src_dir / "components" | |
| components_dir.mkdir() | |
| (components_dir / "Button.tsx").write_text("""import React from 'react'; | |
| interface ButtonProps { | |
| label: string; | |
| onClick: () => void; | |
| disabled?: boolean; | |
| variant?: 'primary' | 'secondary'; | |
| } | |
| export const Button: React.FC<ButtonProps> = ({ | |
| label, | |
| onClick, | |
| disabled = false, | |
| variant = 'primary' | |
| }) => { | |
| return ( | |
| <button | |
| className={`btn btn-${variant}`} | |
| onClick={onClick} | |
| disabled={disabled} | |
| > | |
| {label} | |
| </button> | |
| ); | |
| }; | |
| export default Button; | |
| """) | |
| (components_dir / "Card.tsx").write_text("""import React from 'react'; | |
| import { Button } from './Button'; | |
| interface CardProps { | |
| title: string; | |
| content: string; | |
| onAction?: () => void; | |
| } | |
| export const Card: React.FC<CardProps> = ({ title, content, onAction }) => { | |
| return ( | |
| <div className="card"> | |
| <h2>{title}</h2> | |
| <p>{content}</p> | |
| {onAction && <Button label="Action" onClick={onAction} />} | |
| </div> | |
| ); | |
| }; | |
| """) | |
| # JavaScript file | |
| (src_dir / "config.js").write_text("""// Configuration module | |
| const config = { | |
| apiUrl: process.env.API_URL || 'http://localhost:3000', | |
| timeout: 5000, | |
| retries: 3, | |
| }; | |
| function getConfig(key) { | |
| return config[key]; | |
| } | |
| function setConfig(key, value) { | |
| config[key] = value; | |
| } | |
| module.exports = { config, getConfig, setConfig }; | |
| """) | |
| # Go file | |
| (src_dir / "server.go").write_text("""package main | |
| import ( | |
| "fmt" | |
| "net/http" | |
| ) | |
| // Server represents an HTTP server | |
| type Server struct { | |
| Port int | |
| Host string | |
| } | |
| // NewServer creates a new server instance | |
| func NewServer(host string, port int) *Server { | |
| return &Server{ | |
| Host: host, | |
| Port: port, | |
| } | |
| } | |
| // Start starts the server | |
| func (s *Server) Start() error { | |
| addr := fmt.Sprintf("%s:%d", s.Host, s.Port) | |
| return http.ListenAndServe(addr, nil) | |
| } | |
| func main() { | |
| server := NewServer("localhost", 8080) | |
| server.Start() | |
| } | |
| """) | |
| # Rust file | |
| (src_dir / "lib.rs").write_text("""//! Library module for data processing | |
| use std::collections::HashMap; | |
| /// A data store for key-value pairs | |
| pub struct DataStore { | |
| data: HashMap<String, String>, | |
| } | |
| impl DataStore { | |
| /// Create a new empty data store | |
| pub fn new() -> Self { | |
| DataStore { | |
| data: HashMap::new(), | |
| } | |
| } | |
| /// Insert a key-value pair | |
| pub fn insert(&mut self, key: String, value: String) { | |
| self.data.insert(key, value); | |
| } | |
| /// Get a value by key | |
| pub fn get(&self, key: &str) -> Option<&String> { | |
| self.data.get(key) | |
| } | |
| } | |
| #[cfg(test)] | |
| mod tests { | |
| use super::*; | |
| #[test] | |
| fn test_insert_and_get() { | |
| let mut store = DataStore::new(); | |
| store.insert("key".to_string(), "value".to_string()); | |
| assert_eq!(store.get("key"), Some(&"value".to_string())); | |
| } | |
| } | |
| """) | |
| return temp_dir | |
| class TestCodebaseProcessing: | |
| """Tests for processing generic codebases.""" | |
| def test_detect_project_files(self, sample_project: Path): | |
| """Test detecting file types in a project.""" | |
| from contextfs.filetypes.registry import FileTypeRegistry | |
| registry = FileTypeRegistry() | |
| # Find all source files | |
| extensions_found = set() | |
| for file in sample_project.rglob("*"): | |
| if file.is_file(): | |
| ext = file.suffix | |
| if ext and registry.supports_extension(ext): | |
| extensions_found.add(ext) | |
| # Should find multiple file types | |
| assert len(extensions_found) >= 3 | |
| assert ".py" in extensions_found | |
| assert ".tsx" in extensions_found or ".ts" in extensions_found | |
| def test_parse_python_files(self, sample_project: Path): | |
| """Test parsing Python files.""" | |
| from contextfs.filetypes.registry import get_handler | |
| handler = get_handler(file_path="test.py") | |
| assert handler is not None | |
| py_files = list(sample_project.glob("**/*.py")) | |
| assert len(py_files) >= 1 | |
| for test_file in py_files: | |
| content = test_file.read_text(encoding="utf-8") | |
| doc = handler.parse(content, str(test_file)) | |
| assert doc is not None | |
| assert doc.file_type == "python" | |
| assert doc.line_count > 0 | |
| def test_parse_typescript_files(self, sample_project: Path): | |
| """Test parsing TypeScript files.""" | |
| from contextfs.filetypes.registry import get_handler | |
| handler = get_handler(file_path="test.tsx") | |
| assert handler is not None | |
| ts_files = list(sample_project.glob("**/*.tsx")) | |
| assert len(ts_files) >= 1 | |
| for test_file in ts_files: | |
| content = test_file.read_text(encoding="utf-8") | |
| doc = handler.parse(content, str(test_file)) | |
| assert doc is not None | |
| assert doc.file_type in ["typescript", "javascript"] | |
| assert doc.line_count > 0 | |
| def test_process_project_directory(self, sample_project: Path): | |
| """Test processing an entire project directory.""" | |
| from contextfs.filetypes.integration import SmartDocumentProcessor | |
| processor = SmartDocumentProcessor() | |
| src_dir = sample_project / "src" | |
| chunks = processor.process_directory( | |
| src_dir, | |
| extensions=[".py", ".ts", ".tsx", ".js", ".go", ".rs"], | |
| recursive=True, | |
| ) | |
| assert len(chunks) > 0 | |
| # Check chunk metadata | |
| for chunk in chunks: | |
| assert "file_type" in chunk["metadata"] | |
| assert "source_file" in chunk["metadata"] | |
| # Should have chunks from multiple file types | |
| file_types = {c["metadata"]["file_type"] for c in chunks} | |
| assert len(file_types) >= 2 | |
| def test_index_and_search_codebase(self, sample_project: Path, temp_dir: Path): | |
| """Test indexing codebase and searching.""" | |
| from contextfs.filetypes.integration import RAGIntegration | |
| from contextfs.rag import RAGBackend | |
| # Create RAG backend in separate temp dir | |
| rag_dir = temp_dir / "rag_data" | |
| rag_dir.mkdir() | |
| rag = RAGBackend( | |
| data_dir=rag_dir, | |
| embedding_model="all-MiniLM-L6-v2", | |
| collection_name="codebase_test", | |
| ) | |
| integration = RAGIntegration(rag) | |
| src_dir = sample_project / "src" | |
| stats = integration.index_directory( | |
| src_dir, | |
| extensions=[".py", ".ts", ".tsx", ".js", ".go", ".rs"], | |
| ) | |
| assert stats["files_processed"] > 0 | |
| assert stats["memories_added"] > 0 | |
| # Search for Python class | |
| results = integration.search_with_context("Application class", limit=5) | |
| assert len(results) >= 0 | |
| # Search for TypeScript component | |
| results = integration.search_with_context("Button component props", limit=5) | |
| assert len(results) >= 0 | |
| rag.close() | |
| def test_extract_relationships(self, sample_project: Path): | |
| """Test extracting import relationships.""" | |
| from contextfs.filetypes.registry import get_handler | |
| # Test Python imports | |
| py_handler = get_handler(file_path="test.py") | |
| main_py = sample_project / "src" / "main.py" | |
| content = main_py.read_text() | |
| doc = py_handler.parse(content, str(main_py)) | |
| py_relationships = py_handler.extract_relationships(doc) | |
| # Should find imports | |
| assert len(py_relationships) > 0 | |
| # Test TypeScript imports | |
| ts_handler = get_handler(file_path="test.tsx") | |
| card_tsx = sample_project / "src" / "components" / "Card.tsx" | |
| content = card_tsx.read_text() | |
| doc = ts_handler.parse(content, str(card_tsx)) | |
| ts_relationships = ts_handler.extract_relationships(doc) | |
| # Should find React and Button imports | |
| assert len(ts_relationships) > 0 | |
| def test_cross_reference_linking(self, sample_project: Path): | |
| """Test cross-reference linking in codebase.""" | |
| from contextfs.filetypes.integration import SmartDocumentProcessor | |
| processor = SmartDocumentProcessor() | |
| src_dir = sample_project / "src" | |
| # Process all files | |
| for file in src_dir.rglob("*"): | |
| if file.is_file() and file.suffix in [".py", ".tsx", ".ts", ".js", ".go", ".rs"]: | |
| try: | |
| processor.process_file(file) | |
| except Exception: | |
| pass # Skip files that can't be processed | |
| # Link cross-references | |
| xrefs = processor.linker.link_all() | |
| # Should have some cross-references (imports between files) | |
| # May be 0 if files don't reference each other by resolvable paths | |
| assert isinstance(xrefs, list) | |
| # Check unresolved references exist (external deps) | |
| unresolved = processor.linker.get_unresolved() | |
| assert isinstance(unresolved, list) | |
| def test_multi_language_chunking(self, sample_project: Path): | |
| """Test that chunking works consistently across languages.""" | |
| from contextfs.filetypes.integration import SmartDocumentProcessor | |
| processor = SmartDocumentProcessor() | |
| src_dir = sample_project / "src" | |
| chunks_by_type = {} | |
| for file in src_dir.rglob("*"): | |
| if file.is_file() and file.suffix in [".py", ".tsx", ".go", ".rs"]: | |
| try: | |
| chunks = processor.process_file(file) | |
| file_type = chunks[0]["metadata"]["file_type"] if chunks else None | |
| if file_type: | |
| chunks_by_type.setdefault(file_type, []).extend(chunks) | |
| except Exception: | |
| pass | |
| # Should have chunks from multiple languages | |
| assert len(chunks_by_type) >= 2 | |
| # Each language should have valid chunks | |
| for file_type, chunks in chunks_by_type.items(): | |
| assert len(chunks) > 0 | |
| for chunk in chunks: | |
| assert "embedding_text" in chunk | |
| assert chunk["metadata"]["file_type"] == file_type | |
Xet Storage Details
- Size:
- 11.9 kB
- Xet hash:
- 663a147a355e47c2edbd4cce38bde9db808cdf82bed3f36b039e865da816804b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.