Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /auto.py
| """Zero-configuration API for automatic language setup and chunking. | |
| This module provides a simple, zero-config interface for chunking code files | |
| with automatic language detection and grammar management. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING, Any, ClassVar | |
| from .chunker import chunk_file_with_token_limit, chunk_text_with_token_limit | |
| from .contracts.auto_contract import AutoChunkResult, ZeroConfigContract | |
| from .core import chunk_file, chunk_text | |
| from .exceptions import ChunkerError, LanguageNotFoundError, ParserError | |
| from .fallback.sliding_window_fallback import SlidingWindowFallback | |
| if TYPE_CHECKING: | |
| from .contracts.registry_contract import UniversalRegistryContract | |
| class ZeroConfigAPI(ZeroConfigContract): | |
| """Zero-configuration API for automatic chunking. | |
| This class provides a simple interface that automatically: | |
| - Detects languages from file extensions and content | |
| - Downloads and sets up grammars as needed | |
| - Falls back to text chunking when tree-sitter is unavailable | |
| """ | |
| EXTENSION_MAP: ClassVar[dict[str, str]] = { | |
| ".py": "python", | |
| ".js": "javascript", | |
| ".jsx": "javascript", | |
| ".ts": "typescript", | |
| ".tsx": "typescript", | |
| ".java": "java", | |
| ".c": "c", | |
| ".h": "c", | |
| ".cc": "cpp", | |
| ".cpp": "cpp", | |
| ".cxx": "cpp", | |
| ".hpp": "cpp", | |
| ".rs": "rust", | |
| ".go": "go", | |
| ".rb": "ruby", | |
| ".php": "php", | |
| ".cs": "csharp", | |
| ".swift": "swift", | |
| ".kt": "kotlin", | |
| ".scala": "scala", | |
| ".r": "r", | |
| ".lua": "lua", | |
| ".jl": "julia", | |
| ".sh": "bash", | |
| ".bash": "bash", | |
| ".zsh": "bash", | |
| ".ps1": "powershell", | |
| ".yaml": "yaml", | |
| ".yml": "yaml", | |
| ".json": "json", | |
| ".xml": "xml", | |
| ".html": "html", | |
| ".css": "css", | |
| ".scss": "css", | |
| ".sql": "sql", | |
| ".md": "markdown", | |
| ".rst": "restructuredtext", | |
| ".tex": "latex", | |
| ".vim": "vim", | |
| ".el": "elisp", | |
| ".clj": "clojure", | |
| ".ex": "elixir", | |
| ".exs": "elixir", | |
| ".erl": "erlang", | |
| ".hrl": "erlang", | |
| ".fs": "fsharp", | |
| ".fsx": "fsharp", | |
| ".ml": "ocaml", | |
| ".mli": "ocaml", | |
| ".pl": "perl", | |
| ".pm": "perl", | |
| ".raku": "raku", | |
| ".dart": "dart", | |
| ".zig": "zig", | |
| ".nim": "nim", | |
| ".v": "verilog", | |
| ".vhdl": "vhdl", | |
| ".m": "matlab", | |
| ".f90": "fortran", | |
| ".f95": "fortran", | |
| ".cob": "cobol", | |
| ".pas": "pascal", | |
| ".asm": "assembly", | |
| ".s": "assembly", | |
| } | |
| SHEBANG_PATTERNS: ClassVar[dict[str, str]] = { | |
| "python[0-9.]*": "python", | |
| "ruby": "ruby", | |
| "node|nodejs": "javascript", | |
| "perl": "perl", | |
| "bash|sh": "bash", | |
| "zsh": "bash", | |
| "fish": "bash", | |
| "php": "php", | |
| "lua": "lua", | |
| "julia": "julia", | |
| "Rscript": "r", | |
| } | |
| def __init__(self, registry: UniversalRegistryContract): | |
| """Initialize the zero-config API. | |
| Args: | |
| registry: Universal language registry instance | |
| """ | |
| self.registry = registry | |
| self._fallback_chunker = SlidingWindowFallback() | |
| def ensure_language( | |
| self, | |
| language: str, | |
| version: str | None = None, | |
| ) -> bool: | |
| """Ensure a language is available for use. | |
| Args: | |
| language: Language name | |
| version: Specific version required | |
| Returns: | |
| True if language is ready to use | |
| """ | |
| if self.registry.is_language_installed(language): | |
| if version: | |
| installed_version = self.registry.get_language_version( | |
| language, | |
| ) | |
| if installed_version == version: | |
| return True | |
| success, _ = self.registry.update_language(language) | |
| return success | |
| return True | |
| available = self.registry.list_available_languages() | |
| if language not in available: | |
| return False | |
| return self.registry.install_language(language, version) | |
| def auto_chunk_file( | |
| self, | |
| file_path: str | Path, | |
| language: str | None = None, | |
| token_limit: int | None = None, | |
| ) -> AutoChunkResult: | |
| """Automatically chunk a file with zero configuration. | |
| Args: | |
| file_path: Path to file | |
| language: Override language detection | |
| token_limit: Optional token limit per chunk | |
| Returns: | |
| Chunking result with metadata | |
| """ | |
| file_path = Path(file_path) | |
| if not file_path.exists(): | |
| raise ValueError(f"File not found: {file_path}") | |
| if not language: | |
| detected = self.detect_language(file_path) | |
| if not detected: | |
| content = file_path.read_text(encoding="utf-8", errors="replace") | |
| code_chunks = self._fallback_chunker.chunk_text(content, str(file_path)) | |
| chunks = [] | |
| for chunk in code_chunks: | |
| chunk_dict = { | |
| "content": chunk.content, | |
| "type": chunk.node_type, | |
| "start_line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| } | |
| if hasattr(chunk, "metadata") and chunk.metadata: | |
| chunk_dict["metadata"] = chunk.metadata | |
| chunks.append(chunk_dict) | |
| return AutoChunkResult( | |
| chunks=chunks, | |
| language="unknown", | |
| grammar_downloaded=False, | |
| fallback_used=True, | |
| metadata={"file_path": str(file_path)}, | |
| ) | |
| language = detected | |
| grammar_downloaded = False | |
| if not self.registry.is_language_installed(language): | |
| grammar_downloaded = self.ensure_language(language) | |
| try: | |
| if self.registry.is_language_installed(language): | |
| if token_limit: | |
| chunks = chunk_file_with_token_limit( | |
| file_path, | |
| language, | |
| max_tokens=token_limit, | |
| ) | |
| else: | |
| chunks = chunk_file(file_path, language) | |
| chunk_dicts = [] | |
| for chunk in chunks: | |
| chunk_dict = { | |
| "content": chunk.content, | |
| "type": chunk.node_type, | |
| "start_line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| } | |
| if hasattr(chunk, "metadata") and chunk.metadata: | |
| chunk_dict["metadata"] = chunk.metadata | |
| chunk_dicts.append(chunk_dict) | |
| return AutoChunkResult( | |
| chunks=chunk_dicts, | |
| language=language, | |
| grammar_downloaded=grammar_downloaded, | |
| fallback_used=False, | |
| metadata={ | |
| "file_path": str(file_path), | |
| "tree_sitter_version": "0.20.0", | |
| }, | |
| ) | |
| except (LanguageNotFoundError, ParserError, OSError, IndexError): | |
| pass | |
| content = file_path.read_text(encoding="utf-8", errors="replace") | |
| code_chunks = self._fallback_chunker.chunk_text( | |
| content, | |
| str(file_path), | |
| ) | |
| chunks = [] | |
| for chunk in code_chunks: | |
| chunk_dict = { | |
| "content": chunk.content, | |
| "type": chunk.node_type, | |
| "start_line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| } | |
| if hasattr(chunk, "metadata") and chunk.metadata: | |
| chunk_dict["metadata"] = chunk.metadata | |
| chunks.append(chunk_dict) | |
| return AutoChunkResult( | |
| chunks=chunks, | |
| language=language, | |
| grammar_downloaded=grammar_downloaded, | |
| fallback_used=True, | |
| metadata={ | |
| "file_path": str(file_path), | |
| "fallback_reason": "tree_sitter_failed", | |
| }, | |
| ) | |
| def detect_language(self, file_path: str | Path) -> str | None: | |
| """Detect the language of a file. | |
| Args: | |
| file_path: Path to file | |
| Returns: | |
| Detected language name or None | |
| """ | |
| file_path = Path(file_path) | |
| # Check extension first | |
| lang = self._detect_by_extension(file_path) | |
| if lang: | |
| return lang | |
| # Check shebang | |
| lang = self._detect_by_shebang(file_path) | |
| if lang: | |
| return lang | |
| # Check special filenames | |
| return ZeroConfigAPI._detect_by_filename(file_path) | |
| def _detect_by_extension(self, file_path: Path) -> str | None: | |
| """Detect language by file extension.""" | |
| suffix = file_path.suffix.lower() | |
| return self.EXTENSION_MAP.get(suffix) | |
| def _detect_by_shebang(self, file_path: Path) -> str | None: | |
| """Detect language by shebang line.""" | |
| try: | |
| with file_path.open("rb") as f: | |
| first_line = f.readline() | |
| if first_line.startswith(b"#!"): | |
| shebang = first_line.decode( | |
| "utf-8", | |
| errors="ignore", | |
| ).strip() | |
| for pattern, lang in self.SHEBANG_PATTERNS.items(): | |
| if re.search(pattern, shebang): | |
| return lang | |
| except (FileNotFoundError, OSError): | |
| pass | |
| return None | |
| def _detect_by_filename(file_path: Path) -> str | None: | |
| """Detect language by special filenames.""" | |
| filename_map = { | |
| "Makefile": "makefile", | |
| "Dockerfile": "dockerfile", | |
| "CMakeLists.txt": "cmake", | |
| "Cargo.toml": "toml", | |
| } | |
| lang = filename_map.get(file_path.name) | |
| if lang: | |
| return lang | |
| if file_path.name.endswith(".gemspec"): | |
| return "ruby" | |
| if file_path.name == "package.json": | |
| return "json" | |
| return None | |
| def chunk_text( | |
| self, | |
| text: str, | |
| language: str, | |
| token_limit: int | None = None, | |
| include_retrieval_metadata: bool = False, | |
| ) -> AutoChunkResult: | |
| """Chunk text content with automatic setup. | |
| Args: | |
| text: Text content to chunk | |
| language: Language of the text | |
| token_limit: Optional token limit | |
| Returns: | |
| Chunking result | |
| """ | |
| if not text: | |
| raise ValueError("Text cannot be empty") | |
| if not language: | |
| raise ValueError("Language must be specified for text chunking") | |
| grammar_downloaded = False | |
| if not self.registry.is_language_installed(language): | |
| grammar_downloaded = self.ensure_language(language) | |
| try: | |
| if self.registry.is_language_installed(language): | |
| if token_limit: | |
| chunks = chunk_text_with_token_limit( | |
| text, | |
| language, | |
| max_tokens=token_limit, | |
| include_retrieval_metadata=include_retrieval_metadata, | |
| ) | |
| else: | |
| chunks = chunk_text( | |
| text, | |
| language, | |
| include_retrieval_metadata=include_retrieval_metadata, | |
| ) | |
| chunk_dicts = [] | |
| for chunk in chunks: | |
| chunk_dict = { | |
| "content": chunk.content, | |
| "type": chunk.node_type, | |
| "start_line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| } | |
| if hasattr(chunk, "metadata") and chunk.metadata: | |
| chunk_dict["metadata"] = chunk.metadata | |
| chunk_dicts.append(chunk_dict) | |
| return AutoChunkResult( | |
| chunks=chunk_dicts, | |
| language=language, | |
| grammar_downloaded=grammar_downloaded, | |
| fallback_used=False, | |
| metadata={"tree_sitter_version": "0.20.0"}, | |
| ) | |
| except (LanguageNotFoundError, ParserError, IndexError, KeyError): | |
| pass | |
| code_chunks = self._fallback_chunker.chunk_text(text, "<text>") | |
| chunks = [] | |
| for chunk in code_chunks: | |
| chunk_dict = { | |
| "content": chunk.content, | |
| "type": chunk.node_type, | |
| "start_line": chunk.start_line, | |
| "end_line": chunk.end_line, | |
| } | |
| if hasattr(chunk, "metadata") and chunk.metadata: | |
| chunk_dict["metadata"] = chunk.metadata | |
| chunks.append(chunk_dict) | |
| return AutoChunkResult( | |
| chunks=chunks, | |
| language=language, | |
| grammar_downloaded=grammar_downloaded, | |
| fallback_used=True, | |
| metadata={"fallback_reason": "tree_sitter_failed"}, | |
| ) | |
| def list_supported_extensions(self) -> dict[str, list[str]]: | |
| """List all supported file extensions. | |
| Returns: | |
| Dict mapping language to extensions | |
| """ | |
| language_extensions: dict[str, list[str]] = {} | |
| for ext, lang in self.EXTENSION_MAP.items(): | |
| if lang not in language_extensions: | |
| language_extensions[lang] = [] | |
| language_extensions[lang].append(ext) | |
| for exts in language_extensions.values(): | |
| exts.sort() | |
| return language_extensions | |
| def get_chunker_for_language( | |
| self, | |
| language: str, | |
| auto_download: bool = True, | |
| ) -> Any: | |
| """Get a chunker instance for a specific language. | |
| Args: | |
| language: Language name | |
| auto_download: Download grammar if needed | |
| Returns: | |
| Configured chunker instance | |
| """ | |
| if ( | |
| auto_download | |
| and not self.registry.is_language_installed( | |
| language, | |
| ) | |
| and not self.ensure_language(language) | |
| ): | |
| raise ChunkerError(f"Failed to setup language: {language}") | |
| parser = self.registry.get_parser(language, auto_download=False) | |
| if not parser: | |
| raise ChunkerError(f"No parser available for language: {language}") | |
| class LanguageChunker: | |
| def __init__(self, lang: str): | |
| self.language = lang | |
| def chunk_file( | |
| self, | |
| file_path: str | Path, | |
| include_retrieval_metadata: bool = False, | |
| ) -> list[Any]: | |
| return chunk_file( | |
| file_path, | |
| self.language, | |
| include_retrieval_metadata=include_retrieval_metadata, | |
| ) | |
| def chunk_text( | |
| self, | |
| text: str, | |
| include_retrieval_metadata: bool = False, | |
| ) -> list[Any]: | |
| return chunk_text( | |
| text, | |
| self.language, | |
| include_retrieval_metadata=include_retrieval_metadata, | |
| ) | |
| return LanguageChunker(language) | |
| def preload_languages(self, languages: list[str]) -> dict[str, bool]: | |
| """Preload multiple language grammars. | |
| Args: | |
| languages: List of languages to preload | |
| Returns: | |
| Dict of language -> success status | |
| """ | |
| results = {} | |
| for language in languages: | |
| results[language] = self.ensure_language(language) | |
| return results | |
Xet Storage Details
- Size:
- 16.2 kB
- Xet hash:
- bbce7b31ec2eba3d7d54b15f595c7e32f3ea6d659137684868a169d26f90e8fa
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.