Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /grammar_manager.py
| """Grammar manager implementation for tree-sitter language support.""" | |
| import ctypes | |
| import json | |
| import logging | |
| import subprocess | |
| import threading | |
| from pathlib import Path | |
| from .contracts.grammar_manager_contract import GrammarManagerContract | |
| from .exceptions import ChunkerError | |
| from .grammar.source_validation import validate_grammar_source | |
| logger = logging.getLogger(__name__) | |
| class GrammarManagerError(ChunkerError): | |
| """Base exception for grammar manager operations.""" | |
| class GrammarManager(GrammarManagerContract): | |
| """Manages tree-sitter grammar downloads and compilation.""" | |
| def __init__( | |
| self, | |
| root_dir: Path | None = None, | |
| config_file: Path | None = None, | |
| max_workers: int = 4, | |
| ): | |
| """Initialize the grammar manager. | |
| Args: | |
| root_dir: Root directory for grammars (defaults to project root) | |
| config_file: Path to grammar sources config (defaults to config/grammar_sources.json) | |
| max_workers: Maximum parallel workers for fetch/compile operations | |
| """ | |
| self._root_dir = root_dir or Path(__file__).parent.parent | |
| self._config_file = ( | |
| config_file or self._root_dir / "config" / "grammar_sources.json" | |
| ) | |
| self._grammars_dir = self._root_dir / "grammars" | |
| self._build_dir = self._root_dir / "build" | |
| self._lib_path = self._build_dir / "my-languages.so" | |
| self._max_workers = max_workers | |
| # Use reentrant lock to allow nested calls that also acquire the lock (e.g., _save_config) | |
| self._lock = threading.RLock() | |
| self._grammar_sources = self._load_config() | |
| self._grammars_dir.mkdir(parents=True, exist_ok=True) | |
| self._build_dir.mkdir(parents=True, exist_ok=True) | |
| def _load_config(self) -> dict[str, str]: | |
| """Load grammar sources from config file.""" | |
| if not self._config_file.exists(): | |
| logger.warning("Config file not found: %s", self._config_file) | |
| return {} | |
| try: | |
| with self._config_file.open("r", encoding="utf-8") as f: | |
| return json.load(f) | |
| except (OSError, FileNotFoundError, IndexError) as e: | |
| logger.error("Failed to load config: %s", e) | |
| return {} | |
| def _save_config(self) -> None: | |
| """Save grammar sources to config file.""" | |
| with self._lock: | |
| self._config_file.parent.mkdir(parents=True, exist_ok=True) | |
| with self._config_file.open("w", encoding="utf-8") as f: | |
| json.dump(self._grammar_sources, f, indent=2, sort_keys=True) | |
| def add_grammar_source(self, language: str, repo_url: str) -> bool: | |
| """Add a new grammar source to be fetched. | |
| Args: | |
| language: Language identifier | |
| repo_url: GitHub repository URL | |
| Returns: | |
| True if successfully added | |
| Raises: | |
| GrammarManagerError: If URL is invalid or language already exists | |
| """ | |
| try: | |
| repo_url = validate_grammar_source(repo_url) | |
| except ValueError as e: | |
| raise GrammarManagerError(f"Invalid GitHub URL: {repo_url}") from e | |
| if language in self._grammar_sources: | |
| logger.warning( | |
| "Language '%s' already exists with URL: %s", | |
| language, | |
| self._grammar_sources[language], | |
| ) | |
| return False | |
| with self._lock: | |
| self._grammar_sources[language] = repo_url | |
| self._save_config() | |
| logger.info("Added grammar source for '%s': %s", language, repo_url) | |
| return True | |
| def fetch_grammars(self, languages: list[str] | None = None) -> dict[str, bool]: | |
| """Fetch grammar repositories. | |
| Args: | |
| languages: Optional list of specific languages to fetch | |
| Returns: | |
| Dict mapping language to fetch success status | |
| """ | |
| if languages is None: | |
| languages_to_fetch = list(self._grammar_sources.keys()) | |
| else: | |
| missing = set(languages) - set(self._grammar_sources.keys()) | |
| if missing: | |
| logger.warning("Unknown languages requested: %s", missing) | |
| languages_to_fetch = [ | |
| lang for lang in languages if lang in self._grammar_sources | |
| ] | |
| if not languages_to_fetch: | |
| logger.warning("No languages to fetch") | |
| return {} | |
| results = {} | |
| def fetch_single(lang: str) -> tuple[str, bool]: | |
| """Fetch a single grammar repository.""" | |
| repo_url = validate_grammar_source(self._grammar_sources[lang]) | |
| target_dir = self._grammars_dir / f"tree-sitter-{lang}" | |
| if target_dir.exists(): | |
| logger.info( | |
| "[skip] %s already present at %s", | |
| lang, | |
| target_dir, | |
| ) | |
| return lang, True | |
| try: | |
| logger.info("[clone] %s from %s", lang, repo_url) | |
| cmd = ["git", "clone", "--depth=1", "--", repo_url, str(target_dir)] | |
| result = subprocess.run(cmd, capture_output=True, text=True, check=True) | |
| logger.debug("Clone output for %s: %s", lang, result.stdout) | |
| return lang, True | |
| except subprocess.CalledProcessError as e: | |
| logger.error("Failed to clone %s: %s", lang, e.stderr) | |
| return lang, False | |
| except (OSError, IndexError, KeyError, ValueError) as e: | |
| logger.error("Unexpected error cloning %s: %s", lang, e) | |
| return lang, False | |
| except Exception as e: | |
| logger.error("Unexpected exception cloning %s: %s", lang, e) | |
| return lang, False | |
| # Process sequentially to ensure deterministic behavior and predictable mocking in tests | |
| for lang in languages_to_fetch: | |
| language, success = fetch_single(lang) | |
| results[language] = success | |
| successful = sum(1 for s in results.values() if s) | |
| logger.info("Fetched %s/%s grammars successfully", successful, len(results)) | |
| return results | |
| def compile_grammars(self, languages: list[str] | None = None) -> dict[str, bool]: | |
| """Compile fetched grammars into shared library. | |
| Args: | |
| languages: Optional list of specific languages to compile | |
| Returns: | |
| Dict mapping language to compilation success status | |
| """ | |
| available_grammars = {} | |
| for gram_dir in self._grammars_dir.glob("tree-sitter-*"): | |
| if gram_dir.is_dir(): | |
| lang_name = gram_dir.name.replace("tree-sitter-", "") | |
| available_grammars[lang_name] = gram_dir | |
| if languages is None: | |
| languages_to_compile = list(available_grammars.keys()) | |
| else: | |
| missing = set(languages) - set(available_grammars.keys()) | |
| if missing: | |
| logger.warning("Languages not fetched: %s", missing) | |
| languages_to_compile = [ | |
| lang for lang in languages if lang in available_grammars | |
| ] | |
| if not languages_to_compile: | |
| logger.warning("No languages to compile") | |
| return {} | |
| c_files = [] | |
| include_dirs = set() | |
| results = {} | |
| for lang in languages_to_compile: | |
| gram_dir = available_grammars[lang] | |
| src_dir = gram_dir / "src" | |
| if not src_dir.exists(): | |
| logger.warning("No src directory for %s at %s", lang, src_dir) | |
| results[lang] = False | |
| continue | |
| include_dirs.add(str(src_dir)) | |
| lang_c_files = list(src_dir.glob("*.c")) | |
| if not lang_c_files: | |
| logger.warning("No C files found for %s in %s", lang, src_dir) | |
| results[lang] = False | |
| else: | |
| c_files.extend(str(f) for f in lang_c_files) | |
| results[lang] = True | |
| if not c_files: | |
| logger.error("No C source files found to compile") | |
| return results | |
| cmd = ["gcc", "-shared", "-fPIC"] | |
| for inc in sorted(include_dirs): | |
| cmd.extend(["-I", inc]) | |
| cmd += ["-o", str(self._lib_path), *c_files] | |
| try: | |
| logger.info( | |
| "Compiling %d C files from %d languages", | |
| len(c_files), | |
| len(languages_to_compile), | |
| ) | |
| logger.debug( | |
| "Compilation command: %s... (%d args total)", | |
| " ".join(cmd[:10]), | |
| len(cmd), | |
| ) | |
| completed = subprocess.run( | |
| cmd, | |
| capture_output=True, | |
| text=True, | |
| check=False, | |
| ) | |
| if completed.returncode != 0: | |
| logger.error("Compilation failed: %s", completed.stderr) | |
| for lang in languages_to_compile: | |
| results[lang] = False | |
| return results | |
| logger.info("✅ Successfully compiled to %s", self._lib_path) | |
| return results | |
| except ( | |
| subprocess.CalledProcessError, | |
| OSError, | |
| FileNotFoundError, | |
| IndexError, | |
| Exception, | |
| ) as e: | |
| logger.error("Compilation error: %s", e) | |
| for lang in languages_to_compile: | |
| results[lang] = False | |
| return results | |
| def get_available_languages(self) -> set[str]: | |
| """Get set of languages with compiled grammars. | |
| Returns: | |
| Set of available language identifiers | |
| """ | |
| if not self._lib_path.exists(): | |
| return set() | |
| try: | |
| lib = ctypes.CDLL(str(self._lib_path)) | |
| available = set() | |
| for lang in self._grammar_sources: | |
| symbol_name = f"tree_sitter_{lang}" | |
| try: | |
| getattr(lib, symbol_name) | |
| available.add(lang) | |
| except AttributeError: | |
| pass | |
| for gram_dir in self._grammars_dir.glob("tree-sitter-*"): | |
| if gram_dir.is_dir(): | |
| lang = gram_dir.name.replace("tree-sitter-", "") | |
| symbol_name = f"tree_sitter_{lang}" | |
| try: | |
| getattr(lib, symbol_name) | |
| available.add(lang) | |
| except AttributeError: | |
| pass | |
| return available | |
| except Exception as e: | |
| # Fallback to directory-based discovery if library fails to load or symbols are missing | |
| logger.error("Error discovering available languages: %s", e) | |
| available = set() | |
| for gram_dir in self._grammars_dir.glob("tree-sitter-*"): | |
| if gram_dir.is_dir(): | |
| lang = gram_dir.name.replace("tree-sitter-", "") | |
| available.add(lang) | |
| return available | |
Xet Storage Details
- Size:
- 11.1 kB
- Xet hash:
- a3557c80ac26a8697a5ffbf2b5e568811b3f46d9ff28af01c250af275884eb63
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.