Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /parser.py
| """Parser module for tree-sitter chunker with dynamic language discovery.""" | |
| from __future__ import annotations | |
| import threading | |
| import json | |
| import logging | |
| import sys | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING | |
| from ._internal.factory import ParserConfig, ParserFactory, ParserLease | |
| from ._internal.registry import LanguageMetadata, LanguageRegistry | |
| from .exceptions import ( | |
| LanguageNotFoundError, | |
| ParserConfigError, | |
| ParserError, | |
| ) | |
| from .grammar.manager import TreeSitterGrammarManager | |
| if TYPE_CHECKING: | |
| from tree_sitter import Parser | |
| logger = logging.getLogger(__name__) | |
| __all__ = [ | |
| "ParserConfig", | |
| "ParserLease", | |
| "acquire_parser", | |
| "clear_cache", | |
| "get_language_info", | |
| "get_parser", | |
| "list_languages", | |
| "return_parser", | |
| ] | |
| class _ParserState: | |
| """Singleton state holder for parser module.""" | |
| def __init__(self) -> None: | |
| self.registry: LanguageRegistry | None = None | |
| self.factory: ParserFactory | None = None | |
| self._init_lock = threading.Lock() | |
| # Prefer in-package built artifacts when present (for wheels with prebuilt grammars) | |
| _lib_ext = {"darwin": ".dylib", "win32": ".dll"}.get(sys.platform, ".so") | |
| package_build = Path(__file__).parent / "data" / "grammars" / "build" | |
| combined_lib = package_build / f"languages{_lib_ext}" | |
| self.default_library_path = ( | |
| combined_lib | |
| if combined_lib.exists() | |
| else Path(__file__).parent.parent / "build" / f"my-languages{_lib_ext}" | |
| ) | |
| def initialize(self, library_path: Path | None = None) -> None: | |
| """Lazy initialization of registry and factory. | |
| Args: | |
| library_path: Optional path to the compiled library | |
| """ | |
| # Guard the whole init under a lock and use `factory` (set LAST) as the | |
| # initialized sentinel, so a concurrent cold-start thread can never observe | |
| # registry-set-but-factory-None and fail with "factory not initialized". | |
| if self.factory is not None: | |
| return | |
| with self._init_lock: | |
| # Double-checked locking: another thread may have initialized the | |
| # factory between the first check and acquiring the lock. mypy narrows | |
| # self.factory to None from the check above (it assumes single-thread) | |
| # and flags this re-check as unreachable — but it is load-bearing. | |
| if self.factory is not None: | |
| return # type: ignore[unreachable] | |
| path = library_path or self.default_library_path | |
| # If the compiled library doesn't exist or fails to load, initialize registry | |
| # with no shared library so that list_languages() can still function for tests | |
| # Always initialize the registry; it now tolerates missing combined library | |
| registry = LanguageRegistry(path) | |
| factory = ParserFactory(registry) | |
| self.registry = registry | |
| self.factory = factory # published last; the sentinel above gates on it | |
| languages = registry.list_languages() | |
| logger.info( | |
| "Initialized parser with %d languages: %s", | |
| len(languages), | |
| ", ".join(languages), | |
| ) | |
| _state = _ParserState() | |
| def _initialize(library_path: Path | None = None) -> None: | |
| """Lazy initialization of registry and factory. | |
| Args: | |
| library_path: Optional path to the compiled library | |
| """ | |
| _state.initialize(library_path) | |
| def get_parser(language: str, config: ParserConfig | None = None) -> Parser: | |
| """Get a parser for the specified language with optional configuration. | |
| Args: | |
| language: Language name (e.g., 'python', 'rust') | |
| config: Optional parser configuration | |
| Returns: | |
| Configured Parser instance | |
| Raises: | |
| LanguageNotFoundError: If language is not available | |
| ParserError: If parser initialization fails | |
| """ | |
| _initialize() | |
| if _state.factory is None: | |
| raise ParserError("Parser factory not initialized") | |
| try: | |
| # Normalize common aliases - use "csharp" as canonical since tree-sitter-language-pack uses it | |
| alias_map = { | |
| "csharp": "csharp", | |
| "c_sharp": "csharp", | |
| "typescript": "typescript", | |
| "tsx": "tsx", | |
| } | |
| normalized = alias_map.get(language, language) | |
| return _state.factory.get_parser(normalized, config) | |
| except LanguageNotFoundError: | |
| # Attempt on-demand grammar fetch/build for missing language | |
| try: | |
| # Load repository URL from config | |
| sources_path = ( | |
| Path(__file__).parent.parent / "config" / "grammar_sources.json" | |
| ) | |
| repo_url: str | None = None | |
| if sources_path.exists(): | |
| try: | |
| with sources_path.open("r", encoding="utf-8") as f: | |
| sources = json.load(f) | |
| except json.JSONDecodeError as e: | |
| logger.warning( | |
| "Invalid JSON in %s: %s at line %s", | |
| sources_path, | |
| e.msg, | |
| e.lineno, | |
| ) | |
| sources = {} | |
| # Handle common alias differences for repo lookup | |
| repo_alias_map = { | |
| "csharp": "csharp", | |
| "c_sharp": "csharp", | |
| "typescript": "typescript", | |
| } | |
| repo_key = repo_alias_map.get(language, language) | |
| repo_url = sources.get(repo_key) | |
| if repo_url: | |
| gm = TreeSitterGrammarManager() | |
| if not gm.get_grammar_info(language): | |
| gm.add_grammar(language, repo_url) | |
| # Fetch and build; ignore failures silently so we raise the original error | |
| if gm.fetch_grammar(language): | |
| gm.build_grammar(language) | |
| # After building, try again (factory will now be able to load individual lib | |
| return _state.factory.get_parser(normalized, config) | |
| except Exception as e: | |
| # Fall back to raising the original error, but log for diagnostics in CI | |
| logger.debug("On-demand grammar build attempt failed: %s", e) | |
| available = _state.registry.list_languages() if _state.registry else [] | |
| raise LanguageNotFoundError(normalized, available) from None | |
| except ParserConfigError: | |
| raise | |
| except (IndexError, KeyError, SyntaxError) as e: | |
| logger.error("Failed to get parser for %s: %s", language, e) | |
| raise ParserError(f"Parser initialization failed: {e}") from e | |
| def acquire_parser(language: str, config: ParserConfig | None = None) -> ParserLease: | |
| """Exclusively lease a parser for the duration of a context manager. | |
| The leased parser is not available to another lease until the context exits. | |
| Prefer this API for work that needs a parser only temporarily. | |
| """ | |
| _initialize() | |
| if _state.factory is None: | |
| raise ParserError("Parser factory not initialized") | |
| alias_map = { | |
| "csharp": "csharp", | |
| "c_sharp": "csharp", | |
| "typescript": "typescript", | |
| "tsx": "tsx", | |
| } | |
| normalized = alias_map.get(language, language) | |
| try: | |
| return _state.factory.acquire_parser(normalized, config) | |
| except LanguageNotFoundError: | |
| try: | |
| sources_path = ( | |
| Path(__file__).parent.parent / "config" / "grammar_sources.json" | |
| ) | |
| repo_url: str | None = None | |
| if sources_path.exists(): | |
| try: | |
| with sources_path.open("r", encoding="utf-8") as f: | |
| sources = json.load(f) | |
| except json.JSONDecodeError as e: | |
| logger.warning( | |
| "Invalid JSON in %s: %s at line %s", | |
| sources_path, | |
| e.msg, | |
| e.lineno, | |
| ) | |
| sources = {} | |
| repo_key = alias_map.get(language, language) | |
| repo_url = sources.get(repo_key) | |
| if repo_url: | |
| gm = TreeSitterGrammarManager() | |
| if not gm.get_grammar_info(language): | |
| gm.add_grammar(language, repo_url) | |
| if gm.fetch_grammar(language): | |
| gm.build_grammar(language) | |
| return _state.factory.acquire_parser(normalized, config) | |
| except Exception as e: | |
| logger.debug("On-demand grammar build attempt failed: %s", e) | |
| available = _state.registry.list_languages() if _state.registry else [] | |
| raise LanguageNotFoundError(normalized, available) from None | |
| def list_languages() -> list[str]: | |
| """List all available languages. | |
| Returns: | |
| Sorted list of language names | |
| """ | |
| _initialize() | |
| if _state.registry is None: | |
| raise ParserError("Language registry not initialized") | |
| return _state.registry.list_languages() | |
| def get_language_info(language: str) -> LanguageMetadata: | |
| """Get metadata about a specific language. | |
| Args: | |
| language: Language name | |
| Returns: | |
| Language metadata | |
| Raises: | |
| LanguageNotFoundError: If language is not available | |
| """ | |
| _initialize() | |
| if _state.registry is None: | |
| raise ParserError("Language registry not initialized") | |
| return _state.registry.get_metadata(language) | |
| def return_parser(language: str, parser: Parser) -> None: | |
| """Keep the legacy return call safe for thread-owned parser instances. | |
| Args: | |
| language: Language name | |
| parser: Parser instance to return | |
| """ | |
| _initialize() | |
| if _state.factory is None: | |
| raise ParserError("Parser factory not initialized") | |
| _state.factory.return_parser(language, parser) | |
| def clear_cache() -> None: | |
| """Clear the parser cache. | |
| This forces recreation of parsers on next request. | |
| """ | |
| _initialize() | |
| if _state.factory is not None: | |
| _state.factory.clear_cache() | |
| __all__ = [ | |
| "LanguageMetadata", | |
| "ParserConfig", | |
| "ParserLease", | |
| "acquire_parser", | |
| "clear_cache", | |
| "get_language_info", | |
| "get_parser", | |
| "list_languages", | |
| "return_parser", | |
| ] | |
Xet Storage Details
- Size:
- 10.4 kB
- Xet hash:
- 4abf2ddf3c550f4d860c0d5dc195876787d1c3759a9b1c198c283d9129c23ae2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.