Buckets:

download
raw
10.4 kB
"""Parser module for tree-sitter chunker with dynamic language discovery."""
from __future__ import annotations
import threading
import json
import logging
import sys
from pathlib import Path
from typing import TYPE_CHECKING
from ._internal.factory import ParserConfig, ParserFactory, ParserLease
from ._internal.registry import LanguageMetadata, LanguageRegistry
from .exceptions import (
LanguageNotFoundError,
ParserConfigError,
ParserError,
)
from .grammar.manager import TreeSitterGrammarManager
if TYPE_CHECKING:
from tree_sitter import Parser
logger = logging.getLogger(__name__)
__all__ = [
"ParserConfig",
"ParserLease",
"acquire_parser",
"clear_cache",
"get_language_info",
"get_parser",
"list_languages",
"return_parser",
]
class _ParserState:
"""Singleton state holder for parser module."""
def __init__(self) -> None:
self.registry: LanguageRegistry | None = None
self.factory: ParserFactory | None = None
self._init_lock = threading.Lock()
# Prefer in-package built artifacts when present (for wheels with prebuilt grammars)
_lib_ext = {"darwin": ".dylib", "win32": ".dll"}.get(sys.platform, ".so")
package_build = Path(__file__).parent / "data" / "grammars" / "build"
combined_lib = package_build / f"languages{_lib_ext}"
self.default_library_path = (
combined_lib
if combined_lib.exists()
else Path(__file__).parent.parent / "build" / f"my-languages{_lib_ext}"
)
def initialize(self, library_path: Path | None = None) -> None:
"""Lazy initialization of registry and factory.
Args:
library_path: Optional path to the compiled library
"""
# Guard the whole init under a lock and use `factory` (set LAST) as the
# initialized sentinel, so a concurrent cold-start thread can never observe
# registry-set-but-factory-None and fail with "factory not initialized".
if self.factory is not None:
return
with self._init_lock:
# Double-checked locking: another thread may have initialized the
# factory between the first check and acquiring the lock. mypy narrows
# self.factory to None from the check above (it assumes single-thread)
# and flags this re-check as unreachable — but it is load-bearing.
if self.factory is not None:
return # type: ignore[unreachable]
path = library_path or self.default_library_path
# If the compiled library doesn't exist or fails to load, initialize registry
# with no shared library so that list_languages() can still function for tests
# Always initialize the registry; it now tolerates missing combined library
registry = LanguageRegistry(path)
factory = ParserFactory(registry)
self.registry = registry
self.factory = factory # published last; the sentinel above gates on it
languages = registry.list_languages()
logger.info(
"Initialized parser with %d languages: %s",
len(languages),
", ".join(languages),
)
_state = _ParserState()
def _initialize(library_path: Path | None = None) -> None:
"""Lazy initialization of registry and factory.
Args:
library_path: Optional path to the compiled library
"""
_state.initialize(library_path)
def get_parser(language: str, config: ParserConfig | None = None) -> Parser:
"""Get a parser for the specified language with optional configuration.
Args:
language: Language name (e.g., 'python', 'rust')
config: Optional parser configuration
Returns:
Configured Parser instance
Raises:
LanguageNotFoundError: If language is not available
ParserError: If parser initialization fails
"""
_initialize()
if _state.factory is None:
raise ParserError("Parser factory not initialized")
try:
# Normalize common aliases - use "csharp" as canonical since tree-sitter-language-pack uses it
alias_map = {
"csharp": "csharp",
"c_sharp": "csharp",
"typescript": "typescript",
"tsx": "tsx",
}
normalized = alias_map.get(language, language)
return _state.factory.get_parser(normalized, config)
except LanguageNotFoundError:
# Attempt on-demand grammar fetch/build for missing language
try:
# Load repository URL from config
sources_path = (
Path(__file__).parent.parent / "config" / "grammar_sources.json"
)
repo_url: str | None = None
if sources_path.exists():
try:
with sources_path.open("r", encoding="utf-8") as f:
sources = json.load(f)
except json.JSONDecodeError as e:
logger.warning(
"Invalid JSON in %s: %s at line %s",
sources_path,
e.msg,
e.lineno,
)
sources = {}
# Handle common alias differences for repo lookup
repo_alias_map = {
"csharp": "csharp",
"c_sharp": "csharp",
"typescript": "typescript",
}
repo_key = repo_alias_map.get(language, language)
repo_url = sources.get(repo_key)
if repo_url:
gm = TreeSitterGrammarManager()
if not gm.get_grammar_info(language):
gm.add_grammar(language, repo_url)
# Fetch and build; ignore failures silently so we raise the original error
if gm.fetch_grammar(language):
gm.build_grammar(language)
# After building, try again (factory will now be able to load individual lib
return _state.factory.get_parser(normalized, config)
except Exception as e:
# Fall back to raising the original error, but log for diagnostics in CI
logger.debug("On-demand grammar build attempt failed: %s", e)
available = _state.registry.list_languages() if _state.registry else []
raise LanguageNotFoundError(normalized, available) from None
except ParserConfigError:
raise
except (IndexError, KeyError, SyntaxError) as e:
logger.error("Failed to get parser for %s: %s", language, e)
raise ParserError(f"Parser initialization failed: {e}") from e
def acquire_parser(language: str, config: ParserConfig | None = None) -> ParserLease:
"""Exclusively lease a parser for the duration of a context manager.
The leased parser is not available to another lease until the context exits.
Prefer this API for work that needs a parser only temporarily.
"""
_initialize()
if _state.factory is None:
raise ParserError("Parser factory not initialized")
alias_map = {
"csharp": "csharp",
"c_sharp": "csharp",
"typescript": "typescript",
"tsx": "tsx",
}
normalized = alias_map.get(language, language)
try:
return _state.factory.acquire_parser(normalized, config)
except LanguageNotFoundError:
try:
sources_path = (
Path(__file__).parent.parent / "config" / "grammar_sources.json"
)
repo_url: str | None = None
if sources_path.exists():
try:
with sources_path.open("r", encoding="utf-8") as f:
sources = json.load(f)
except json.JSONDecodeError as e:
logger.warning(
"Invalid JSON in %s: %s at line %s",
sources_path,
e.msg,
e.lineno,
)
sources = {}
repo_key = alias_map.get(language, language)
repo_url = sources.get(repo_key)
if repo_url:
gm = TreeSitterGrammarManager()
if not gm.get_grammar_info(language):
gm.add_grammar(language, repo_url)
if gm.fetch_grammar(language):
gm.build_grammar(language)
return _state.factory.acquire_parser(normalized, config)
except Exception as e:
logger.debug("On-demand grammar build attempt failed: %s", e)
available = _state.registry.list_languages() if _state.registry else []
raise LanguageNotFoundError(normalized, available) from None
def list_languages() -> list[str]:
"""List all available languages.
Returns:
Sorted list of language names
"""
_initialize()
if _state.registry is None:
raise ParserError("Language registry not initialized")
return _state.registry.list_languages()
def get_language_info(language: str) -> LanguageMetadata:
"""Get metadata about a specific language.
Args:
language: Language name
Returns:
Language metadata
Raises:
LanguageNotFoundError: If language is not available
"""
_initialize()
if _state.registry is None:
raise ParserError("Language registry not initialized")
return _state.registry.get_metadata(language)
def return_parser(language: str, parser: Parser) -> None:
"""Keep the legacy return call safe for thread-owned parser instances.
Args:
language: Language name
parser: Parser instance to return
"""
_initialize()
if _state.factory is None:
raise ParserError("Parser factory not initialized")
_state.factory.return_parser(language, parser)
def clear_cache() -> None:
"""Clear the parser cache.
This forces recreation of parsers on next request.
"""
_initialize()
if _state.factory is not None:
_state.factory.clear_cache()
__all__ = [
"LanguageMetadata",
"ParserConfig",
"ParserLease",
"acquire_parser",
"clear_cache",
"get_language_info",
"get_parser",
"list_languages",
"return_parser",
]

Xet Storage Details

Size:
10.4 kB
·
Xet hash:
4abf2ddf3c550f4d860c0d5dc195876787d1c3759a9b1c198c283d9129c23ae2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.