Buckets:

download
raw
4.91 kB
"""
Tree-sitter Chunker - Semantic code chunking for LLMs and embeddings.
Simple usage:
from chunker import chunk_file, chunk_text, chunk_directory
# Chunk a single file
chunks = chunk_file("example.py", language="python")
# Chunk raw text
chunks = chunk_text(code_text, language="javascript")
# Chunk entire directory
results = chunk_directory("src/", language="python")
"""
from ._internal.cache import ASTCache
from .chunker import chunk_text_with_token_limit, count_chunk_tokens
from .chunker_config import ChunkerConfig
from .core import chunk_file
from .boundary import extract_boundary_ir
from .exceptions import LanguageNotFoundError, LibraryNotFoundError, ParserError
from .fallback.intelligent_fallback import (
ChunkingDecision,
DecisionMetrics,
IntelligentFallbackChunker,
)
from .incremental import (
DefaultChangeDetector,
DefaultChunkCache,
DefaultIncrementalProcessor,
)
from .languages.base import PluginConfig
from .multi_language import (
LanguageDetectorImpl,
MultiLanguageProcessorImpl,
ProjectAnalyzerImpl,
)
from .optimization import (
ChunkBoundaryAnalyzer,
ChunkOptimizer,
OptimizationConfig,
OptimizationMetrics,
OptimizationStrategy,
)
from .parallel import chunk_directory_parallel as chunk_directory
from .parser import (
ParserConfig,
clear_cache,
get_language_info,
get_parser,
list_languages,
return_parser,
)
from .plugin_manager import PluginManager, get_plugin_manager
from .semantic_query import SemanticQuery
# Optional advanced query features (may require heavy deps like numpy)
try:
from .query_advanced import (
AdvancedQueryIndex,
NaturalLanguageQueryEngine,
SmartQueryOptimizer,
)
_ADV_QUERY_AVAILABLE = True
except Exception: # pragma: no cover - keep import lightweight during build tests
AdvancedQueryIndex = None # type: ignore[assignment]
NaturalLanguageQueryEngine = None # type: ignore[assignment]
SmartQueryOptimizer = None # type: ignore[assignment]
_ADV_QUERY_AVAILABLE = False
from .smart_context import InMemoryContextCache, TreeSitterSmartContextProvider
from .streaming import chunk_file_streaming
from .types import CodeChunk
# Dynamic version from package metadata
try:
from importlib.metadata import version
__version__ = version("treesitter-chunker")
except ImportError:
# Fallback when package metadata is unavailable (editable import without an
# installed dist). Must match pyproject.toml `version` — the single source of
# truth — so an uninstalled import does not report a stale version (IFACE).
__version__ = "4.0.0"
# Simple text chunking
def chunk_text(text: str, language: str, file_path: str = "", **kwargs):
"""Chunk text content directly, in memory (no temp-file round-trip).
Delegates to ``core.chunk_text``, which parses ``text`` directly. The former
implementation wrote ``text`` to a randomly-named temporary file and chunked
that path, which (a) did needless disk I/O and (b) made the returned
``node_id``/``chunk_id`` NONDETERMINISTIC across calls, because the temp
file name fed ``compute_node_id`` (IFACE). Passing an explicit ``file_path``
(default ``""``) makes identities stable and reproducible.
"""
from .core import chunk_text as _core_chunk_text
return _core_chunk_text(text, language, file_path=file_path, **kwargs)
# Convenient exports for common use cases
__all__ = [
"ASTCache",
"AdvancedQueryIndex",
"ChunkBoundaryAnalyzer",
"ChunkOptimizer",
"ChunkerConfig",
"ChunkingDecision",
"CodeChunk",
"DecisionMetrics",
"DefaultChangeDetector",
"DefaultChunkCache",
"DefaultIncrementalProcessor",
"InMemoryContextCache",
"IntelligentFallbackChunker",
"LanguageDetectorImpl",
"LanguageNotFoundError",
"LibraryNotFoundError",
"MultiLanguageProcessorImpl",
]
# Extend __all__ with advanced query API only if import succeeded
if _ADV_QUERY_AVAILABLE:
__all__.extend(
[
"AdvancedQueryIndex",
"NaturalLanguageQueryEngine",
"SmartQueryOptimizer",
],
)
__all__.extend(
[
"OptimizationConfig",
"OptimizationMetrics",
"OptimizationStrategy",
"ParserConfig",
"ParserError",
"PluginConfig",
"PluginManager",
"ProjectAnalyzerImpl",
"SemanticQuery",
"TreeSitterSmartContextProvider",
"__version__",
"chunk_directory",
"chunk_file",
"chunk_file_streaming",
"chunk_text",
"chunk_text_with_token_limit",
"clear_cache",
"count_chunk_tokens",
"extract_boundary_ir",
"get_language_info",
"get_parser",
"get_plugin_manager",
"list_languages",
"return_parser",
],
)

Xet Storage Details

Size:
4.91 kB
·
Xet hash:
24b7d71e944a97979a7fab9851370332b3f212a42d64e791fc74e4974ab3a0e3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.