Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /chunker /__init__.py
| """ | |
| Tree-sitter Chunker - Semantic code chunking for LLMs and embeddings. | |
| Simple usage: | |
| from chunker import chunk_file, chunk_text, chunk_directory | |
| # Chunk a single file | |
| chunks = chunk_file("example.py", language="python") | |
| # Chunk raw text | |
| chunks = chunk_text(code_text, language="javascript") | |
| # Chunk entire directory | |
| results = chunk_directory("src/", language="python") | |
| """ | |
| from ._internal.cache import ASTCache | |
| from .chunker import chunk_text_with_token_limit, count_chunk_tokens | |
| from .chunker_config import ChunkerConfig | |
| from .core import chunk_file | |
| from .boundary import extract_boundary_ir | |
| from .exceptions import LanguageNotFoundError, LibraryNotFoundError, ParserError | |
| from .fallback.intelligent_fallback import ( | |
| ChunkingDecision, | |
| DecisionMetrics, | |
| IntelligentFallbackChunker, | |
| ) | |
| from .incremental import ( | |
| DefaultChangeDetector, | |
| DefaultChunkCache, | |
| DefaultIncrementalProcessor, | |
| ) | |
| from .languages.base import PluginConfig | |
| from .multi_language import ( | |
| LanguageDetectorImpl, | |
| MultiLanguageProcessorImpl, | |
| ProjectAnalyzerImpl, | |
| ) | |
| from .optimization import ( | |
| ChunkBoundaryAnalyzer, | |
| ChunkOptimizer, | |
| OptimizationConfig, | |
| OptimizationMetrics, | |
| OptimizationStrategy, | |
| ) | |
| from .parallel import chunk_directory_parallel as chunk_directory | |
| from .parser import ( | |
| ParserConfig, | |
| clear_cache, | |
| get_language_info, | |
| get_parser, | |
| list_languages, | |
| return_parser, | |
| ) | |
| from .plugin_manager import PluginManager, get_plugin_manager | |
| from .semantic_query import SemanticQuery | |
| # Optional advanced query features (may require heavy deps like numpy) | |
| try: | |
| from .query_advanced import ( | |
| AdvancedQueryIndex, | |
| NaturalLanguageQueryEngine, | |
| SmartQueryOptimizer, | |
| ) | |
| _ADV_QUERY_AVAILABLE = True | |
| except Exception: # pragma: no cover - keep import lightweight during build tests | |
| AdvancedQueryIndex = None # type: ignore[assignment] | |
| NaturalLanguageQueryEngine = None # type: ignore[assignment] | |
| SmartQueryOptimizer = None # type: ignore[assignment] | |
| _ADV_QUERY_AVAILABLE = False | |
| from .smart_context import InMemoryContextCache, TreeSitterSmartContextProvider | |
| from .streaming import chunk_file_streaming | |
| from .types import CodeChunk | |
| # Dynamic version from package metadata | |
| try: | |
| from importlib.metadata import version | |
| __version__ = version("treesitter-chunker") | |
| except ImportError: | |
| # Fallback when package metadata is unavailable (editable import without an | |
| # installed dist). Must match pyproject.toml `version` — the single source of | |
| # truth — so an uninstalled import does not report a stale version (IFACE). | |
| __version__ = "4.0.0" | |
| # Simple text chunking | |
| def chunk_text(text: str, language: str, file_path: str = "", **kwargs): | |
| """Chunk text content directly, in memory (no temp-file round-trip). | |
| Delegates to ``core.chunk_text``, which parses ``text`` directly. The former | |
| implementation wrote ``text`` to a randomly-named temporary file and chunked | |
| that path, which (a) did needless disk I/O and (b) made the returned | |
| ``node_id``/``chunk_id`` NONDETERMINISTIC across calls, because the temp | |
| file name fed ``compute_node_id`` (IFACE). Passing an explicit ``file_path`` | |
| (default ``""``) makes identities stable and reproducible. | |
| """ | |
| from .core import chunk_text as _core_chunk_text | |
| return _core_chunk_text(text, language, file_path=file_path, **kwargs) | |
| # Convenient exports for common use cases | |
| __all__ = [ | |
| "ASTCache", | |
| "AdvancedQueryIndex", | |
| "ChunkBoundaryAnalyzer", | |
| "ChunkOptimizer", | |
| "ChunkerConfig", | |
| "ChunkingDecision", | |
| "CodeChunk", | |
| "DecisionMetrics", | |
| "DefaultChangeDetector", | |
| "DefaultChunkCache", | |
| "DefaultIncrementalProcessor", | |
| "InMemoryContextCache", | |
| "IntelligentFallbackChunker", | |
| "LanguageDetectorImpl", | |
| "LanguageNotFoundError", | |
| "LibraryNotFoundError", | |
| "MultiLanguageProcessorImpl", | |
| ] | |
| # Extend __all__ with advanced query API only if import succeeded | |
| if _ADV_QUERY_AVAILABLE: | |
| __all__.extend( | |
| [ | |
| "AdvancedQueryIndex", | |
| "NaturalLanguageQueryEngine", | |
| "SmartQueryOptimizer", | |
| ], | |
| ) | |
| __all__.extend( | |
| [ | |
| "OptimizationConfig", | |
| "OptimizationMetrics", | |
| "OptimizationStrategy", | |
| "ParserConfig", | |
| "ParserError", | |
| "PluginConfig", | |
| "PluginManager", | |
| "ProjectAnalyzerImpl", | |
| "SemanticQuery", | |
| "TreeSitterSmartContextProvider", | |
| "__version__", | |
| "chunk_directory", | |
| "chunk_file", | |
| "chunk_file_streaming", | |
| "chunk_text", | |
| "chunk_text_with_token_limit", | |
| "clear_cache", | |
| "count_chunk_tokens", | |
| "extract_boundary_ir", | |
| "get_language_info", | |
| "get_parser", | |
| "get_plugin_manager", | |
| "list_languages", | |
| "return_parser", | |
| ], | |
| ) | |
Xet Storage Details
- Size:
- 4.91 kB
- Xet hash:
- 24b7d71e944a97979a7fab9851370332b3f212a42d64e791fc74e4974ab3a0e3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.