Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /example_sliding_window_integration.py
| """Example demonstrating the sliding window fallback integration. | |
| This example shows how to use the SlidingWindowFallback system with: | |
| - Automatic processor selection | |
| - Custom processor registration | |
| - Configuration support | |
| - Processor chaining for hybrid processing | |
| """ | |
| import time | |
| from typing import Any | |
| from chunker.chunker_config import ChunkerConfig | |
| from chunker.fallback import SlidingWindowFallback, TextProcessor | |
| from chunker.fallback.detection.file_type import FileType | |
| from chunker.types import CodeChunk | |
| class SQLProcessor(TextProcessor): | |
| """Custom processor for SQL files.""" | |
| def can_process(content: str, file_path: str) -> bool: | |
| """Check if content is SQL.""" | |
| return ( | |
| file_path.endswith(".sql") | |
| or "SELECT" in content.upper() | |
| or "CREATE TABLE" in content.upper() | |
| ) | |
| def process(cls, content: str, file_path: str) -> list[CodeChunk]: | |
| """Process SQL content by statements.""" | |
| chunks = [] | |
| statements = content.split(";") | |
| current_pos = 0 | |
| for i, statement in enumerate(statements): | |
| if not statement.strip(): | |
| current_pos += len(statement) + 1 | |
| continue | |
| if i < len(statements) - 1: | |
| statement += ";" | |
| lines_before = content[:current_pos].count("\n") | |
| start_line = lines_before + 1 | |
| end_line = start_line + statement.count("\n") | |
| chunk = CodeChunk( | |
| language="sql", | |
| file_path=file_path, | |
| node_type="sql_statement", | |
| start_line=start_line, | |
| end_line=end_line, | |
| byte_start=current_pos, | |
| byte_end=current_pos + len(statement), | |
| parent_context=f"statement_{i}", | |
| content=statement, | |
| ) | |
| chunks.append(chunk) | |
| current_pos += len(statement) | |
| return chunks | |
| class DataFileProcessor(TextProcessor): | |
| """Processor for structured data files (CSV, TSV).""" | |
| def __init__(self, config: dict[str, Any] | None = None): | |
| super().__init__(config) | |
| self.rows_per_chunk = self.config.get("rows_per_chunk", 100) | |
| self.include_header = self.config.get("include_header", True) | |
| def can_process(content: str, file_path: str) -> bool: | |
| """Check if content is structured data.""" | |
| return ( | |
| file_path.endswith( | |
| (".csv", ".tsv"), | |
| ) | |
| or "\t" in content.splitlines()[0] | |
| if content | |
| else False | |
| ) | |
| def process(self, content: str, file_path: str) -> list[CodeChunk]: | |
| """Process data file by rows.""" | |
| lines = content.splitlines(keepends=True) | |
| if not lines: | |
| return [] | |
| chunks = [] | |
| header = lines[0] if self.include_header else "" | |
| for i in range(0, len(lines), self.rows_per_chunk): | |
| chunk_lines = lines[i : i + self.rows_per_chunk] | |
| if i > 0 and self.include_header and header: | |
| chunk_content = header + "".join(chunk_lines) | |
| else: | |
| chunk_content = "".join(chunk_lines) | |
| chunk = CodeChunk( | |
| language="data", | |
| file_path=file_path, | |
| node_type="data_rows", | |
| start_line=i + 1, | |
| end_line=min(i + self.rows_per_chunk, len(lines)), | |
| byte_start=0, | |
| byte_end=len(chunk_content), | |
| parent_context=f"rows_{i}_{i + self.rows_per_chunk}", | |
| content=chunk_content, | |
| ) | |
| chunks.append(chunk) | |
| return chunks | |
| def main(): | |
| """Demonstrate sliding window fallback integration.""" | |
| print("=== Sliding Window Fallback Integration Demo ===\n") | |
| print("1. Basic Usage - Automatic Processor Selection") | |
| print("-" * 50) | |
| fallback = SlidingWindowFallback() | |
| test_files = { | |
| "example.md": "# Title\n\nSection 1\n\n## Subsection\n\nContent", | |
| "app.log": """[INFO] Starting | |
| [ERROR] Failed to connect | |
| [INFO] Retrying""", | |
| "data.txt": "Just plain text without any special format.\n" * 10, | |
| } | |
| for filename, content in test_files.items(): | |
| chunks = fallback.chunk_text(content, filename) | |
| print(f"\n{filename}: {len(chunks)} chunks") | |
| for i, chunk in enumerate(chunks): | |
| processor = chunk.metadata.get("processor", "unknown") | |
| print(f" Chunk {i + 1}: {chunk.node_type} (processor: {processor})") | |
| print("\n\n2. Custom Processor Registration") | |
| print("-" * 50) | |
| fallback.register_custom_processor( | |
| name="sql_processor", | |
| processor_class=SQLProcessor, | |
| file_types={FileType.TEXT}, | |
| extensions={".sql"}, | |
| priority=150, | |
| ) | |
| fallback.register_custom_processor( | |
| name="data_processor", | |
| processor_class=DataFileProcessor, | |
| file_types={FileType.CSV, FileType.TEXT}, | |
| extensions={".csv", ".tsv"}, | |
| priority=120, | |
| ) | |
| sql_content = """ | |
| CREATE TABLE users ( | |
| id INTEGER PRIMARY KEY, | |
| name VARCHAR(100) | |
| ); | |
| INSERT INTO users (name) VALUES ('Alice'); | |
| INSERT INTO users (name) VALUES ('Bob'); | |
| SELECT * FROM users WHERE id > 0; | |
| """ | |
| chunks = fallback.chunk_text(sql_content, "schema.sql") | |
| print(f"\nSQL file: {len(chunks)} chunks") | |
| for i, chunk in enumerate(chunks): | |
| print(f" Chunk {i + 1}: {chunk.content.strip()[:50]}...") | |
| print("\n\n3. Configuration Support") | |
| print("-" * 50) | |
| config_data = { | |
| "processors": { | |
| "data_processor": { | |
| "enabled": True, | |
| "priority": 150, | |
| "config": {"rows_per_chunk": 50, "include_header": True}, | |
| }, | |
| }, | |
| } | |
| chunker_config = ChunkerConfig() | |
| chunker_config.data = config_data | |
| configured_fallback = SlidingWindowFallback(chunker_config=chunker_config) | |
| configured_fallback.register_custom_processor( | |
| name="data_processor", | |
| processor_class=DataFileProcessor, | |
| file_types={FileType.CSV}, | |
| extensions={".csv"}, | |
| priority=120, | |
| ) | |
| csv_content = "name,age,city\n" + "\n".join( | |
| [f"Person{i},{20 + i},City{i}" for i in range(200)], | |
| ) | |
| chunks = configured_fallback.chunk_text(csv_content, "data.csv") | |
| print(f"\nCSV file: {len(chunks)} chunks (configured for 50 rows/chunk)") | |
| print("\n\n4. Processor Information and Control") | |
| print("-" * 50) | |
| for filename in ["test.sql", "test.csv", "test.md", "test.xyz"]: | |
| info = fallback.get_processor_info(filename) | |
| print(f"\n{filename}:") | |
| print(f" File type: {info['file_type']}") | |
| print(f" Available processors: {info['available_processors']}") | |
| print("\n\nDisabling SQL processor...") | |
| fallback.disable_processor("sql_processor") | |
| chunks = fallback.chunk_text(sql_content, "schema.sql") | |
| processor_used = chunks[0].metadata.get("processor", "unknown") | |
| print(f"SQL file processed with: {processor_used}") | |
| print("\nRe-enabling SQL processor...") | |
| fallback.enable_processor("sql_processor") | |
| chunks = fallback.chunk_text(sql_content, "schema.sql") | |
| processor_used = chunks[0].metadata.get("processor", "unknown") | |
| print(f"SQL file processed with: {processor_used}") | |
| print("\n\n5. Processor Chaining (Hybrid Mode)") | |
| print("-" * 50) | |
| mixed_content = """ | |
| # Database Schema Documentation | |
| This document describes our database schema. | |
| ## Users Table | |
| ```sql | |
| CREATE TABLE users ( | |
| id INTEGER PRIMARY KEY, | |
| name VARCHAR(100) | |
| ); | |
| ``` | |
| ## Sample Data | |
| name,age,city | |
| Alice,25,NYC | |
| Bob,30,LA | |
| Charlie,35,Chicago | |
| """ | |
| chunks = fallback.chunk_text(mixed_content, "schema_doc.md") | |
| print(f"\nSingle processor: {len(chunks)} chunks") | |
| chain = fallback.create_processor_chain(["markdown_processor", "sql_processor"]) | |
| if chain: | |
| print("\nProcessor chain created successfully") | |
| print("\n\n6. Performance and Caching") | |
| print("-" * 50) | |
| large_content = "Line of text\n" * 1000 | |
| start = time.time() | |
| chunks1 = fallback.chunk_text(large_content, "large.txt") | |
| time1 = time.time() - start | |
| start = time.time() | |
| chunks2 = fallback.chunk_text(large_content, "large.txt") | |
| time2 = time.time() - start | |
| print(f"\nFirst processing: {time1:.4f}s ({len(chunks1)} chunks)") | |
| print(f"Second processing: {time2:.4f}s ({len(chunks2)} chunks)") | |
| print(f"Caching benefit: {(time1 - time2) / time1 * 100:.1f}% faster") | |
| print("\n\n=== Summary ===") | |
| print("-" * 50) | |
| print("The SlidingWindowFallback system provides:") | |
| print("- Automatic processor selection based on file type") | |
| print("- Easy custom processor registration") | |
| print("- Configuration support for processor behavior") | |
| print("- Processor chaining for complex formats") | |
| print("- Performance optimization through caching") | |
| print("- Dynamic processor enabling/disabling") | |
| print("\nThis allows flexible text processing for any file type!") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 9.23 kB
- Xet hash:
- 5918e090ca135b37c6c012fb4a94e70337cdc34cfab1d12e74931d3aa010e56a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.