Buckets:

download
raw
9.23 kB
"""Example demonstrating the sliding window fallback integration.
This example shows how to use the SlidingWindowFallback system with:
- Automatic processor selection
- Custom processor registration
- Configuration support
- Processor chaining for hybrid processing
"""
import time
from typing import Any
from chunker.chunker_config import ChunkerConfig
from chunker.fallback import SlidingWindowFallback, TextProcessor
from chunker.fallback.detection.file_type import FileType
from chunker.types import CodeChunk
class SQLProcessor(TextProcessor):
"""Custom processor for SQL files."""
@staticmethod
def can_process(content: str, file_path: str) -> bool:
"""Check if content is SQL."""
return (
file_path.endswith(".sql")
or "SELECT" in content.upper()
or "CREATE TABLE" in content.upper()
)
@classmethod
def process(cls, content: str, file_path: str) -> list[CodeChunk]:
"""Process SQL content by statements."""
chunks = []
statements = content.split(";")
current_pos = 0
for i, statement in enumerate(statements):
if not statement.strip():
current_pos += len(statement) + 1
continue
if i < len(statements) - 1:
statement += ";"
lines_before = content[:current_pos].count("\n")
start_line = lines_before + 1
end_line = start_line + statement.count("\n")
chunk = CodeChunk(
language="sql",
file_path=file_path,
node_type="sql_statement",
start_line=start_line,
end_line=end_line,
byte_start=current_pos,
byte_end=current_pos + len(statement),
parent_context=f"statement_{i}",
content=statement,
)
chunks.append(chunk)
current_pos += len(statement)
return chunks
class DataFileProcessor(TextProcessor):
"""Processor for structured data files (CSV, TSV)."""
def __init__(self, config: dict[str, Any] | None = None):
super().__init__(config)
self.rows_per_chunk = self.config.get("rows_per_chunk", 100)
self.include_header = self.config.get("include_header", True)
@staticmethod
def can_process(content: str, file_path: str) -> bool:
"""Check if content is structured data."""
return (
file_path.endswith(
(".csv", ".tsv"),
)
or "\t" in content.splitlines()[0]
if content
else False
)
def process(self, content: str, file_path: str) -> list[CodeChunk]:
"""Process data file by rows."""
lines = content.splitlines(keepends=True)
if not lines:
return []
chunks = []
header = lines[0] if self.include_header else ""
for i in range(0, len(lines), self.rows_per_chunk):
chunk_lines = lines[i : i + self.rows_per_chunk]
if i > 0 and self.include_header and header:
chunk_content = header + "".join(chunk_lines)
else:
chunk_content = "".join(chunk_lines)
chunk = CodeChunk(
language="data",
file_path=file_path,
node_type="data_rows",
start_line=i + 1,
end_line=min(i + self.rows_per_chunk, len(lines)),
byte_start=0,
byte_end=len(chunk_content),
parent_context=f"rows_{i}_{i + self.rows_per_chunk}",
content=chunk_content,
)
chunks.append(chunk)
return chunks
def main():
"""Demonstrate sliding window fallback integration."""
print("=== Sliding Window Fallback Integration Demo ===\n")
print("1. Basic Usage - Automatic Processor Selection")
print("-" * 50)
fallback = SlidingWindowFallback()
test_files = {
"example.md": "# Title\n\nSection 1\n\n## Subsection\n\nContent",
"app.log": """[INFO] Starting
[ERROR] Failed to connect
[INFO] Retrying""",
"data.txt": "Just plain text without any special format.\n" * 10,
}
for filename, content in test_files.items():
chunks = fallback.chunk_text(content, filename)
print(f"\n{filename}: {len(chunks)} chunks")
for i, chunk in enumerate(chunks):
processor = chunk.metadata.get("processor", "unknown")
print(f" Chunk {i + 1}: {chunk.node_type} (processor: {processor})")
print("\n\n2. Custom Processor Registration")
print("-" * 50)
fallback.register_custom_processor(
name="sql_processor",
processor_class=SQLProcessor,
file_types={FileType.TEXT},
extensions={".sql"},
priority=150,
)
fallback.register_custom_processor(
name="data_processor",
processor_class=DataFileProcessor,
file_types={FileType.CSV, FileType.TEXT},
extensions={".csv", ".tsv"},
priority=120,
)
sql_content = """
CREATE TABLE users (
id INTEGER PRIMARY KEY,
name VARCHAR(100)
);
INSERT INTO users (name) VALUES ('Alice');
INSERT INTO users (name) VALUES ('Bob');
SELECT * FROM users WHERE id > 0;
"""
chunks = fallback.chunk_text(sql_content, "schema.sql")
print(f"\nSQL file: {len(chunks)} chunks")
for i, chunk in enumerate(chunks):
print(f" Chunk {i + 1}: {chunk.content.strip()[:50]}...")
print("\n\n3. Configuration Support")
print("-" * 50)
config_data = {
"processors": {
"data_processor": {
"enabled": True,
"priority": 150,
"config": {"rows_per_chunk": 50, "include_header": True},
},
},
}
chunker_config = ChunkerConfig()
chunker_config.data = config_data
configured_fallback = SlidingWindowFallback(chunker_config=chunker_config)
configured_fallback.register_custom_processor(
name="data_processor",
processor_class=DataFileProcessor,
file_types={FileType.CSV},
extensions={".csv"},
priority=120,
)
csv_content = "name,age,city\n" + "\n".join(
[f"Person{i},{20 + i},City{i}" for i in range(200)],
)
chunks = configured_fallback.chunk_text(csv_content, "data.csv")
print(f"\nCSV file: {len(chunks)} chunks (configured for 50 rows/chunk)")
print("\n\n4. Processor Information and Control")
print("-" * 50)
for filename in ["test.sql", "test.csv", "test.md", "test.xyz"]:
info = fallback.get_processor_info(filename)
print(f"\n{filename}:")
print(f" File type: {info['file_type']}")
print(f" Available processors: {info['available_processors']}")
print("\n\nDisabling SQL processor...")
fallback.disable_processor("sql_processor")
chunks = fallback.chunk_text(sql_content, "schema.sql")
processor_used = chunks[0].metadata.get("processor", "unknown")
print(f"SQL file processed with: {processor_used}")
print("\nRe-enabling SQL processor...")
fallback.enable_processor("sql_processor")
chunks = fallback.chunk_text(sql_content, "schema.sql")
processor_used = chunks[0].metadata.get("processor", "unknown")
print(f"SQL file processed with: {processor_used}")
print("\n\n5. Processor Chaining (Hybrid Mode)")
print("-" * 50)
mixed_content = """
# Database Schema Documentation
This document describes our database schema.
## Users Table
```sql
CREATE TABLE users (
id INTEGER PRIMARY KEY,
name VARCHAR(100)
);
```
## Sample Data
name,age,city
Alice,25,NYC
Bob,30,LA
Charlie,35,Chicago
"""
chunks = fallback.chunk_text(mixed_content, "schema_doc.md")
print(f"\nSingle processor: {len(chunks)} chunks")
chain = fallback.create_processor_chain(["markdown_processor", "sql_processor"])
if chain:
print("\nProcessor chain created successfully")
print("\n\n6. Performance and Caching")
print("-" * 50)
large_content = "Line of text\n" * 1000
start = time.time()
chunks1 = fallback.chunk_text(large_content, "large.txt")
time1 = time.time() - start
start = time.time()
chunks2 = fallback.chunk_text(large_content, "large.txt")
time2 = time.time() - start
print(f"\nFirst processing: {time1:.4f}s ({len(chunks1)} chunks)")
print(f"Second processing: {time2:.4f}s ({len(chunks2)} chunks)")
print(f"Caching benefit: {(time1 - time2) / time1 * 100:.1f}% faster")
print("\n\n=== Summary ===")
print("-" * 50)
print("The SlidingWindowFallback system provides:")
print("- Automatic processor selection based on file type")
print("- Easy custom processor registration")
print("- Configuration support for processor behavior")
print("- Processor chaining for complex formats")
print("- Performance optimization through caching")
print("- Dynamic processor enabling/disabling")
print("\nThis allows flexible text processing for any file type!")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
9.23 kB
·
Xet hash:
5918e090ca135b37c6c012fb4a94e70337cdc34cfab1d12e74931d3aa010e56a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.