Buckets:

download
raw
10.8 kB
#!/usr/bin/env python3
"""
Demonstration of overlapping chunks for non-Tree-sitter files.
This example shows how to use the OverlappingFallbackChunker for files
that don't have Tree-sitter support, such as text files, markdown, logs, etc.
"""
from chunker import OverlappingFallbackChunker, OverlapStrategy
def print_chunk_info(chunks, title):
"""Print information about chunks."""
print(f"\n{title}")
print("=" * len(title))
print(f"Total chunks: {len(chunks)}")
for i, chunk in enumerate(chunks):
print(f"\nChunk {i + 1}:")
print(f" Lines: {chunk.start_line}-{chunk.end_line}")
print(f" Bytes: {chunk.byte_start}-{chunk.byte_end}")
print(f" Size: {len(chunk.content)} characters")
print(
(
f" Preview: {chunk.content[:50]}..."
if len(chunk.content) > 50
else f" Content: {chunk.content}"
),
)
def demo_fixed_overlap():
"""Demonstrate fixed overlap strategy."""
print("\n" + "=" * 60)
print("FIXED OVERLAP STRATEGY DEMO")
print("=" * 60)
# Create sample text content
content = """# Project Documentation
## Introduction
This is a comprehensive guide to our project. The documentation covers all aspects
of the system architecture, implementation details, and usage instructions.
## System Architecture
The system is built using a microservices architecture with the following components:
- API Gateway: Handles all incoming requests and routes them to appropriate services
- Authentication Service: Manages user authentication and authorization
- Data Processing Service: Handles all data transformation and analysis
- Storage Service: Manages data persistence and retrieval
## Implementation Details
### API Gateway
The API Gateway is implemented using Node.js and Express. It provides:
1. Request routing based on URL patterns
2. Load balancing across service instances
3. Request/response transformation
4. Rate limiting and throttling
### Authentication Service
Built with Python and FastAPI, this service handles:
- User registration and login
- JWT token generation and validation
- Role-based access control
- Password reset functionality
## Usage Instructions
To get started with the system:
1. Install all dependencies using the provided scripts
2. Configure environment variables as described in .env.example
3. Run the initialization script to set up the database
4. Start all services using docker-compose
## Troubleshooting
Common issues and their solutions:
- Connection refused: Check if all services are running
- Authentication failed: Verify JWT token is valid
- Data not persisting: Check database connection settings
"""
chunker = OverlappingFallbackChunker()
# Chunk by lines with fixed overlap
chunks_lines = chunker.chunk_with_overlap(
content=content,
file_path="docs/README.md",
chunk_size=10, # 10 lines per chunk
overlap_size=3, # 3 lines overlap
strategy=OverlapStrategy.FIXED,
unit="lines",
)
print_chunk_info(chunks_lines, "Line-based chunking with 3-line overlap")
# Show overlap between first two chunks
if len(chunks_lines) >= 2:
print("\n### Overlap Demonstration ###")
chunk1_lines = chunks_lines[0].content.splitlines()
chunk2_lines = chunks_lines[1].content.splitlines()
print("\nEnd of Chunk 1 (last 3 lines):")
for line in chunk1_lines[-3:]:
print(f" {line}")
print("\nStart of Chunk 2 (first 3 lines):")
for line in chunk2_lines[:3]:
print(f" {line}")
# Chunk by characters with fixed overlap
chunks_chars = chunker.chunk_with_overlap(
content=content,
file_path="docs/README.md",
chunk_size=300, # 300 characters per chunk
overlap_size=50, # 50 characters overlap
strategy=OverlapStrategy.FIXED,
unit="characters",
)
print_chunk_info(chunks_chars, "\nCharacter-based chunking with 50-char overlap")
def demo_asymmetric_overlap():
"""Demonstrate asymmetric overlap for log files."""
print("\n" + "=" * 60)
print("ASYMMETRIC OVERLAP STRATEGY DEMO")
print("=" * 60)
# Create sample log content
log_content = """2024-01-20 10:15:23 INFO Starting application server
2024-01-20 10:15:24 INFO Loading configuration from config.yaml
2024-01-20 10:15:25 INFO Database connection established
2024-01-20 10:15:26 INFO Starting HTTP server on port 8080
2024-01-20 10:15:27 INFO Server ready to accept connections
2024-01-20 10:16:45 INFO Received GET request for /api/users
2024-01-20 10:16:45 INFO Authenticating user token
2024-01-20 10:16:46 INFO Token validated successfully
2024-01-20 10:16:46 INFO Fetching user data from database
2024-01-20 10:16:47 INFO Returning 25 user records
2024-01-20 10:18:12 WARNING High memory usage detected: 85%
2024-01-20 10:18:13 INFO Running garbage collection
2024-01-20 10:18:15 INFO Memory usage reduced to 62%
2024-01-20 10:20:34 ERROR Failed to connect to external API
2024-01-20 10:20:34 ERROR Timeout after 30 seconds
2024-01-20 10:20:35 INFO Retrying connection attempt 1 of 3
2024-01-20 10:20:45 ERROR Connection failed again
2024-01-20 10:20:46 INFO Retrying connection attempt 2 of 3
2024-01-20 10:20:56 INFO Connection successful
2024-01-20 10:20:57 INFO Resuming normal operation"""
chunker = OverlappingFallbackChunker()
# For logs, we want more forward context than backward
chunks = chunker.chunk_with_asymmetric_overlap(
content=log_content,
file_path="logs/app.log",
chunk_size=5, # 5 lines per chunk base
overlap_before=1, # Only 1 line before
overlap_after=3, # 3 lines after for context
unit="lines",
)
print_chunk_info(chunks, "Log file with asymmetric overlap (1 before, 3 after)")
# Show how errors get more context
for i, chunk in enumerate(chunks):
if "ERROR" in chunk.content:
print(f"\n### Error Context in Chunk {i + 1} ###")
print("This chunk contains an error with forward context:")
print(chunk.content)
def demo_dynamic_overlap():
"""Demonstrate dynamic overlap based on content structure."""
print("\n" + "=" * 60)
print("DYNAMIC OVERLAP STRATEGY DEMO")
print("=" * 60)
# Create structured content with varying density
structured_content = """# Configuration Guide
## Overview
This document describes the configuration options available in our system.
## Basic Settings
### Server Configuration
```yaml
server:
host: localhost
port: 8080
workers: 4
```
### Database Configuration
```yaml
database:
host: db.example.com
port: 5432
name: myapp
user: appuser
```
## Advanced Settings
For production deployments, consider these additional settings:
### Performance Tuning
Connection pooling can significantly improve performance:
```yaml
pool:
min_connections: 10
max_connections: 100
idle_timeout: 300
```
### Security Settings
Always enable these in production:
```yaml
security:
ssl_enabled: true
cert_path: /etc/ssl/certs/app.crt
key_path: /etc/ssl/private/app.key
```
## Environment Variables
The following environment variables override config file settings:
- APP_HOST: Override server.host
- APP_PORT: Override server.port
- DB_HOST: Override database.host
- DB_PASSWORD: Database password (never store in config file)
## Troubleshooting
If settings don't take effect:
1. Check environment variables first
2. Verify config file syntax
3. Look for typos in setting names
4. Restart the application
Remember that changes require a full restart.
"""
chunker = OverlappingFallbackChunker()
# Dynamic overlap adjusts based on content structure
chunks = chunker.chunk_with_dynamic_overlap(
content=structured_content,
file_path="docs/config.md",
chunk_size=400, # 400 characters per chunk
min_overlap=30, # Minimum 30 chars
max_overlap=100, # Maximum 100 chars
unit="characters",
)
print_chunk_info(chunks, "Dynamic overlap based on content structure")
# Analyze overlap variation
print("\n### Overlap Analysis ###")
for i in range(len(chunks) - 1):
chunk1 = chunks[i].content
chunk2 = chunks[i + 1].content
# Find actual overlap
overlap_size = 0
for size in range(min(len(chunk1), len(chunk2)), 0, -1):
if chunk1[-size:] == chunk2[:size]:
overlap_size = size
break
if overlap_size > 0:
print(f"\nOverlap between chunks {i + 1} and {i + 2}: {overlap_size} chars")
overlap_text = chunk1[-overlap_size:]
# Show what was included in overlap
if "\n\n" in overlap_text:
print(" -> Includes paragraph break")
elif "\n" in overlap_text:
print(" -> Includes line break")
elif "```" in overlap_text:
print(" -> Includes code block boundary")
def demo_natural_boundaries():
"""Demonstrate natural boundary detection."""
print("\n" + "=" * 60)
print("NATURAL BOUNDARY DETECTION DEMO")
print("=" * 60)
content = """This is the first sentence. This is the second sentence.
This is a new paragraph with its own content. It continues here.
## New Section
This section has different content altogether."""
chunker = OverlappingFallbackChunker()
# Test boundary detection at different positions
test_positions = [28, 57, 85, 120]
print("Testing natural boundary detection:")
for pos in test_positions:
if pos < len(content):
boundary = chunker.find_natural_overlap_boundary(content, pos, 20)
print(f"\nDesired position: {pos}")
print(f"Natural boundary: {boundary}")
print(f"Character at boundary: {content[boundary - 1:boundary + 1]!r}")
def main():
"""Run all demonstrations."""
print("Overlapping Fallback Chunker Demonstration")
print("This demo shows overlapping chunks for non-Tree-sitter files")
# Run demonstrations
demo_fixed_overlap()
demo_asymmetric_overlap()
demo_dynamic_overlap()
demo_natural_boundaries()
print("\n" + "=" * 60)
print("Demo complete!")
print("\nKey takeaways:")
print("- Fixed overlap maintains consistent context between chunks")
print("- Asymmetric overlap is useful for logs and streaming data")
print("- Dynamic overlap adapts to content structure")
print("- Natural boundaries improve readability and context preservation")
print("- This is ONLY for files without Tree-sitter support")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
10.8 kB
·
Xet hash:
2a239694e3a23fd762190969d3b7b0d732db027f40bf9b5b0520bb731e112289

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.