Buckets:

download
raw
5.96 kB
"""Example of using incremental processing for efficient chunk updates."""
import pathlib
import shutil
import tempfile
from chunker import (
DefaultChangeDetector,
DefaultChunkCache,
DefaultIncrementalProcessor,
chunk_file,
)
from chunker.types import CodeChunk
def demonstrate_incremental_processing():
"""Demonstrate incremental processing workflow."""
# Initialize components
processor = DefaultIncrementalProcessor()
cache = DefaultChunkCache()
detector = DefaultChangeDetector()
# Example: Processing a Python file
initial_content = '''def hello():
"""Say hello."""
print("Hello, World!")
def calculate(a, b):
"""Calculate sum."""
return a + b
class Calculator:
"""Basic calculator."""
def __init__(self):
self.result = 0
def add(self, value):
"""Add to result."""
self.result += value
'''
# Create a temporary file
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
suffix=".py",
delete=False,
) as f:
f.write(initial_content)
file_path = f.name
try:
# Initial processing
print("=== Initial Processing ===")
initial_chunks = chunk_file(file_path, language="python")
file_hash = detector.compute_file_hash(initial_content)
# Cache the results
cache.store(file_path, initial_chunks, file_hash)
print(f"Cached {len(initial_chunks)} chunks")
# Simulate file modification
modified_content = '''def hello():
"""Say hello with name."""
name = input("Enter name: ")
print(f"Hello, {name}!")
def calculate(a, b, operation="+"):
"""Calculate with operation."""
if operation == "+":
return a + b
elif operation == "-":
return a - b
return 0
class Calculator:
"""Advanced calculator."""
def __init__(self):
self.result = 0
self.history = []
def add(self, value):
"""Add to result and track history."""
self.result += value
self.history.append(f"Added {value}")
def subtract(self, value):
"""Subtract from result."""
self.result -= value
self.history.append(f"Subtracted {value}")
'''
# Write modified content
with open(file_path, "w", encoding="utf-8") as f:
f.write(modified_content)
# Check if file changed
new_hash = detector.compute_file_hash(modified_content)
if new_hash != file_hash:
print("\n=== File Changed - Computing Diff ===")
# Retrieve cached chunks
cache_entry = cache.retrieve(file_path, file_hash)
if cache_entry:
old_chunks = cache_entry.chunks
# Compute diff
diff = processor.compute_diff(old_chunks, modified_content, "python")
print("\nDiff Summary:")
print(f" Added chunks: {diff.summary['added']}")
print(f" Deleted chunks: {diff.summary['deleted']}")
print(f" Modified chunks: {diff.summary['modified']}")
print(f" Moved chunks: {diff.summary['moved']}")
print(f" Unchanged chunks: {diff.summary['unchanged']}")
# Show specific changes
print("\nDetailed Changes:")
for change in diff.changes:
print(f" - {change.change_type.value}: {change.chunk_id}")
if change.change_type.value == "modified":
print(f" Line changes: {change.line_changes}")
# Update chunks
updated_chunks = processor.update_chunks(old_chunks, diff)
# Cache updated results
cache.store(file_path, updated_chunks, new_hash)
print(f"\nUpdated cache with {len(updated_chunks)} chunks")
# Show cache statistics
stats = cache.get_statistics()
print("\nCache Statistics:")
print(f" Entries: {stats['entries']}")
print(f" Size: {stats['total_size_mb']:.2f} MB")
print(f" Hit rate: {stats['hit_rate']:.2%}")
# Demonstrate change detection
print("\n=== Change Detection ===")
regions = detector.find_changed_regions(initial_content, modified_content)
print(f"Found {len(regions)} changed regions:")
for start, end in regions:
print(f" Lines {start}-{end}")
finally:
# Cleanup
pathlib.Path(file_path).unlink()
def demonstrate_cache_persistence():
"""Demonstrate cache export/import functionality."""
print("\n=== Cache Persistence Demo ===")
# Create sample chunks
chunks = [
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=1,
end_line=3,
byte_start=0,
byte_end=50,
parent_context="",
content="def example():\n pass\n",
),
]
# Create and populate cache
cache1 = DefaultChunkCache(".cache1")
cache1.store("example.py", chunks, "hash123")
# Export cache
export_file = "cache_export.json"
cache1.export_cache(export_file)
print(f"Exported cache to {export_file}")
# Import into new cache
cache2 = DefaultChunkCache(".cache2")
cache2.import_cache(export_file)
print(f"Imported cache from {export_file}")
# Verify
entry = cache2.retrieve("example.py")
if entry:
print(f"Successfully retrieved {len(entry.chunks)} chunks from imported cache")
# Cleanup
shutil.rmtree(".cache1", ignore_errors=True)
shutil.rmtree(".cache2", ignore_errors=True)
pathlib.Path(export_file).unlink()
if __name__ == "__main__":
demonstrate_incremental_processing()
demonstrate_cache_persistence()

Xet Storage Details

Size:
5.96 kB
·
Xet hash:
7ed90e279588bf20a447bcba715b2feea2ef124a7a530b341e9856a649e1aca7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.