Buckets:

download
raw
9.8 kB
"""Simple demonstration of incremental processing components."""
import tempfile
from pathlib import Path
from chunker import (
DefaultChangeDetector,
DefaultChunkCache,
DefaultIncrementalProcessor,
SimpleIncrementalIndex,
)
from chunker.interfaces.incremental import ChangeType, ChunkChange, ChunkDiff
from chunker.types import CodeChunk
def create_sample_chunks():
"""Create sample chunks for demonstration."""
return [
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=1,
end_line=5,
byte_start=0,
byte_end=100,
parent_context="",
content="def hello():\n print('Hello')\n return True\n",
chunk_id="func_hello",
),
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=7,
end_line=10,
byte_start=101,
byte_end=200,
parent_context="",
content="def process():\n data = load()\n return analyze(data)\n",
chunk_id="func_process",
),
]
def demonstrate_change_detection():
"""Demonstrate change detection capabilities."""
print("=== Change Detection Demo ===")
detector = DefaultChangeDetector()
# Original content
original = """def hello():
print('Hello')
return True
def process():
data = load()
return analyze(data)
"""
# Modified content
modified = """def hello():
print('Hello, World!') # Modified
return True
def process():
# Added comment
data = load()
result = analyze(data) # Changed variable name
log_result(result) # Added line
return result # Modified
"""
# Compute hash to detect changes
original_hash = detector.compute_file_hash(original)
modified_hash = detector.compute_file_hash(modified)
print(f"Original hash: {original_hash[:16]}...")
print(f"Modified hash: {modified_hash[:16]}...")
print(f"File changed: {original_hash != modified_hash}")
# Find changed regions
regions = detector.find_changed_regions(original, modified)
print("\nChanged regions:")
for start, end in regions:
print(f" Lines {start}-{end}")
def demonstrate_diff_computation():
"""Demonstrate diff computation between chunks."""
print("\n=== Diff Computation Demo ===")
DefaultIncrementalProcessor()
# Original chunks
old_chunks = create_sample_chunks()
# Create modified chunks (simulating what would come from parsing)
new_chunks = [
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=1,
end_line=5,
byte_start=0,
byte_end=120,
parent_context="",
content="def hello():\n print('Hello, World!')\n return True\n",
chunk_id="func_hello", # Same ID, modified content
),
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=7,
end_line=12,
byte_start=121,
byte_end=250,
parent_context="",
content="def process():\n # Process data\n data = load()\n result = analyze(data)\n return result\n",
chunk_id="func_process", # Same ID, modified content
),
CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=14,
end_line=16,
byte_start=251,
byte_end=300,
parent_context="",
content="def cleanup():\n clear_cache()\n",
chunk_id="func_cleanup", # New function
),
]
# Manually create a diff (simulating what compute_diff would do)
diff = ChunkDiff(
changes=[
ChunkChange(
chunk_id="func_hello",
change_type=ChangeType.MODIFIED,
old_chunk=old_chunks[0],
new_chunk=new_chunks[0],
line_changes=[(2, 2)],
confidence=0.9,
),
ChunkChange(
chunk_id="func_process",
change_type=ChangeType.MODIFIED,
old_chunk=old_chunks[1],
new_chunk=new_chunks[1],
line_changes=[(8, 12)],
confidence=0.85,
),
ChunkChange(
chunk_id="func_cleanup",
change_type=ChangeType.ADDED,
old_chunk=None,
new_chunk=new_chunks[2],
line_changes=[(14, 16)],
confidence=1.0,
),
],
added_chunks=[new_chunks[2]],
deleted_chunks=[],
modified_chunks=[
(old_chunks[0], new_chunks[0]),
(old_chunks[1], new_chunks[1]),
],
unchanged_chunks=[],
summary={
"total_old_chunks": 2,
"total_new_chunks": 3,
"added": 1,
"deleted": 0,
"modified": 2,
"moved": 0,
"unchanged": 0,
},
)
print(f"Diff Summary: {diff.summary}")
print("\nDetailed Changes:")
for change in diff.changes:
print(f" - {change.change_type.value}: {change.chunk_id}")
print(f" Confidence: {change.confidence:.2f}")
if change.line_changes:
print(f" Line changes: {change.line_changes}")
def demonstrate_cache():
"""Demonstrate cache functionality."""
print("\n=== Cache Demo ===")
with tempfile.TemporaryDirectory() as temp_dir:
cache = DefaultChunkCache(Path(temp_dir) / ".cache")
# Store chunks
chunks = create_sample_chunks()
cache.store("example.py", chunks, "hash123", metadata={"version": "1.0"})
print("Stored 2 chunks in cache")
# Retrieve with correct hash
entry = cache.retrieve("example.py", "hash123")
print(f"Retrieved {len(entry.chunks)} chunks with correct hash")
# Try with wrong hash
wrong_entry = cache.retrieve("example.py", "wronghash")
print(f"Retrieved with wrong hash: {wrong_entry is None}")
# Get statistics
stats = cache.get_statistics()
print("\nCache Statistics:")
print(f" Entries: {stats['entries']}")
print(f" Hit rate: {stats['hit_rate']:.0%}")
print(f" Total retrievals: {stats['stats']['retrievals']}")
def demonstrate_index():
"""Demonstrate incremental index functionality."""
print("\n=== Incremental Index Demo ===")
index = SimpleIncrementalIndex()
chunks = create_sample_chunks()
# Add chunks to index
for chunk in chunks:
index.update_chunk(None, chunk)
print(f"Indexed {len(index.index)} chunks")
# Search
results = index.search("hello")
print(f"\nSearch for 'hello': found {len(results)} results")
for chunk_id in results:
print(f" - {chunk_id}")
# Simulate updating a chunk
modified_chunk = CodeChunk(
language="python",
file_path="example.py",
node_type="function_definition",
start_line=1,
end_line=5,
byte_start=0,
byte_end=100,
parent_context="",
content="def hello():\n print('Goodbye')\n return False\n",
chunk_id="func_hello",
)
index.update_chunk(chunks[0], modified_chunk)
# Search again
results = index.search("goodbye")
print(f"\nSearch for 'goodbye': found {len(results)} results")
# Check update log
print(f"\nUpdate log has {len(index.update_log)} entries")
def demonstrate_move_detection():
"""Demonstrate code move detection."""
print("\n=== Move Detection Demo ===")
processor = DefaultIncrementalProcessor()
# Function in original location
old_chunk = CodeChunk(
language="python",
file_path="utils.py",
node_type="function_definition",
start_line=10,
end_line=15,
byte_start=200,
byte_end=300,
parent_context="",
content="def helper_function():\n '''Helper logic'''\n return do_work()\n",
chunk_id="utils_helper",
)
# Same function moved to different file and location
new_chunk = CodeChunk(
language="python",
file_path="helpers.py",
node_type="function_definition",
start_line=25,
end_line=30,
byte_start=500,
byte_end=600,
parent_context="HelperClass",
content="def helper_function():\n '''Helper logic'''\n return do_work()\n",
chunk_id="helpers_helper",
)
# Detect move
moved = processor.detect_moved_chunks([old_chunk], [new_chunk])
if moved:
print(f"Detected {len(moved)} moved chunks:")
for old, new in moved:
print(f" - From {old.file_path}:{old.start_line}")
print(f" To {new.file_path}:{new.start_line}")
print(" Similarity: Very High (same content)")
else:
print("No moves detected")
if __name__ == "__main__":
demonstrate_change_detection()
demonstrate_diff_computation()
demonstrate_cache()
demonstrate_index()
demonstrate_move_detection()
print("\n=== Demo Complete ===")
print("This demonstrates the key components of incremental processing.")
print("In real usage, these components work together to efficiently")
print("process only the changed parts of large codebases.")

Xet Storage Details

Size:
9.8 kB
·
Xet hash:
6be93e69f0ac58573328e4b14f7d44727316aef5ba1ee4ae17350930ad1ccdb1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.