Buckets:

download
raw
5.07 kB
"""Example demonstrating token counting integration with Tree-sitter chunks."""
import json
import pathlib
import tempfile
from chunker.token import TiktokenCounter
from chunker.token.chunker import TreeSitterTokenAwareChunker
def main():
"""Demonstrate token counting features."""
# Example 1: Basic token counting
print("=== Basic Token Counting ===")
counter = TiktokenCounter()
sample_code = """
def calculate_fibonacci(n):
if n <= 1:
return n
return calculate_fibonacci(n-1) + calculate_fibonacci(n-2)
"""
token_count = counter.count_tokens(sample_code)
print(f"Sample code has {token_count} tokens")
print(f"GPT-4 limit: {counter.get_token_limit('gpt-4')} tokens")
print(f"Claude limit: {counter.get_token_limit('claude')} tokens")
# Example 2: Chunking with token information
print("\n=== Token-Aware Chunking ===")
chunker = TreeSitterTokenAwareChunker()
# Create a test file
test_file = "examples/example.py"
# Regular chunking with token info added
chunks = chunker.chunk_file(test_file, "python")
print(f"\nFound {len(chunks)} chunks in {test_file}")
for i, chunk in enumerate(chunks[:3]): # Show first 3 chunks
print(f"\nChunk {i + 1}:")
print(f" Type: {chunk.node_type}")
print(f" Lines: {chunk.start_line}-{chunk.end_line}")
print(f" Tokens: {chunk.metadata.get('token_count', 'N/A')}")
print(f" Content preview: {chunk.content[:50]}...")
# Example 3: Chunking with token limits
print("\n=== Chunking with Token Limits ===")
# Create a large class for demonstration
large_class = '''
class DataProcessor:
"""A class with many methods to demonstrate splitting."""
def __init__(self, config):
self.config = config
self.data = []
self.results = {}
self.cache = {}
self.errors = []
def load_data(self, file_path):
"""Load data from file."""
with Path(file_path).open('r', ) as f:
self.data = json.load(f)
return len(self.data)
def validate_data(self):
"""Validate loaded data."""
errors = []
for i, item in enumerate(self.data):
if not isinstance(item, dict):
errors.append(f"Item {i} is not a dictionary")
if 'id' not in item:
errors.append(f"Item {i} missing 'id' field")
if 'value' not in item:
errors.append(f"Item {i} missing 'value' field")
self.errors = errors
return len(errors) == 0
def process_data(self):
"""Process the validated data."""
for item in self.data:
item_id = item.get('id')
value = item.get('value', 0)
# Complex processing logic
processed = value * 2 + len(str(value))
# Store in results
self.results[item_id] = {
'original': value,
'processed': processed,
'timestamp': 'now'
}
# Cache for future use
self.cache[item_id] = processed
return len(self.results)
def get_summary(self):
"""Get processing summary."""
return {
'total_items': len(self.data),
'processed_items': len(self.results),
'errors': len(self.errors),
'cache_size': len(self.cache)
}
'''
# Write to a temporary file
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
suffix=".py",
delete=False,
) as f:
f.write(large_class)
temp_file = f.name
# Chunk with a small token limit to force splitting
limited_chunks = chunker.chunk_with_token_limit(
temp_file,
"python",
max_tokens=150,
model="gpt-4",
)
print("\nOriginal class would be 1 chunk, but with 150 token limit:")
print(f"Split into {len(limited_chunks)} chunks")
for i, chunk in enumerate(limited_chunks):
is_split = chunk.metadata.get("is_split", False)
print(f"\nChunk {i + 1}:")
print(f" Type: {chunk.node_type}")
print(f" Tokens: {chunk.metadata['token_count']}")
print(f" Is split: {is_split}")
if is_split:
print(f" Split index: {chunk.metadata.get('split_index')}")
print(f" First line: {chunk.content.split(chr(10))[0][:60]}...")
# Example 4: Different models
print("\n=== Different Tokenizer Models ===")
sample = "This is a sample text to show different tokenizer models."
for model in ["gpt-4", "claude", "gpt-3.5-turbo"]:
count = counter.count_tokens(sample, model)
limit = counter.get_token_limit(model)
print(f"{model}: {count} tokens (limit: {limit})")
# Clean up
pathlib.Path(temp_file).unlink()
print("\n=== Token Metadata Structure ===")
if chunks:
print("Sample chunk metadata:")
print(json.dumps(chunks[0].metadata, indent=2))
if __name__ == "__main__":
main()

Xet Storage Details

Size:
5.07 kB
·
Xet hash:
203fb66dff85da95cfb0060ee9d20913a36466356c9c69d8e8c28d9b0a45c091

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.