File size: 6,688 Bytes
7500ff7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | #!/usr/bin/env python3
"""
Benchmark tokenizer performance for Quillan-Ronin.
Tests BPE tokenizer speed and compares against alternatives.
"""
import sys
import time
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT / '_dev'))
# Test text samples
test_texts = [
"The future of artificial intelligence is bright and promising.",
"Once upon a time, in a land far away, there lived a brave knight.",
"In a world where technology advances rapidly, adaptation is key.",
"The most important lesson I learned is that perseverance pays off.",
"Machine learning models require large datasets to generalize well.",
"Python is a popular programming language for data science and AI.",
"The model architecture uses mixture of experts for efficiency.",
"Quantum computing promises to revolutionize cryptographic systems.",
"Natural language processing has made significant progress recently.",
"Deep learning networks can learn hierarchical representations."
]
def benchmark_bpe():
"""Benchmark BPE tokenizer."""
print("=== BPE Tokenizer Benchmark ===")
try:
from quillan_bpe_tokenizer import QuillanBPETokenizer
tokenizer = QuillanBPETokenizer()
tok_path = ROOT / 'training_data' / 'tokenizer.json'
if not tok_path.exists():
tok_path = ROOT / '_dev' / 'quillan_bpe_tokenizer_hf' / 'tokenizer.json'
tokenizer.load(str(tok_path))
print(f"Vocab size: {tokenizer.vocab_size}")
# Warmup
for text in test_texts[:2]:
tokenizer.encode(text)
# Benchmark encode
encode_times = []
for text in test_texts:
start = time.time()
tokens = tokenizer.encode(text)
elapsed = time.time() - start
encode_times.append(elapsed)
# Benchmark decode
decode_times = []
for text in test_texts:
tokens = tokenizer.encode(text)
start = time.time()
decoded = tokenizer.decode(tokens)
elapsed = time.time() - start
decode_times.append(elapsed)
avg_encode = sum(encode_times) / len(encode_times)
avg_decode = sum(decode_times) / len(decode_times)
print(f"Average encode time: {avg_encode*1000:.2f} ms")
print(f"Average decode time: {avg_decode*1000:.2f} ms")
print(f"Total (encode+decode): {(avg_encode+avg_decode)*1000:.2f} ms")
return avg_encode, avg_decode
except Exception as e:
print(f"Error benchmarking BPE: {e}")
return None, None
def benchmark_sentencepiece():
"""Benchmark SentencePiece tokenizer."""
print("\n=== SentencePiece Tokenizer Benchmark ===")
try:
import sentencepiece as spm
sp = spm.SentencePieceProcessor()
sp_model = ROOT / 'training_data' / 'tokenizer.model'
if sp_model.exists():
sp.load(str(sp_model))
print(f"Vocab size: {sp.get_piece_size()}")
# Warmup
for text in test_texts[:2]:
sp.encode(text)
# Benchmark encode
encode_times = []
for text in test_texts:
start = time.time()
tokens = sp.encode(text)
elapsed = time.time() - start
encode_times.append(elapsed)
# Benchmark decode
decode_times = []
for text in test_texts:
tokens = sp.encode(text)
start = time.time()
decoded = sp.decode(tokens)
elapsed = time.time() - start
decode_times.append(elapsed)
avg_encode = sum(encode_times) / len(encode_times)
avg_decode = sum(decode_times) / len(decode_times)
print(f"Average encode time: {avg_encode*1000:.2f} ms")
print(f"Average decode time: {avg_decode*1000:.2f} ms")
print(f"Total (encode+decode): {(avg_encode+avg_decode)*1000:.2f} ms")
return avg_encode, avg_decode
else:
print("SentencePiece model not found")
return None, None
except ImportError:
print("sentencepiece not installed")
return None, None
except Exception as e:
print(f"Error benchmarking SentencePiece: {e}")
return None, None
def benchmark_gpt2():
"""Benchmark GPT-2 tokenizer."""
print("\n=== GPT-2 Tokenizer Benchmark ===")
try:
from transformers import GPT2Tokenizer
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
print(f"Vocab size: {tokenizer.vocab_size}")
# Warmup
for text in test_texts[:2]:
tokenizer.encode(text)
# Benchmark encode
encode_times = []
for text in test_texts:
start = time.time()
tokens = tokenizer.encode(text)
elapsed = time.time() - start
encode_times.append(elapsed)
# Benchmark decode
decode_times = []
for text in test_texts:
tokens = tokenizer.encode(text)
start = time.time()
decoded = tokenizer.decode(tokens)
elapsed = time.time() - start
decode_times.append(elapsed)
avg_encode = sum(encode_times) / len(encode_times)
avg_decode = sum(decode_times) / len(decode_times)
print(f"Average encode time: {avg_encode*1000:.2f} ms")
print(f"Average decode time: {avg_decode*1000:.2f} ms")
print(f"Total (encode+decode): {(avg_encode+avg_decode)*1000:.2f} ms")
return avg_encode, avg_decode
except ImportError:
print("transformers not installed")
return None, None
except Exception as e:
print(f"Error benchmarking GPT-2: {e}")
return None, None
if __name__ == '__main__':
bpe_enc, bpe_dec = benchmark_bpe()
sp_enc, sp_dec = benchmark_sentencepiece()
gpt2_enc, gpt2_dec = benchmark_gpt2()
print("\n=== Summary ===")
print("Tokenizer | Encode (ms) | Decode (ms) | Total (ms)")
print("-" * 50)
if bpe_enc is not None:
print(f"BPE | {bpe_enc*1000:10.2f} | {bpe_dec*1000:10.2f} | {(bpe_enc+bpe_dec)*1000:10.2f}")
if sp_enc is not None:
print(f"SentencePiece | {(sp_enc)*1000:8.2f} | {sp_dec*1000:10.2f} | {(sp_enc+sp_dec)*1000:10.2f}")
if gpt2_enc is not None:
print(f"GPT-2 | {gpt2_enc*1000:10.2f} | {gpt2_dec*1000:10.2f} | {(gpt2_enc+gpt2_dec)*1000:10.2f}")
|