tokenizer/__init__.py

"""tokenizer package — BPE tokenizer and training utilities."""

from tokenizer.bpe_tokenizer import BPETokenizer

__all__ = ["BPETokenizer"]