Download tokenizer/wrap_tokenizer.py from geeteshcodes/sllm: direct link, hf CLI and curl.
- Browser
- Download file 8.13 kB
-
https://huggingface.co/geeteshcodes/sllm/resolve/main/tokenizer/wrap_tokenizer.py
- Command line
-
hf download hf://geeteshcodes/sllm/tokenizer/wrap_tokenizer.py
-
curl -L -o wrap_tokenizer.py https://huggingface.co/geeteshcodes/sllm/resolve/main/tokenizer/wrap_tokenizer.py
8.13 kB
| from tokenizers import Tokenizer | |
| from transformers import PreTrainedTokenizerFast | |
| import json | |
| import os | |
| # ------------------------------------------------------------------ # | |
| # CONSTANTS | |
| # ------------------------------------------------------------------ # | |
| import os | |
| SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| TOKENIZER_PATH = os.path.join(SCRIPT_DIR, "fineweb_edu_tokenizer.json") | |
| SAVE_DIR = os.path.join(SCRIPT_DIR, "fineweb_edu_tokenizer") # output folder | |
| MODEL_MAX_LENGTH = 1024 # context length | |
| PADDING_SIDE = "right" # causal LM standard | |
| # ------------------------------------------------------------------ # | |
| # WRAP | |
| # ------------------------------------------------------------------ # | |
| def wrap_tokenizer( | |
| tokenizer_path: str = TOKENIZER_PATH, | |
| save_dir: str = SAVE_DIR, | |
| ) -> PreTrainedTokenizerFast: | |
| """ | |
| Wraps a trained HuggingFace Tokenizer as a PreTrainedTokenizerFast. | |
| This gives us: | |
| - datasets.map() compatibility for bulk tokenization | |
| - HuggingFace Trainer + DataCollator compatibility | |
| - Automatic padding, truncation, attention masks | |
| - from_pretrained() loading support | |
| - return_tensors="pt" for PyTorch tensors | |
| Args: | |
| tokenizer_path : path to trained tokenizer .json file | |
| save_dir : folder to save the wrapped tokenizer | |
| Returns: | |
| PreTrainedTokenizerFast ready for training | |
| """ | |
| print(f"Loading trained tokenizer from: {tokenizer_path}") | |
| base_tokenizer = Tokenizer.from_file(tokenizer_path) | |
| # ---- Wrap -------------------------------------------------------- | |
| # We map <|endoftext|> to all three roles: | |
| # | |
| # eos_token - end of sequence marker, used during generation | |
| # to know when to stop | |
| # | |
| # bos_token - beginning of sequence, GPT-2 style uses eos | |
| # for both since there is no separate BOS token | |
| # | |
| # pad_token - safe to reuse eos here because we are packing | |
| # sequences and will never actually pad during | |
| # pretraining. Defined so HuggingFace doesn't | |
| # complain about missing pad token | |
| # | |
| # unk_token - None because byte-level means no unknowns ever | |
| tokenizer = PreTrainedTokenizerFast( | |
| tokenizer_object=base_tokenizer, | |
| # Special token mappings | |
| eos_token="<|endoftext|>", | |
| bos_token="<|endoftext|>", | |
| pad_token="<|endoftext|>", | |
| unk_token=None, | |
| # Context length | |
| model_max_length=MODEL_MAX_LENGTH, | |
| # Padding behavior | |
| padding_side=PADDING_SIDE, | |
| # Truncation side - truncate from the right | |
| # (keep the beginning of the sequence, drop the end) | |
| truncation_side="right", | |
| ) | |
| tokenizer.add_special_tokens({ | |
| "eos_token": "<|endoftext|>", | |
| "bos_token": "<|endoftext|>", | |
| "pad_token": "<|endoftext|>", | |
| }) | |
| special_tokens_map = { | |
| "bos_token": "<|endoftext|>", | |
| "eos_token": "<|endoftext|>", | |
| "pad_token": "<|endoftext|>", | |
| } | |
| os.makedirs(save_dir, exist_ok=True) | |
| with open(os.path.join(save_dir, "special_tokens_map.json"), "w") as f: | |
| json.dump(special_tokens_map, f, indent=2) | |
| print("special_tokens_map.json written manually") | |
| # ---- Save -------------------------------------------------------- | |
| # Saves three files to save_dir/: | |
| # tokenizer.json - the trained BPE tokenizer | |
| # tokenizer_config.json - max length, pad token, special tokens | |
| # special_tokens_map.json - maps eos/bos/pad to actual tokens | |
| tokenizer.save_pretrained(save_dir) | |
| print(f"Tokenizer saved to: {save_dir}/") | |
| print(f" tokenizer.json") | |
| print(f" tokenizer_config.json") | |
| print(f" special_tokens_map.json") | |
| return tokenizer | |
| # ------------------------------------------------------------------ # | |
| # VERIFICATION | |
| # ------------------------------------------------------------------ # | |
| def verify_wrapped_tokenizer(tokenizer: PreTrainedTokenizerFast): | |
| """ | |
| Verifies the wrapped tokenizer behaves correctly. | |
| Tests encoding, decoding, padding, truncation and batch encoding. | |
| """ | |
| print("\n" + "="*60) | |
| print(" WRAPPED TOKENIZER VERIFICATION") | |
| print("="*60 + "\n") | |
| eot_id = tokenizer.eos_token_id | |
| # ---- 1. Basic config ----------------------------------------- | |
| print("Config:") | |
| print(f" vocab size : {tokenizer.vocab_size:,}") | |
| print(f" model_max_length : {tokenizer.model_max_length}") | |
| print(f" padding_side : {tokenizer.padding_side}") | |
| print(f" eos_token : {tokenizer.eos_token!r} (ID: {eot_id})") | |
| print(f" bos_token : {tokenizer.bos_token!r}") | |
| print(f" pad_token : {tokenizer.pad_token!r} (ID: {tokenizer.pad_token_id})") | |
| print(f" unk_token : {tokenizer.unk_token!r}") | |
| print() | |
| # ---- 2. Basic encode/decode ---------------------------------- | |
| text = "The mitochondria is the powerhouse of the cell." | |
| encoded = tokenizer(text) | |
| decoded = tokenizer.decode(encoded["input_ids"]) | |
| print("Basic encode/decode:") | |
| print(f" input : {repr(text)}") | |
| print(f" input_ids: {encoded['input_ids']}") | |
| print(f" decoded : {repr(decoded)}") | |
| print() | |
| # ---- 3. Padding ---------------------------------------------- | |
| # Batch of two sequences with different lengths | |
| # shorter one should be right-padded to match the longer | |
| batch = [ | |
| "Short sentence.", | |
| "This is a much longer sentence that has more tokens in it.", | |
| ] | |
| encoded_batch = tokenizer( | |
| batch, | |
| padding=True, # pad to longest in batch | |
| return_tensors="pt", # return PyTorch tensors | |
| ) | |
| print("Batch padding (right padding):") | |
| print(f" input_ids shape : {encoded_batch['input_ids'].shape}") | |
| print(f" attention_mask shape : {encoded_batch['attention_mask'].shape}") | |
| print(f" input_ids[0] : {encoded_batch['input_ids'][0].tolist()}") | |
| print(f" input_ids[1] : {encoded_batch['input_ids'][1].tolist()}") | |
| print(f" attention_mask[0] : {encoded_batch['attention_mask'][0].tolist()}") | |
| print() | |
| # ---- 4. Truncation ------------------------------------------- | |
| # Sequence longer than model_max_length should be truncated | |
| long_text = "word " * 2000 # 2000 words >> 1024 tokens | |
| encoded_long = tokenizer( | |
| long_text, | |
| truncation=True, | |
| max_length=MODEL_MAX_LENGTH, | |
| ) | |
| print("Truncation:") | |
| print(f" input length : {len(long_text.split())} words") | |
| print(f" token count : {len(encoded_long['input_ids'])} (max: {MODEL_MAX_LENGTH})") | |
| print(f" truncated : {len(encoded_long['input_ids']) <= MODEL_MAX_LENGTH}") | |
| print() | |
| # ---- 5. Load from disk and verify ---------------------------- | |
| print("Loading from disk:") | |
| reloaded = PreTrainedTokenizerFast.from_pretrained(SAVE_DIR) | |
| reloaded_ids = reloaded(text)["input_ids"] | |
| original_ids = encoded["input_ids"] | |
| match = reloaded_ids == original_ids | |
| print(f" from_pretrained() : OK") | |
| print(f" IDs match original: {match}") | |
| # ------------------------------------------------------------------ # | |
| # ENTRY POINT | |
| # ------------------------------------------------------------------ # | |
| if __name__ == "__main__": | |
| tokenizer = wrap_tokenizer() | |
| verify_wrapped_tokenizer(tokenizer) | |
| print("\n" + "="*60) | |
| print(" USAGE EXAMPLES") | |
| print("="*60) | |
| print(""" | |
| # Load anywhere with one line | |
| from transformers import PreTrainedTokenizerFast | |
| tokenizer = PreTrainedTokenizerFast.from_pretrained("fineweb_edu_tokenizer") | |
| # Single encode | |
| ids = tokenizer("Hello world")["input_ids"] | |
| # Batch encode with padding and tensors | |
| batch = tokenizer( | |
| ["sentence one", "sentence two"], | |
| padding=True, | |
| truncation=True, | |
| max_length=1024, | |
| return_tensors="pt", | |
| ) | |
| # Decode | |
| text = tokenizer.decode(ids, skip_special_tokens=True) | |
| # Get eos token id (use as document separator when packing) | |
| eot_id = tokenizer.eos_token_id | |
| """) | |