Text Generation
PEFT
Safetensors
PyTorch
English
causal-lm
lora
qwen
sft
dpo
seal
self-adaptation
hitl
Instructions to use aaronr43210/MiniSEAL with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use aaronr43210/MiniSEAL with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download src/tokenizer/train_bpe.py from aaronr43210/MiniSEAL: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/aaronr43210/MiniSEAL/resolve/main/src/tokenizer/train_bpe.py
- Command line
-
hf download hf://aaronr43210/MiniSEAL/src/tokenizer/train_bpe.py
-
curl -L -o train_bpe.py https://huggingface.co/aaronr43210/MiniSEAL/resolve/main/src/tokenizer/train_bpe.py
3.65 kB
| """ | |
| Phase 1A: Train custom BPE tokenizer. | |
| Research grounding: | |
| - BPE is widely used in modern LMs. | |
| - The goal is to learn subword units from your own corpus. | |
| - This satisfies the "self-owned tokenizer" requirement from the PRD. | |
| Input: | |
| Folder containing .txt files. | |
| Output: | |
| tokenizer.json | |
| tokenizer_config.json | |
| """ | |
| import argparse | |
| from pathlib import Path | |
| import os | |
| import sys | |
| # Add project root to python path so 'src' can be imported | |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) | |
| from tokenizers import Tokenizer | |
| from tokenizers.models import BPE | |
| from tokenizers.normalizers import NFKC, Sequence | |
| from tokenizers.pre_tokenizers import ByteLevel | |
| from tokenizers.decoders import ByteLevel as ByteLevelDecoder | |
| from tokenizers.trainers import BpeTrainer | |
| from src.common.utils import ensure_dir, save_json | |
| SPECIAL_TOKENS = { | |
| "pad_token": "<|pad|>", | |
| "bos_token": "<|bos|>", | |
| "eos_token": "<|eos|>", | |
| "unk_token": "<|unk|>", | |
| } | |
| def collect_text_files(input_dir: str) -> list[str]: | |
| """ | |
| Input: | |
| input_dir: Folder path containing .txt files. | |
| Output: | |
| List of text file paths. | |
| """ | |
| paths = sorted(str(p) for p in Path(input_dir).rglob("*.txt")) | |
| if not paths: | |
| raise FileNotFoundError(f"No .txt files found in {input_dir}") | |
| return paths | |
| def train_bpe_tokenizer( | |
| input_dir: str, | |
| output_dir: str, | |
| vocab_size: int, | |
| min_frequency: int, | |
| ) -> None: | |
| """ | |
| Input: | |
| input_dir: Folder of raw .txt files. | |
| output_dir: Folder where tokenizer files will be saved. | |
| vocab_size: Target vocabulary size. | |
| min_frequency: Minimum token frequency for BPE merges. | |
| Output: | |
| tokenizer.json | |
| tokenizer_config.json | |
| """ | |
| ensure_dir(output_dir) | |
| files = collect_text_files(input_dir) | |
| tokenizer = Tokenizer( | |
| BPE( | |
| unk_token=SPECIAL_TOKENS["unk_token"], | |
| fuse_unk=True, | |
| ) | |
| ) | |
| # NFKC normalizes Unicode text consistently. | |
| tokenizer.normalizer = Sequence([NFKC()]) | |
| # ByteLevel makes the tokenizer robust to unknown characters. | |
| tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False) | |
| tokenizer.decoder = ByteLevelDecoder() | |
| trainer = BpeTrainer( | |
| vocab_size=vocab_size, | |
| min_frequency=min_frequency, | |
| special_tokens=list(SPECIAL_TOKENS.values()), | |
| show_progress=True, | |
| ) | |
| tokenizer.train(files=files, trainer=trainer) | |
| tokenizer_path = Path(output_dir) / "tokenizer.json" | |
| tokenizer.save(str(tokenizer_path)) | |
| config = { | |
| "tokenizer_type": "ByteLevel-BPE", | |
| "vocab_size": tokenizer.get_vocab_size(), | |
| "special_tokens": SPECIAL_TOKENS, | |
| "special_token_ids": { | |
| name: tokenizer.token_to_id(token) | |
| for name, token in SPECIAL_TOKENS.items() | |
| }, | |
| } | |
| save_json(config, Path(output_dir) / "tokenizer_config.json") | |
| print(f"Saved tokenizer to: {tokenizer_path}") | |
| print(f"Vocabulary size: {tokenizer.get_vocab_size()}") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--input_dir", type=str, required=True) | |
| parser.add_argument("--output_dir", type=str, required=True) | |
| parser.add_argument("--vocab_size", type=int, default=32000) | |
| parser.add_argument("--min_frequency", type=int, default=2) | |
| args = parser.parse_args() | |
| train_bpe_tokenizer( | |
| input_dir=args.input_dir, | |
| output_dir=args.output_dir, | |
| vocab_size=args.vocab_size, | |
| min_frequency=args.min_frequency, | |
| ) | |
| if __name__ == "__main__": | |
| main() | |