| import sentencepiece as spm | |
| print("🚀 Training tokenizer...") | |
| spm.SentencePieceTrainer.train( | |
| input='sangraha_hindi_final_dataset.txt', | |
| model_prefix='hindi_tokenizer', | |
| vocab_size=16000, | |
| model_type='bpe', | |
| character_coverage=1.0, | |
| pad_id=0, | |
| unk_id=1, | |
| bos_id=2, | |
| eos_id=3 | |
| ) | |
| print("✅ Tokenizer retrained successfully") | |