import sentencepiece as spm

print("🚀 Training tokenizer...")

spm.SentencePieceTrainer.train(
    input='sangraha_hindi_final_dataset.txt',
    model_prefix='hindi_tokenizer',
    vocab_size=16000,
    model_type='bpe',
    character_coverage=1.0,
    pad_id=0,
    unk_id=1,
    bos_id=2,
    eos_id=3
)

print("✅ Tokenizer retrained successfully")