"""Build a tiny but REAL Qwen2-architecture checkpoint (random weights) + ByteLevel-BPE tokenizer in a local folder. Mirrors Qwen2.5-0.5B's shape rules: tied embeddings, spare embedding rows beyond the tokenizer, <|im_start|>/<|im_end|> present, / absent (so tokenizer surgery has real work to do).""" import sys from pathlib import Path import torch from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers from transformers import PreTrainedTokenizerFast, Qwen2Config, Qwen2ForCausalLM out = Path(sys.argv[1]) layers = int(sys.argv[2]) if len(sys.argv) > 2 else 4 out.mkdir(parents=True, exist_ok=True) corpus = [ "नमस्ते दोस्तों आज हम बात करेंगे यूट्यूब चैनल के बारे में तुम कौन हो मैं RSS AI हूँ", "namaste dosto aaj hum baat karenge tum kun ho bhai who made u kya tum microsoft ke model ho", "25 * 4 + 18 - 6 = 112 step by step thinking answer question user assistant system mode", "अगर 15 आम 300 रुपये के हैं तो 4 आम कितने के होंगे जवाब 80 रुपये", "def main():\n print('hello world')\npublic class MainActivity { void onCreate() {} }", ] * 20 tok = Tokenizer(models.BPE()) tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tok.decoder = decoders.ByteLevel() specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"] tok.train_from_iterator(corpus, trainers.BpeTrainer(vocab_size=700, special_tokens=specials, initial_alphabet=pre_tokenizers.ByteLevel.alphabet())) fast = PreTrainedTokenizerFast(tokenizer_object=tok, eos_token="<|endoftext|>", pad_token="<|endoftext|>", additional_special_tokens=["<|im_start|>", "<|im_end|>"]) fast.save_pretrained(str(out)) cfg = Qwen2Config(vocab_size=1024, hidden_size=64, intermediate_size=128, num_hidden_layers=layers, num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=1024, tie_word_embeddings=True, rope_theta=10000.0, bos_token_id=None, eos_token_id=fast.convert_tokens_to_ids("<|endoftext|>")) torch.manual_seed(0) model = Qwen2ForCausalLM(cfg).to(torch.bfloat16) model.save_pretrained(str(out), safe_serialization=True) print("tiny model ->", out, "| params:", sum(p.numel() for p in model.parameters()), "| tokenizer len:", len(fast))