RRas-ai-engine / make_tiny_qwen.py
Biraboro's picture
Upload folder using huggingface_hub
9d614f1 verified
Raw History Blame Contribute Delete
2.53 kB
"""Build a tiny but REAL Qwen2-architecture checkpoint (random weights) + ByteLevel-BPE tokenizer in a local folder.
Mirrors Qwen2.5-0.5B's shape rules: tied embeddings, spare embedding rows beyond the tokenizer, <|im_start|>/<|im_end|> present,
<think>/</think> absent (so tokenizer surgery has real work to do)."""
import sys
from pathlib import Path
import torch
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers
from transformers import PreTrainedTokenizerFast, Qwen2Config, Qwen2ForCausalLM
out = Path(sys.argv[1])
layers = int(sys.argv[2]) if len(sys.argv) > 2 else 4
out.mkdir(parents=True, exist_ok=True)
corpus = [
"नमस्ते दोस्तों आज हम बात करेंगे यूट्यूब चैनल के बारे में तुम कौन हो मैं RSS AI हूँ",
"namaste dosto aaj hum baat karenge tum kun ho bhai who made u kya tum microsoft ke model ho",
"25 * 4 + 18 - 6 = 112 step by step thinking answer question user assistant system mode",
"अगर 15 आम 300 रुपये के हैं तो 4 आम कितने के होंगे जवाब 80 रुपये",
"def main():\n print('hello world')\npublic class MainActivity { void onCreate() {} }",
] * 20
tok = Tokenizer(models.BPE())
tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tok.decoder = decoders.ByteLevel()
specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
tok.train_from_iterator(corpus, trainers.BpeTrainer(vocab_size=700, special_tokens=specials,
initial_alphabet=pre_tokenizers.ByteLevel.alphabet()))
fast = PreTrainedTokenizerFast(tokenizer_object=tok, eos_token="<|endoftext|>", pad_token="<|endoftext|>",
additional_special_tokens=["<|im_start|>", "<|im_end|>"])
fast.save_pretrained(str(out))
cfg = Qwen2Config(vocab_size=1024, hidden_size=64, intermediate_size=128, num_hidden_layers=layers,
num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=1024,
tie_word_embeddings=True, rope_theta=10000.0, bos_token_id=None,
eos_token_id=fast.convert_tokens_to_ids("<|endoftext|>"))
torch.manual_seed(0)
model = Qwen2ForCausalLM(cfg).to(torch.bfloat16)
model.save_pretrained(str(out), safe_serialization=True)
print("tiny model ->", out, "| params:", sum(p.numel() for p in model.parameters()), "| tokenizer len:", len(fast))