Download make_tiny_qwen.py from Biraboro/RRas-ai-engine: direct link, hf CLI and curl.
- Browser
- Download file 2.53 kB
-
https://huggingface.co/Biraboro/RRas-ai-engine/resolve/main/make_tiny_qwen.py
- Command line
-
hf download hf://Biraboro/RRas-ai-engine/make_tiny_qwen.py
-
curl -L -o make_tiny_qwen.py https://huggingface.co/Biraboro/RRas-ai-engine/resolve/main/make_tiny_qwen.py
2.53 kB
| """Build a tiny but REAL Qwen2-architecture checkpoint (random weights) + ByteLevel-BPE tokenizer in a local folder. | |
| Mirrors Qwen2.5-0.5B's shape rules: tied embeddings, spare embedding rows beyond the tokenizer, <|im_start|>/<|im_end|> present, | |
| <think>/</think> absent (so tokenizer surgery has real work to do).""" | |
| import sys | |
| from pathlib import Path | |
| import torch | |
| from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers | |
| from transformers import PreTrainedTokenizerFast, Qwen2Config, Qwen2ForCausalLM | |
| out = Path(sys.argv[1]) | |
| layers = int(sys.argv[2]) if len(sys.argv) > 2 else 4 | |
| out.mkdir(parents=True, exist_ok=True) | |
| corpus = [ | |
| "नमस्ते दोस्तों आज हम बात करेंगे यूट्यूब चैनल के बारे में तुम कौन हो मैं RSS AI हूँ", | |
| "namaste dosto aaj hum baat karenge tum kun ho bhai who made u kya tum microsoft ke model ho", | |
| "25 * 4 + 18 - 6 = 112 step by step thinking answer question user assistant system mode", | |
| "अगर 15 आम 300 रुपये के हैं तो 4 आम कितने के होंगे जवाब 80 रुपये", | |
| "def main():\n print('hello world')\npublic class MainActivity { void onCreate() {} }", | |
| ] * 20 | |
| tok = Tokenizer(models.BPE()) | |
| tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) | |
| tok.decoder = decoders.ByteLevel() | |
| specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"] | |
| tok.train_from_iterator(corpus, trainers.BpeTrainer(vocab_size=700, special_tokens=specials, | |
| initial_alphabet=pre_tokenizers.ByteLevel.alphabet())) | |
| fast = PreTrainedTokenizerFast(tokenizer_object=tok, eos_token="<|endoftext|>", pad_token="<|endoftext|>", | |
| additional_special_tokens=["<|im_start|>", "<|im_end|>"]) | |
| fast.save_pretrained(str(out)) | |
| cfg = Qwen2Config(vocab_size=1024, hidden_size=64, intermediate_size=128, num_hidden_layers=layers, | |
| num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=1024, | |
| tie_word_embeddings=True, rope_theta=10000.0, bos_token_id=None, | |
| eos_token_id=fast.convert_tokens_to_ids("<|endoftext|>")) | |
| torch.manual_seed(0) | |
| model = Qwen2ForCausalLM(cfg).to(torch.bfloat16) | |
| model.save_pretrained(str(out), safe_serialization=True) | |
| print("tiny model ->", out, "| params:", sum(p.numel() for p in model.parameters()), "| tokenizer len:", len(fast)) | |