wiki-model-tiny-GGUF / convert_tiny.py
Lucasgitguy-co's picture
Add GGUF conversion (tested with llama.cpp build 12359)
bb16b1f verified
Raw History Blame Contribute Delete
3.47 kB
"""Write wiki-model-tiny (TinyGPT) as a llama-arch GGUF directly with gguf-py.
TinyGPT already uses interleaved (GPT-J style) RoPE pairs, which is llama.cpp's LLAMA rope, so q/k are NOT permuted.
usage: python convert_tiny.py MODEL_DIR OUT.gguf [f32|f16]"""
import json, sys
import numpy as np
from safetensors.numpy import load_file
sys.path.insert(0, sys.argv[4] if len(sys.argv) > 4 else "/Users/lucasvandeweerd/.cache/spot-snake/llama.cpp/gguf-py")
import gguf
mdir, out = sys.argv[1], sys.argv[2]
ft = sys.argv[3] if len(sys.argv) > 3 else "f16"
c = json.load(open(f"{mdir}/config.json"))
sd = load_file(f"{mdir}/model.safetensors")
tj = json.load(open(f"{mdir}/tokenizer.json"))
D, H, L, F = c["dim"], c["heads"], c["layers"], c["hidden"]
w = gguf.GGUFWriter(out, "llama")
w.add_name("wiki-model-tiny")
w.add_context_length(c["ctx"]); w.add_embedding_length(D); w.add_block_count(L)
w.add_feed_forward_length(F); w.add_head_count(H); w.add_head_count_kv(H)
w.add_layer_norm_rms_eps(1e-5); w.add_rope_freq_base(c["rope_base"]); w.add_rope_dimension_count(D // H)
w.add_vocab_size(c["vocab"])
w.add_file_type(gguf.LlamaFileType.MOSTLY_F16 if ft == "f16" else gguf.LlamaFileType.ALL_F32)
# tokenizer: byte-level BPE with the GPT-2 regex == llama.cpp "gpt2" model + "gpt-2" pre-tokenizer
vocab = tj["model"]["vocab"]
added = {a["id"]: a for a in tj["added_tokens"]}
tokens = [None] * c["vocab"]
for t, i in vocab.items(): tokens[i] = t
for i, a in added.items(): tokens[i] = a["content"]
assert all(t is not None for t in tokens), "vocab has holes"
types = []
for i in range(len(tokens)):
if i in added:
types.append(gguf.TokenType.CONTROL if added[i]["content"] in ("<pad>", "<eos>") else gguf.TokenType.USER_DEFINED)
else:
types.append(gguf.TokenType.NORMAL)
merges = [m if isinstance(m, str) else " ".join(m) for m in tj["model"]["merges"]]
eos = tj["model"]["vocab"].get("<eos>", next(i for i, a in added.items() if a["content"] == "<eos>"))
pad = next(i for i, a in added.items() if a["content"] == "<pad>")
w.add_tokenizer_model("gpt2"); w.add_tokenizer_pre("gpt-2")
w.add_token_list(tokens); w.add_token_types(types); w.add_token_merges(merges)
w.add_bos_token_id(eos); w.add_eos_token_id(eos); w.add_pad_token_id(pad)
w.add_add_bos_token(False); w.add_add_eos_token(False)
def put(name, a, norm=False):
a = np.ascontiguousarray(a, dtype=np.float32 if (norm or ft == "f32") else np.float16)
w.add_tensor(name, a)
put("token_embd.weight", sd["emb.weight"])
for i in range(L):
p = f"blocks.{i}."
q, k, v = np.split(sd[p + "qkv.weight"], 3, axis=0)
g, u = np.split(sd[p + "gate_up.weight"], 2, axis=0) # chunk(2): gate first, then up
put(f"blk.{i}.attn_norm.weight", sd[p + "n1.w"], norm=True)
put(f"blk.{i}.attn_q.weight", q); put(f"blk.{i}.attn_k.weight", k); put(f"blk.{i}.attn_v.weight", v)
put(f"blk.{i}.attn_output.weight", sd[p + "o.weight"])
put(f"blk.{i}.ffn_norm.weight", sd[p + "n2.w"], norm=True)
put(f"blk.{i}.ffn_gate.weight", g); put(f"blk.{i}.ffn_up.weight", u)
put(f"blk.{i}.ffn_down.weight", sd[p + "down.weight"])
put("output_norm.weight", sd["norm.w"], norm=True)
used = {"emb.weight", "norm.w"} | {f"blocks.{i}.{n}" for i in range(L) for n in ("qkv.weight", "gate_up.weight", "n1.w", "n2.w", "o.weight", "down.weight")}
assert used == set(sd), set(sd) ^ used
w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()
print("wrote", out)