tiny-llm / src /tinyllm /hf_llama.py
CoolDip00's picture
tiny-llm 75e4579: 6.5b step 5: 5a Space created, 5b secrets added
d656a30 verified
Raw History Blame Contribute Delete
10.1 kB
"""Load a pretrained Hugging Face Llama checkpoint into our Transformer (step 4.9b, D-015 / D-026).
Only our code runs the model. Hugging Face pieces used here are file formats and downloads:
`huggingface_hub` fetches the files, our `safetensors_io` reads the weights, and `tokenizers`
reads `tokenizer.json`. HF Transformers is used only as the logits reference in tests (4.9c).
Our parameter names already follow HF Llama (embed_tokens, layers.N.self_attn.q_proj, ...,
input_layernorm, mlp.gate_proj, norm, lm_head), so the mapping just drops HF's "model." prefix.
RoPE uses the same rotate_half layout as HF, so q/k weights need no permutation.
"""
from __future__ import annotations
import json
from pathlib import Path
import torch
from tinyllm.config import ModelConfig
from tinyllm.model import RotaryEmbedding, Transformer
from tinyllm.safetensors_io import load_file
TINYLLAMA = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
TINYLLAMA_REV = "fe8a4ea1ffedaf415f4da2f062534de366a451e6" # pinned (D-026)
FILES = ["config.json", "generation_config.json", "model.safetensors",
"tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
def config_from_hf(hf: dict) -> ModelConfig:
"""HF LlamaConfig dict -> ModelConfig. Raises on any feature our model doesn't implement."""
unsupported = []
if hf.get("model_type") != "llama":
unsupported.append(f"model_type={hf.get('model_type')!r}")
if hf.get("hidden_act", "silu") != "silu":
unsupported.append(f"hidden_act={hf['hidden_act']!r}")
if hf.get("rope_scaling"):
unsupported.append(f"rope_scaling={hf['rope_scaling']!r}")
# transformers >= 5 writes {"rope_parameters": {"rope_theta": ..., "rope_type": ...}} instead of
# a top-level rope_theta; scaled variants (linear, llama3, yarn, ...) are a rope_type there.
rp = hf.get("rope_parameters") or {}
if rp.get("rope_type", "default") != "default":
unsupported.append(f"rope_parameters={rp!r}")
rope_theta = rp.get("rope_theta", hf.get("rope_theta", 10000.0))
if hf.get("attention_bias") or hf.get("mlp_bias"):
unsupported.append("bias in attention / MLP")
if hf.get("pretraining_tp", 1) != 1:
unsupported.append(f"pretraining_tp={hf['pretraining_tp']}")
hd = hf.get("head_dim")
if hd is not None and hd != hf["hidden_size"] // hf["num_attention_heads"]:
unsupported.append(f"head_dim={hd} != hidden_size / num_attention_heads")
if unsupported:
raise ValueError("unsupported Llama config: " + ", ".join(unsupported))
return ModelConfig(
vocab_size=hf["vocab_size"],
max_seq_len=hf["max_position_embeddings"],
d_model=hf["hidden_size"],
n_layers=hf["num_hidden_layers"],
n_heads=hf["num_attention_heads"],
n_kv_heads=hf.get("num_key_value_heads", hf["num_attention_heads"]),
ffn_hidden=hf["intermediate_size"],
norm_eps=hf["rms_norm_eps"],
rope_theta=rope_theta,
init_std=hf.get("initializer_range", 0.02),
tie_embeddings=hf.get("tie_word_embeddings", False),
)
def hf_to_ours(name: str) -> str:
return name.removeprefix("model.")
def ours_to_hf(name: str) -> str:
return name if name == "lm_head.weight" else "model." + name
def download(repo: str = TINYLLAMA, revision: str = TINYLLAMA_REV, local_only: bool = False) -> Path:
"""Snapshot directory in the HF cache (~/.cache/huggingface). Weights never go into the repo."""
from huggingface_hub import snapshot_download
return Path(snapshot_download(repo, revision=revision, allow_patterns=FILES, local_files_only=local_only))
def load_from_dir(path: str | Path, device: str = "cuda", dtype: torch.dtype = torch.bfloat16) -> Transformer:
"""Build our Transformer from <path>/config.json and <path>/model.safetensors, strict.
The model is created on the meta device, so no fp32 copy is ever allocated (TinyLlama: 4.4 GB);
each weight is copied once from the memory-mapped file to `device` in `dtype` (D-023: bf16).
"""
path = Path(path)
cfg = config_from_hf(json.loads((path / "config.json").read_text()))
state = {hf_to_ours(k): v for k, v in load_file(path / "model.safetensors", device, dtype).items()}
# Rotary inv_freq is a buffer in some HF exports; ours are recomputed and never stored.
state = {k: v for k, v in state.items() if not k.endswith("rotary_emb.inv_freq")}
if cfg.tie_embeddings:
state.pop("lm_head.weight", None) # tied: the embedding is the head
with torch.device("meta"):
model = Transformer(cfg)
expected = {k: v for k, v in model.state_dict().items() if not (cfg.tie_embeddings and k == "lm_head.weight")}
missing, unexpected = sorted(set(expected) - set(state)), sorted(set(state) - set(expected))
bad_shape = [k for k in expected.keys() & state.keys() if tuple(expected[k].shape) != tuple(state[k].shape)]
if missing or unexpected or bad_shape:
raise ValueError(f"checkpoint mismatch: missing {missing[:5]}, unexpected {unexpected[:5]}, "
f"shape {bad_shape[:5]}")
model.load_state_dict(state, strict=False, assign=True) # strictness checked above (tie aware)
if cfg.tie_embeddings:
model.lm_head.weight = model.embed_tokens.weight
# Non-persistent buffers were created on meta: rebuild the RoPE tables on the real device.
model.rope = RotaryEmbedding(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta).to(device)
return model.eval()
def load_hf_llama(repo: str = TINYLLAMA, revision: str = TINYLLAMA_REV, device: str = "cuda",
dtype: torch.dtype = torch.bfloat16, local_only: bool = False) -> Transformer:
return load_from_dir(download(repo, revision, local_only), device, dtype)
def save_as_hf(model: Transformer, path: str | Path, eos_id: int | None = None, bos_id: int | None = None) -> None:
"""Write our model as an HF-style Llama directory (config.json, generation_config.json,
model.safetensors). Used by tests and 4.8b.
The token ids are written explicitly, even as null: a config without them gets LlamaConfig's
defaults (bos 1, eos 2), and HF generate() would then stop on the wrong token.
"""
from tinyllm.safetensors_io import save_file
c = model.cfg
other = sorted(set(c.layer_types) - {"mha", "gqa"})
if other:
raise ValueError(f"only mha / gqa layers map to Llama; this model has {other}")
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
sd = model.state_dict()
if c.tie_embeddings:
sd.pop("lm_head.weight")
dtype = str(next(iter(sd.values())).dtype).removeprefix("torch.")
ids = {"bos_token_id": bos_id, "eos_token_id": eos_id, "pad_token_id": None}
hf = {"architectures": ["LlamaForCausalLM"], "model_type": "llama", "hidden_act": "silu",
"vocab_size": c.vocab_size, "max_position_embeddings": c.max_seq_len, "hidden_size": c.d_model,
"num_hidden_layers": c.n_layers, "num_attention_heads": c.n_heads,
"num_key_value_heads": c.n_kv_heads, "intermediate_size": c.ffn_hidden, "rms_norm_eps": c.norm_eps,
"rope_theta": c.rope_theta, "initializer_range": c.init_std, "tie_word_embeddings": c.tie_embeddings,
"attention_bias": False, "rope_scaling": None, "dtype": dtype, **ids}
(path / "config.json").write_text(json.dumps(hf, indent=2))
# Padding a batch with EOS (GPT-2 style); our model has no pad token.
(path / "generation_config.json").write_text(json.dumps({**ids, "pad_token_id": eos_id}, indent=2))
save_file({ours_to_hf(k): v for k, v in sd.items()}, path / "model.safetensors", {"format": "pt"})
def save_tokenizer_as_hf(path: str | Path, model_max_length: int | None = None) -> None:
"""Our TinyStories BPE as an HF tokenizer directory (4.8b).
It was trained with the `tokenizers` library (D-007), so the file already is a tokenizer.json;
tokenizer_config.json only tells `AutoTokenizer` which class to use and which token is EOS.
Like our server: no BOS, nothing added on encode (the file has no post-processor), and no
"clean up" of spaces on decode, so decode(encode(text)) == text.
"""
import shutil
from tinyllm.tokenizer import EOS, TOKENIZER_PATH
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
shutil.copyfile(TOKENIZER_PATH, path / "tokenizer.json")
cfg = {"tokenizer_class": "PreTrainedTokenizerFast", "eos_token": EOS, "bos_token": None,
"unk_token": None, "pad_token": EOS, "add_bos_token": False, "add_eos_token": False,
"clean_up_tokenization_spaces": False, "model_max_length": model_max_length}
(path / "tokenizer_config.json").write_text(json.dumps(cfg, indent=2))
# --- tokenizer -------------------------------------------------------------------------------------
class HFTokenizer:
"""tokenizer.json via the `tokenizers` library. encode() adds BOS (<s>) like the Llama tokenizer."""
def __init__(self, path: str | Path):
from tokenizers import Tokenizer
path = Path(path)
self.tok = Tokenizer.from_file(str(path / "tokenizer.json"))
cfg = json.loads((path / "tokenizer_config.json").read_text()) if (path / "tokenizer_config.json").exists() else {}
self.chat_template = cfg.get("chat_template")
self.bos_id = self.tok.token_to_id("<s>")
self.eos_id = self.tok.token_to_id("</s>")
def encode(self, text: str, bos: bool = True) -> list[int]:
ids = self.tok.encode(text, add_special_tokens=False).ids
return ([self.bos_id] if bos else []) + ids
def decode(self, ids: list[int]) -> str:
return self.tok.decode(ids, skip_special_tokens=True)
@staticmethod
def chat_prompt(user: str, system: str | None = None) -> str:
"""TinyLlama-Chat v1.0 (Zephyr) template, with the assistant turn opened for generation."""
sys = f"<|system|>\n{system}</s>\n" if system else ""
return f"{sys}<|user|>\n{user}</s>\n<|assistant|>\n"