haru / .meta /create_model.py
yujiepan's picture
Upload folder using huggingface_hub
fca9437 verified
Raw History Blame Contribute Delete
3.1 kB
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
import json
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import save_file
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
hidden_size = 32
num_layers = 8
latent_dim = raw["codec"]["latent_dim"]
text = raw["lm"]
text.update(
hidden_size=hidden_size,
intermediate_size=2 * hidden_size,
num_hidden_layers=num_layers,
num_attention_heads=4,
num_key_value_heads=1,
head_dim=16,
linear_num_key_heads=2,
linear_num_value_heads=4,
linear_key_head_dim=16,
linear_value_head_dim=16,
max_position_embeddings=2048,
tie_word_embeddings=True,
)
# Keep the source 3-linear/1-full cadence for two complete cycles.
text["layer_types"] = list(text["layer_types"][:num_layers])
codec = raw["codec"]
codec.update(
base_channels=1,
encoder_depths=[1] * len(codec["encoder_depths"]),
decoder_depths=[1] * len(codec["decoder_depths"]),
)
raw["head"].update(hidden_size=hidden_size, num_layers=1)
raw["dtype"] = "bfloat16"
from haru.config import HaruConfig
from haru.models.haru import HaruModel
config = HaruConfig.from_dict(raw)
torch.manual_seed(42)
model = HaruModel(config)
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if name.endswith("bias"):
parameter.zero_()
elif "norm" in name and parameter.ndim == 1:
parameter.fill_(1)
elif name.endswith(("gamma", "weight_scale")):
parameter.fill_(1)
else:
# The codec allocates conv weights with torch.empty and never
# initializes them, so give every remaining tensor a deterministic
# value instead of leftover process memory.
parameter.normal_(0, 0.02)
state = {}
for name, tensor in sorted(model.state_dict().items()):
value = tensor.detach().cpu().contiguous()
state[name] = value.to(torch.bfloat16) if value.is_floating_point() else value
assert not any(name.startswith("lm_head") for name in state)
assert state["lm.embed_tokens.weight"].shape == (text["vocab_size"], hidden_size)
save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
(folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
(folder / "tokenizer.json").write_bytes(
Path(hf_hub_download(source_model_id, "tokenizer.json", revision=source_revision)).read_bytes()
)
return str(model)
if __name__ == "__main__":
create_model(
source_model_id="higurashi-ai/Haru-4B",
source_revision="8d6e83bcd3701a76fc27a764965665da1df7da78",
save_folder="/tmp/haru",
)