Download .meta/create_model.py from tiny-random/haru: direct link, hf CLI and curl.
- Browser
- Download file 3.1 kB
-
https://huggingface.co/tiny-random/haru/resolve/main/.meta/create_model.py
- Command line
-
hf download hf://tiny-random/haru/.meta/create_model.py
-
curl -L -o create_model.py https://huggingface.co/tiny-random/haru/resolve/main/.meta/create_model.py
3.1 kB
| def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str: | |
| import json | |
| from pathlib import Path | |
| import torch | |
| from huggingface_hub import hf_hub_download | |
| from safetensors.torch import save_file | |
| folder = Path(save_folder) | |
| folder.mkdir(parents=True, exist_ok=True) | |
| raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()) | |
| hidden_size = 32 | |
| num_layers = 8 | |
| latent_dim = raw["codec"]["latent_dim"] | |
| text = raw["lm"] | |
| text.update( | |
| hidden_size=hidden_size, | |
| intermediate_size=2 * hidden_size, | |
| num_hidden_layers=num_layers, | |
| num_attention_heads=4, | |
| num_key_value_heads=1, | |
| head_dim=16, | |
| linear_num_key_heads=2, | |
| linear_num_value_heads=4, | |
| linear_key_head_dim=16, | |
| linear_value_head_dim=16, | |
| max_position_embeddings=2048, | |
| tie_word_embeddings=True, | |
| ) | |
| # Keep the source 3-linear/1-full cadence for two complete cycles. | |
| text["layer_types"] = list(text["layer_types"][:num_layers]) | |
| codec = raw["codec"] | |
| codec.update( | |
| base_channels=1, | |
| encoder_depths=[1] * len(codec["encoder_depths"]), | |
| decoder_depths=[1] * len(codec["decoder_depths"]), | |
| ) | |
| raw["head"].update(hidden_size=hidden_size, num_layers=1) | |
| raw["dtype"] = "bfloat16" | |
| from haru.config import HaruConfig | |
| from haru.models.haru import HaruModel | |
| config = HaruConfig.from_dict(raw) | |
| torch.manual_seed(42) | |
| model = HaruModel(config) | |
| with torch.no_grad(): | |
| for name, parameter in sorted(model.named_parameters()): | |
| if name.endswith("bias"): | |
| parameter.zero_() | |
| elif "norm" in name and parameter.ndim == 1: | |
| parameter.fill_(1) | |
| elif name.endswith(("gamma", "weight_scale")): | |
| parameter.fill_(1) | |
| else: | |
| # The codec allocates conv weights with torch.empty and never | |
| # initializes them, so give every remaining tensor a deterministic | |
| # value instead of leftover process memory. | |
| parameter.normal_(0, 0.02) | |
| state = {} | |
| for name, tensor in sorted(model.state_dict().items()): | |
| value = tensor.detach().cpu().contiguous() | |
| state[name] = value.to(torch.bfloat16) if value.is_floating_point() else value | |
| assert not any(name.startswith("lm_head") for name in state) | |
| assert state["lm.embed_tokens.weight"].shape == (text["vocab_size"], hidden_size) | |
| save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"}) | |
| (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n") | |
| (folder / "tokenizer.json").write_bytes( | |
| Path(hf_hub_download(source_model_id, "tokenizer.json", revision=source_revision)).read_bytes() | |
| ) | |
| return str(model) | |
| if __name__ == "__main__": | |
| create_model( | |
| source_model_id="higurashi-ai/Haru-4B", | |
| source_revision="8d6e83bcd3701a76fc27a764965665da1df7da78", | |
| save_folder="/tmp/haru", | |
| ) | |