def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str: import json from pathlib import Path import torch from huggingface_hub import hf_hub_download from safetensors.torch import save_file folder = Path(save_folder) folder.mkdir(parents=True, exist_ok=True) raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()) hidden_size = 32 num_layers = 8 latent_dim = raw["codec"]["latent_dim"] text = raw["lm"] text.update( hidden_size=hidden_size, intermediate_size=2 * hidden_size, num_hidden_layers=num_layers, num_attention_heads=4, num_key_value_heads=1, head_dim=16, linear_num_key_heads=2, linear_num_value_heads=4, linear_key_head_dim=16, linear_value_head_dim=16, max_position_embeddings=2048, tie_word_embeddings=True, ) # Keep the source 3-linear/1-full cadence for two complete cycles. text["layer_types"] = list(text["layer_types"][:num_layers]) codec = raw["codec"] codec.update( base_channels=1, encoder_depths=[1] * len(codec["encoder_depths"]), decoder_depths=[1] * len(codec["decoder_depths"]), ) raw["head"].update(hidden_size=hidden_size, num_layers=1) raw["dtype"] = "bfloat16" from haru.config import HaruConfig from haru.models.haru import HaruModel config = HaruConfig.from_dict(raw) torch.manual_seed(42) model = HaruModel(config) with torch.no_grad(): for name, parameter in sorted(model.named_parameters()): if name.endswith("bias"): parameter.zero_() elif "norm" in name and parameter.ndim == 1: parameter.fill_(1) elif name.endswith(("gamma", "weight_scale")): parameter.fill_(1) else: # The codec allocates conv weights with torch.empty and never # initializes them, so give every remaining tensor a deterministic # value instead of leftover process memory. parameter.normal_(0, 0.02) state = {} for name, tensor in sorted(model.state_dict().items()): value = tensor.detach().cpu().contiguous() state[name] = value.to(torch.bfloat16) if value.is_floating_point() else value assert not any(name.startswith("lm_head") for name in state) assert state["lm.embed_tokens.weight"].shape == (text["vocab_size"], hidden_size) save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"}) (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n") (folder / "tokenizer.json").write_bytes( Path(hf_hub_download(source_model_id, "tokenizer.json", revision=source_revision)).read_bytes() ) return str(model) if __name__ == "__main__": create_model( source_model_id="higurashi-ai/Haru-4B", source_revision="8d6e83bcd3701a76fc27a764965665da1df7da78", save_folder="/tmp/haru", )