File size: 3,095 Bytes
fca9437
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
    import json
    from pathlib import Path

    import torch
    from huggingface_hub import hf_hub_download
    from safetensors.torch import save_file

    folder = Path(save_folder)
    folder.mkdir(parents=True, exist_ok=True)
    raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())

    hidden_size = 32
    num_layers = 8
    latent_dim = raw["codec"]["latent_dim"]
    text = raw["lm"]
    text.update(
        hidden_size=hidden_size,
        intermediate_size=2 * hidden_size,
        num_hidden_layers=num_layers,
        num_attention_heads=4,
        num_key_value_heads=1,
        head_dim=16,
        linear_num_key_heads=2,
        linear_num_value_heads=4,
        linear_key_head_dim=16,
        linear_value_head_dim=16,
        max_position_embeddings=2048,
        tie_word_embeddings=True,
    )
    # Keep the source 3-linear/1-full cadence for two complete cycles.
    text["layer_types"] = list(text["layer_types"][:num_layers])
    codec = raw["codec"]
    codec.update(
        base_channels=1,
        encoder_depths=[1] * len(codec["encoder_depths"]),
        decoder_depths=[1] * len(codec["decoder_depths"]),
    )
    raw["head"].update(hidden_size=hidden_size, num_layers=1)
    raw["dtype"] = "bfloat16"

    from haru.config import HaruConfig
    from haru.models.haru import HaruModel

    config = HaruConfig.from_dict(raw)
    torch.manual_seed(42)
    model = HaruModel(config)
    with torch.no_grad():
        for name, parameter in sorted(model.named_parameters()):
            if name.endswith("bias"):
                parameter.zero_()
            elif "norm" in name and parameter.ndim == 1:
                parameter.fill_(1)
            elif name.endswith(("gamma", "weight_scale")):
                parameter.fill_(1)
            else:
                # The codec allocates conv weights with torch.empty and never
                # initializes them, so give every remaining tensor a deterministic
                # value instead of leftover process memory.
                parameter.normal_(0, 0.02)

    state = {}
    for name, tensor in sorted(model.state_dict().items()):
        value = tensor.detach().cpu().contiguous()
        state[name] = value.to(torch.bfloat16) if value.is_floating_point() else value
    assert not any(name.startswith("lm_head") for name in state)
    assert state["lm.embed_tokens.weight"].shape == (text["vocab_size"], hidden_size)
    save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
    (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
    (folder / "tokenizer.json").write_bytes(
        Path(hf_hub_download(source_model_id, "tokenizer.json", revision=source_revision)).read_bytes()
    )
    return str(model)


if __name__ == "__main__":
    create_model(
        source_model_id="higurashi-ai/Haru-4B",
        source_revision="8d6e83bcd3701a76fc27a764965665da1df7da78",
        save_folder="/tmp/haru",
    )