File size: 12,563 Bytes
08c77f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6c0013a
08c77f0
6c0013a
 
08c77f0
 
 
 
 
6c0013a
08c77f0
6c0013a
 
08c77f0
6c0013a
08c77f0
4c88f31
08c77f0
 
4c88f31
6c0013a
 
 
 
 
 
08c77f0
 
 
 
 
 
 
 
 
 
 
6c0013a
08c77f0
6c0013a
08c77f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6c0013a
08c77f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4c88f31
08c77f0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
    import json
    import re
    from pathlib import Path

    import torch
    from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata
    from safetensors.torch import save_file
    from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed

    folder = Path(save_folder)
    folder.mkdir(parents=True, exist_ok=True)
    raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
    precision = json.loads(
        Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
    )
    # Two full 5-SWA/1-global cycles, sampling early and late source layers.
    source_layers = [*range(6), *range(42, 48)]
    hidden_size = 128
    intermediate_size = 128
    text = raw["text_config"]
    text.update(
        hidden_size=hidden_size,
        intermediate_size=intermediate_size,
        num_hidden_layers=12,
        num_attention_heads=8,
        num_key_value_heads=4,
        head_dim=128,
        swa_num_attention_heads=8,
        swa_num_key_value_heads=4,
        swa_head_dim=128,
        n_routed_experts=96,
        tie_word_embeddings=False,
        local_layer_ids=[i for i in range(12) if i % 6 != 5],
    )
    raw["tie_word_embeddings"] = False
    import transformers

    raw.pop("auto_map", None)
    raw.pop("quantization_config", None)
    raw["transformers_version"] = transformers.__version__
    raw["dtype"] = "bfloat16"
    raw["vision_config"].update(
        hidden_size=32,
        num_hidden_layers=4,
        num_attention_heads=4,
        projection_hidden_dim=32,
        text_hidden_size=hidden_size,
    )
    mtp = raw["mtp_config"]
    mtp.update(
        hidden_size=hidden_size,
        intermediate_size=128,
        num_attention_heads=8,
        num_key_value_heads=4,
        head_dim=128,
        markov_rank=8,
        target_layer_ids=[5, 12],
    )
    # from_dict adds an MXFP8-only quantizer to source configs. Construction must
    # bypass it; the final artifact retains its authoritative hf_quant_config.
    construction = json.loads(json.dumps(raw))
    construction["vision_config"]["text_hidden_size"] = hidden_size
    config = Inkling2Config(**construction)
    set_seed(42)
    previous_dtype = torch.get_default_dtype()
    try:
        torch.set_default_dtype(torch.bfloat16)
        model = Inkling2ForConditionalGeneration(config)
    finally:
        torch.set_default_dtype(previous_dtype)
    with torch.no_grad():
        for name, parameter in sorted(model.named_parameters()):
            if name.endswith("bias"):
                parameter.zero_()
            elif "norm" in name and parameter.ndim == 1:
                parameter.fill_(1)
            elif name.endswith(".scale"):
                parameter.fill_(1)
            else:
                parameter.normal_(0, 0.02)

    # Reverse Transformers' documented source-to-HF name conversion. Retain the
    # source's interleaved gate/up rows and original serialized precision.
    def original_name(name):
        name = name.replace("lm_head.weight", "model.llm.unembed.weight")
        name = name.replace("model.language_model.", "model.llm.")
        name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.")
        if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name):
            return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}"
        if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name):
            return f"model.llm.attn_res.mlp_queries.{match[1]}"
        name = name.replace("final_input.query", "attn_res.output_query")
        name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name
        for old, new in (
            ("attn.q_proj", "attn.wq_du"),
            ("attn.k_proj", "attn.wk_dv"),
            ("attn.v_proj", "attn.wv_dv"),
            ("attn.r_proj", "attn.wr_du"),
            ("attn.o_proj", "attn.wo_ud"),
            ("attn.gate_proj", "attn.gate_attn"),
            ("attn.sink_proj", "attn.attn_sink_proj"),
            ("post_attention_convolution.conv1d", "attn_sconv"),
            ("post_mlp_convolution.conv1d", "mlp_sconv"),
            ("sconv.conv1d", "sconv"),
            ("sconv.static_weight", "sconv.static_w"),
            ("sconv.weight_proj.weight", "sconv.weight_proj"),
            ("post_attention_layernorm", "mlp_norm"),
            ("input_layernorm", "attn_norm"),
            ("q_attn_norm", "q_input_norm"),
            ("k_attn_norm", "k_input_norm"),
            ("v_attn_norm", "v_input_norm"),
            ("mlp.gate.e_score_correction_bias", "mlp.gate.bias"),
            ("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"),
            ("mlp.experts.down_proj", "mlp.experts.w2_weight"),
            (
                "model.vision_tower.embeddings.position_embedding.weight",
                "model.visual.backbone.embeddings.position_embedding",
            ),
            ("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."),
            ("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."),
            ("model.vision_tower.merger.linear_1.", "model.visual.proj_in."),
            ("model.vision_tower.merger.linear_2.", "model.visual.proj_out."),
            ("model.vision_tower.layers.", "model.visual.backbone.layers."),
            (".layernorm_before.", ".ln_1."),
            (".layernorm_after.", ".ln_2."),
            (".mlp.fc1.", ".mlp.c_fc."),
            (".mlp.fc2.", ".mlp.c_proj."),
            (".lambda_1", ".ls_1.gamma"),
            (".lambda_2", ".ls_2.gamma"),
        ):
            if old.startswith("attn.") and not name.startswith("model.llm."):
                continue
            name = name.replace(old, new)
        return name

    def source_name(name):
        if match := re.search(r"model\.llm\.layers\.(\d+)\.", name):
            return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1)
        return name

    groups = precision["model"]["parameter_groups"]
    group_for = {name: group for group in groups for name in group["tensors"]}
    source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision)
    headers = {
        name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items()
    }
    state = {}
    for hf_name, value in sorted(model.state_dict().items()):
        name = original_name(hf_name)
        source = source_name(name)
        if source not in headers:
            raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}")
        group = group_for[source]
        value = value.detach().cpu().contiguous()
        if name.endswith("w13_weight"):
            value = (
                value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape)
            )
        compute = group["compute_dtype"]
        if compute == "nvfp4":
            # Source E2M1 nibbles use per-16-element E4M3 scales, with each
            # expert's outer scale stored as amax / denominator in FP32.
            value = value.float().contiguous()
            amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12)
            outer = amax[:, None, None, None] / 1536.0
            blocks = value.reshape(*value.shape[:-1], -1, 16)
            scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn)
            normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12)
            levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6])
            codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8)
            codes |= (normalized < 0).to(torch.uint8) * 8
            codes = codes.reshape(value.shape)
            state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
            state[name + "_scale"] = scales.contiguous()
            state[name + "_amax"] = amax
            state[name + "_outer_scale_denominator"] = torch.tensor([1536.0])
        elif compute == "mxfp8":
            # The pinned source stores UE8M0 scales per row/32 columns, even
            # though its computation descriptor says weight_block_shape=[32,32].
            blocks = value.float().reshape(*value.shape[:-1], -1, 32)
            exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127)
            scales = torch.pow(2.0, exponents)
            state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn)
            state[name + "_scale"] = (exponents + 127).to(torch.uint8)
        else:
            state[name] = value.to(getattr(torch, group["output_dtype"]))

    # Build the independent DSpark draft using its authoritative BF16 header.
    draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
    draft = {}
    draft_module = torch.nn.Module()
    replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128}
    for name, tensor in sorted(draft_metadata.tensors.items()):
        if name == "fuse_weight":
            shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
        elif name == "markov_head.markov_w1.weight":
            shape = (text["vocab_size"], mtp["markov_rank"])
        elif name == "markov_head.hidden_out.weight":
            shape = (hidden_size, mtp["markov_rank"])
        elif name == "markov_head.rnn_proj.weight":
            shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"])
        elif name == "markov_head.rnn_proj.bias":
            shape = (3 * mtp["markov_rank"],)
        else:
            shape = tuple(replacements.get(n, n) for n in tensor.shape)
        value = torch.empty(shape, dtype=torch.bfloat16)
        if name.endswith("bias"):
            value.zero_()
        elif "norm" in name:
            value.fill_(1)
        else:
            value.normal_(0, 0.02)
        draft[name] = value
        module = draft_module
        parts = name.split(".")
        for part in parts[:-1]:
            if part not in module._modules:
                module.add_module(part, torch.nn.Module())
            module = module._modules[part]
        module.register_parameter(parts[-1], torch.nn.Parameter(value))
    model.add_module("mtp", draft_module)
    dtype_names = {
        torch.float32: "F32",
        torch.bfloat16: "BF16",
        torch.uint8: "U8",
        torch.float8_e4m3fn: "F8_E4M3",
    }
    for name, tensor in state.items():
        source_tensor = headers[source_name(name)]
        if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape):
            raise ValueError(f"Source precision/rank mismatch: {name}")
        if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all():
            raise ValueError(f"Nonfinite checkpoint tensor: {name}")
    save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
    save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"})

    # Remap every precision descriptor to exactly the retained source layers.
    remapped_groups = []
    for group in groups:
        names = [name for name in state if source_name(name) in group["tensors"]]
        if names:
            remapped_groups.append({**group, "tensors": sorted(names)})
    precision["model"]["parameter_groups"] = remapped_groups
    (folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n")
    (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
    for name in (
        "tokenizer.json",
        "tokenizer_config.json",
        "special_tokens_map.json",
        "chat_template.jinja",
        "processor_config.json",
        "tiktoken/tokenizer.model",
    ):
        path = folder / name
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
    return str(model)


if __name__ == "__main__":
    create_model(
        source_model_id="thinkingmachines/Inkling-2-Small",
        source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
        save_folder="/tmp/inkling-2",
    )