inkling-2 / .meta /create_model.py
yujiepan's picture
Preserve native mixed NVFP4/MXFP8 Inkling 2 without model-specific plugins
6c0013a verified
Raw History Blame Contribute Delete
12.6 kB
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
import json
import re
from pathlib import Path
import torch
from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata
from safetensors.torch import save_file
from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
precision = json.loads(
Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
)
# Two full 5-SWA/1-global cycles, sampling early and late source layers.
source_layers = [*range(6), *range(42, 48)]
hidden_size = 128
intermediate_size = 128
text = raw["text_config"]
text.update(
hidden_size=hidden_size,
intermediate_size=intermediate_size,
num_hidden_layers=12,
num_attention_heads=8,
num_key_value_heads=4,
head_dim=128,
swa_num_attention_heads=8,
swa_num_key_value_heads=4,
swa_head_dim=128,
n_routed_experts=96,
tie_word_embeddings=False,
local_layer_ids=[i for i in range(12) if i % 6 != 5],
)
raw["tie_word_embeddings"] = False
import transformers
raw.pop("auto_map", None)
raw.pop("quantization_config", None)
raw["transformers_version"] = transformers.__version__
raw["dtype"] = "bfloat16"
raw["vision_config"].update(
hidden_size=32,
num_hidden_layers=4,
num_attention_heads=4,
projection_hidden_dim=32,
text_hidden_size=hidden_size,
)
mtp = raw["mtp_config"]
mtp.update(
hidden_size=hidden_size,
intermediate_size=128,
num_attention_heads=8,
num_key_value_heads=4,
head_dim=128,
markov_rank=8,
target_layer_ids=[5, 12],
)
# from_dict adds an MXFP8-only quantizer to source configs. Construction must
# bypass it; the final artifact retains its authoritative hf_quant_config.
construction = json.loads(json.dumps(raw))
construction["vision_config"]["text_hidden_size"] = hidden_size
config = Inkling2Config(**construction)
set_seed(42)
previous_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(torch.bfloat16)
model = Inkling2ForConditionalGeneration(config)
finally:
torch.set_default_dtype(previous_dtype)
with torch.no_grad():
for name, parameter in sorted(model.named_parameters()):
if name.endswith("bias"):
parameter.zero_()
elif "norm" in name and parameter.ndim == 1:
parameter.fill_(1)
elif name.endswith(".scale"):
parameter.fill_(1)
else:
parameter.normal_(0, 0.02)
# Reverse Transformers' documented source-to-HF name conversion. Retain the
# source's interleaved gate/up rows and original serialized precision.
def original_name(name):
name = name.replace("lm_head.weight", "model.llm.unembed.weight")
name = name.replace("model.language_model.", "model.llm.")
name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.")
if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name):
return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}"
if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name):
return f"model.llm.attn_res.mlp_queries.{match[1]}"
name = name.replace("final_input.query", "attn_res.output_query")
name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name
for old, new in (
("attn.q_proj", "attn.wq_du"),
("attn.k_proj", "attn.wk_dv"),
("attn.v_proj", "attn.wv_dv"),
("attn.r_proj", "attn.wr_du"),
("attn.o_proj", "attn.wo_ud"),
("attn.gate_proj", "attn.gate_attn"),
("attn.sink_proj", "attn.attn_sink_proj"),
("post_attention_convolution.conv1d", "attn_sconv"),
("post_mlp_convolution.conv1d", "mlp_sconv"),
("sconv.conv1d", "sconv"),
("sconv.static_weight", "sconv.static_w"),
("sconv.weight_proj.weight", "sconv.weight_proj"),
("post_attention_layernorm", "mlp_norm"),
("input_layernorm", "attn_norm"),
("q_attn_norm", "q_input_norm"),
("k_attn_norm", "k_input_norm"),
("v_attn_norm", "v_input_norm"),
("mlp.gate.e_score_correction_bias", "mlp.gate.bias"),
("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"),
("mlp.experts.down_proj", "mlp.experts.w2_weight"),
(
"model.vision_tower.embeddings.position_embedding.weight",
"model.visual.backbone.embeddings.position_embedding",
),
("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."),
("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."),
("model.vision_tower.merger.linear_1.", "model.visual.proj_in."),
("model.vision_tower.merger.linear_2.", "model.visual.proj_out."),
("model.vision_tower.layers.", "model.visual.backbone.layers."),
(".layernorm_before.", ".ln_1."),
(".layernorm_after.", ".ln_2."),
(".mlp.fc1.", ".mlp.c_fc."),
(".mlp.fc2.", ".mlp.c_proj."),
(".lambda_1", ".ls_1.gamma"),
(".lambda_2", ".ls_2.gamma"),
):
if old.startswith("attn.") and not name.startswith("model.llm."):
continue
name = name.replace(old, new)
return name
def source_name(name):
if match := re.search(r"model\.llm\.layers\.(\d+)\.", name):
return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1)
return name
groups = precision["model"]["parameter_groups"]
group_for = {name: group for group in groups for name in group["tensors"]}
source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision)
headers = {
name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items()
}
state = {}
for hf_name, value in sorted(model.state_dict().items()):
name = original_name(hf_name)
source = source_name(name)
if source not in headers:
raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}")
group = group_for[source]
value = value.detach().cpu().contiguous()
if name.endswith("w13_weight"):
value = (
value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape)
)
compute = group["compute_dtype"]
if compute == "nvfp4":
# Source E2M1 nibbles use per-16-element E4M3 scales, with each
# expert's outer scale stored as amax / denominator in FP32.
value = value.float().contiguous()
amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12)
outer = amax[:, None, None, None] / 1536.0
blocks = value.reshape(*value.shape[:-1], -1, 16)
scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn)
normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12)
levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6])
codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8)
codes |= (normalized < 0).to(torch.uint8) * 8
codes = codes.reshape(value.shape)
state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
state[name + "_scale"] = scales.contiguous()
state[name + "_amax"] = amax
state[name + "_outer_scale_denominator"] = torch.tensor([1536.0])
elif compute == "mxfp8":
# The pinned source stores UE8M0 scales per row/32 columns, even
# though its computation descriptor says weight_block_shape=[32,32].
blocks = value.float().reshape(*value.shape[:-1], -1, 32)
exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127)
scales = torch.pow(2.0, exponents)
state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn)
state[name + "_scale"] = (exponents + 127).to(torch.uint8)
else:
state[name] = value.to(getattr(torch, group["output_dtype"]))
# Build the independent DSpark draft using its authoritative BF16 header.
draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
draft = {}
draft_module = torch.nn.Module()
replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128}
for name, tensor in sorted(draft_metadata.tensors.items()):
if name == "fuse_weight":
shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
elif name == "markov_head.markov_w1.weight":
shape = (text["vocab_size"], mtp["markov_rank"])
elif name == "markov_head.hidden_out.weight":
shape = (hidden_size, mtp["markov_rank"])
elif name == "markov_head.rnn_proj.weight":
shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"])
elif name == "markov_head.rnn_proj.bias":
shape = (3 * mtp["markov_rank"],)
else:
shape = tuple(replacements.get(n, n) for n in tensor.shape)
value = torch.empty(shape, dtype=torch.bfloat16)
if name.endswith("bias"):
value.zero_()
elif "norm" in name:
value.fill_(1)
else:
value.normal_(0, 0.02)
draft[name] = value
module = draft_module
parts = name.split(".")
for part in parts[:-1]:
if part not in module._modules:
module.add_module(part, torch.nn.Module())
module = module._modules[part]
module.register_parameter(parts[-1], torch.nn.Parameter(value))
model.add_module("mtp", draft_module)
dtype_names = {
torch.float32: "F32",
torch.bfloat16: "BF16",
torch.uint8: "U8",
torch.float8_e4m3fn: "F8_E4M3",
}
for name, tensor in state.items():
source_tensor = headers[source_name(name)]
if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape):
raise ValueError(f"Source precision/rank mismatch: {name}")
if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all():
raise ValueError(f"Nonfinite checkpoint tensor: {name}")
save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"})
# Remap every precision descriptor to exactly the retained source layers.
remapped_groups = []
for group in groups:
names = [name for name in state if source_name(name) in group["tensors"]]
if names:
remapped_groups.append({**group, "tensors": sorted(names)})
precision["model"]["parameter_groups"] = remapped_groups
(folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n")
(folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
for name in (
"tokenizer.json",
"tokenizer_config.json",
"special_tokens_map.json",
"chat_template.jinja",
"processor_config.json",
"tiktoken/tokenizer.model",
):
path = folder / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
return str(model)
if __name__ == "__main__":
create_model(
source_model_id="thinkingmachines/Inkling-2-Small",
source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
save_folder="/tmp/inkling-2",
)