def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str: import json import re from pathlib import Path import torch from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata from safetensors.torch import save_file from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed folder = Path(save_folder) folder.mkdir(parents=True, exist_ok=True) raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()) precision = json.loads( Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text() ) # Two full 5-SWA/1-global cycles, sampling early and late source layers. source_layers = [*range(6), *range(42, 48)] hidden_size = 128 intermediate_size = 128 text = raw["text_config"] text.update( hidden_size=hidden_size, intermediate_size=intermediate_size, num_hidden_layers=12, num_attention_heads=8, num_key_value_heads=4, head_dim=128, swa_num_attention_heads=8, swa_num_key_value_heads=4, swa_head_dim=128, n_routed_experts=96, tie_word_embeddings=False, local_layer_ids=[i for i in range(12) if i % 6 != 5], ) raw["tie_word_embeddings"] = False import transformers raw.pop("auto_map", None) raw.pop("quantization_config", None) raw["transformers_version"] = transformers.__version__ raw["dtype"] = "bfloat16" raw["vision_config"].update( hidden_size=32, num_hidden_layers=4, num_attention_heads=4, projection_hidden_dim=32, text_hidden_size=hidden_size, ) mtp = raw["mtp_config"] mtp.update( hidden_size=hidden_size, intermediate_size=128, num_attention_heads=8, num_key_value_heads=4, head_dim=128, markov_rank=8, target_layer_ids=[5, 12], ) # from_dict adds an MXFP8-only quantizer to source configs. Construction must # bypass it; the final artifact retains its authoritative hf_quant_config. construction = json.loads(json.dumps(raw)) construction["vision_config"]["text_hidden_size"] = hidden_size config = Inkling2Config(**construction) set_seed(42) previous_dtype = torch.get_default_dtype() try: torch.set_default_dtype(torch.bfloat16) model = Inkling2ForConditionalGeneration(config) finally: torch.set_default_dtype(previous_dtype) with torch.no_grad(): for name, parameter in sorted(model.named_parameters()): if name.endswith("bias"): parameter.zero_() elif "norm" in name and parameter.ndim == 1: parameter.fill_(1) elif name.endswith(".scale"): parameter.fill_(1) else: parameter.normal_(0, 0.02) # Reverse Transformers' documented source-to-HF name conversion. Retain the # source's interleaved gate/up rows and original serialized precision. def original_name(name): name = name.replace("lm_head.weight", "model.llm.unembed.weight") name = name.replace("model.language_model.", "model.llm.") name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.") if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name): return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}" if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name): return f"model.llm.attn_res.mlp_queries.{match[1]}" name = name.replace("final_input.query", "attn_res.output_query") name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name for old, new in ( ("attn.q_proj", "attn.wq_du"), ("attn.k_proj", "attn.wk_dv"), ("attn.v_proj", "attn.wv_dv"), ("attn.r_proj", "attn.wr_du"), ("attn.o_proj", "attn.wo_ud"), ("attn.gate_proj", "attn.gate_attn"), ("attn.sink_proj", "attn.attn_sink_proj"), ("post_attention_convolution.conv1d", "attn_sconv"), ("post_mlp_convolution.conv1d", "mlp_sconv"), ("sconv.conv1d", "sconv"), ("sconv.static_weight", "sconv.static_w"), ("sconv.weight_proj.weight", "sconv.weight_proj"), ("post_attention_layernorm", "mlp_norm"), ("input_layernorm", "attn_norm"), ("q_attn_norm", "q_input_norm"), ("k_attn_norm", "k_input_norm"), ("v_attn_norm", "v_input_norm"), ("mlp.gate.e_score_correction_bias", "mlp.gate.bias"), ("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"), ("mlp.experts.down_proj", "mlp.experts.w2_weight"), ( "model.vision_tower.embeddings.position_embedding.weight", "model.visual.backbone.embeddings.position_embedding", ), ("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."), ("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."), ("model.vision_tower.merger.linear_1.", "model.visual.proj_in."), ("model.vision_tower.merger.linear_2.", "model.visual.proj_out."), ("model.vision_tower.layers.", "model.visual.backbone.layers."), (".layernorm_before.", ".ln_1."), (".layernorm_after.", ".ln_2."), (".mlp.fc1.", ".mlp.c_fc."), (".mlp.fc2.", ".mlp.c_proj."), (".lambda_1", ".ls_1.gamma"), (".lambda_2", ".ls_2.gamma"), ): if old.startswith("attn.") and not name.startswith("model.llm."): continue name = name.replace(old, new) return name def source_name(name): if match := re.search(r"model\.llm\.layers\.(\d+)\.", name): return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1) return name groups = precision["model"]["parameter_groups"] group_for = {name: group for group in groups for name in group["tensors"]} source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision) headers = { name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items() } state = {} for hf_name, value in sorted(model.state_dict().items()): name = original_name(hf_name) source = source_name(name) if source not in headers: raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}") group = group_for[source] value = value.detach().cpu().contiguous() if name.endswith("w13_weight"): value = ( value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape) ) compute = group["compute_dtype"] if compute == "nvfp4": # Source E2M1 nibbles use per-16-element E4M3 scales, with each # expert's outer scale stored as amax / denominator in FP32. value = value.float().contiguous() amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12) outer = amax[:, None, None, None] / 1536.0 blocks = value.reshape(*value.shape[:-1], -1, 16) scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn) normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12) levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6]) codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8) codes |= (normalized < 0).to(torch.uint8) * 8 codes = codes.reshape(value.shape) state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous() state[name + "_scale"] = scales.contiguous() state[name + "_amax"] = amax state[name + "_outer_scale_denominator"] = torch.tensor([1536.0]) elif compute == "mxfp8": # The pinned source stores UE8M0 scales per row/32 columns, even # though its computation descriptor says weight_block_shape=[32,32]. blocks = value.float().reshape(*value.shape[:-1], -1, 32) exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127) scales = torch.pow(2.0, exponents) state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn) state[name + "_scale"] = (exponents + 127).to(torch.uint8) else: state[name] = value.to(getattr(torch, group["output_dtype"])) # Build the independent DSpark draft using its authoritative BF16 header. draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision) draft = {} draft_module = torch.nn.Module() replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128} for name, tensor in sorted(draft_metadata.tensors.items()): if name == "fuse_weight": shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"])) elif name == "markov_head.markov_w1.weight": shape = (text["vocab_size"], mtp["markov_rank"]) elif name == "markov_head.hidden_out.weight": shape = (hidden_size, mtp["markov_rank"]) elif name == "markov_head.rnn_proj.weight": shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"]) elif name == "markov_head.rnn_proj.bias": shape = (3 * mtp["markov_rank"],) else: shape = tuple(replacements.get(n, n) for n in tensor.shape) value = torch.empty(shape, dtype=torch.bfloat16) if name.endswith("bias"): value.zero_() elif "norm" in name: value.fill_(1) else: value.normal_(0, 0.02) draft[name] = value module = draft_module parts = name.split(".") for part in parts[:-1]: if part not in module._modules: module.add_module(part, torch.nn.Module()) module = module._modules[part] module.register_parameter(parts[-1], torch.nn.Parameter(value)) model.add_module("mtp", draft_module) dtype_names = { torch.float32: "F32", torch.bfloat16: "BF16", torch.uint8: "U8", torch.float8_e4m3fn: "F8_E4M3", } for name, tensor in state.items(): source_tensor = headers[source_name(name)] if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape): raise ValueError(f"Source precision/rank mismatch: {name}") if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all(): raise ValueError(f"Nonfinite checkpoint tensor: {name}") save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"}) save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"}) # Remap every precision descriptor to exactly the retained source layers. remapped_groups = [] for group in groups: names = [name for name in state if source_name(name) in group["tensors"]] if names: remapped_groups.append({**group, "tensors": sorted(names)}) precision["model"]["parameter_groups"] = remapped_groups (folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n") (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n") for name in ( "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja", "processor_config.json", "tiktoken/tokenizer.model", ): path = folder / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes()) return str(model) if __name__ == "__main__": create_model( source_model_id="thinkingmachines/Inkling-2-Small", source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce", save_folder="/tmp/inkling-2", )