Download .meta/create_model.py from tiny-random/inkling-2: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/tiny-random/inkling-2/resolve/main/.meta/create_model.py
- Command line
-
hf download hf://tiny-random/inkling-2/.meta/create_model.py
-
curl -L -o create_model.py https://huggingface.co/tiny-random/inkling-2/resolve/main/.meta/create_model.py
12.6 kB
| def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str: | |
| import json | |
| import re | |
| from pathlib import Path | |
| import torch | |
| from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata | |
| from safetensors.torch import save_file | |
| from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed | |
| folder = Path(save_folder) | |
| folder.mkdir(parents=True, exist_ok=True) | |
| raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()) | |
| precision = json.loads( | |
| Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text() | |
| ) | |
| # Two full 5-SWA/1-global cycles, sampling early and late source layers. | |
| source_layers = [*range(6), *range(42, 48)] | |
| hidden_size = 128 | |
| intermediate_size = 128 | |
| text = raw["text_config"] | |
| text.update( | |
| hidden_size=hidden_size, | |
| intermediate_size=intermediate_size, | |
| num_hidden_layers=12, | |
| num_attention_heads=8, | |
| num_key_value_heads=4, | |
| head_dim=128, | |
| swa_num_attention_heads=8, | |
| swa_num_key_value_heads=4, | |
| swa_head_dim=128, | |
| n_routed_experts=96, | |
| tie_word_embeddings=False, | |
| local_layer_ids=[i for i in range(12) if i % 6 != 5], | |
| ) | |
| raw["tie_word_embeddings"] = False | |
| import transformers | |
| raw.pop("auto_map", None) | |
| raw.pop("quantization_config", None) | |
| raw["transformers_version"] = transformers.__version__ | |
| raw["dtype"] = "bfloat16" | |
| raw["vision_config"].update( | |
| hidden_size=32, | |
| num_hidden_layers=4, | |
| num_attention_heads=4, | |
| projection_hidden_dim=32, | |
| text_hidden_size=hidden_size, | |
| ) | |
| mtp = raw["mtp_config"] | |
| mtp.update( | |
| hidden_size=hidden_size, | |
| intermediate_size=128, | |
| num_attention_heads=8, | |
| num_key_value_heads=4, | |
| head_dim=128, | |
| markov_rank=8, | |
| target_layer_ids=[5, 12], | |
| ) | |
| # from_dict adds an MXFP8-only quantizer to source configs. Construction must | |
| # bypass it; the final artifact retains its authoritative hf_quant_config. | |
| construction = json.loads(json.dumps(raw)) | |
| construction["vision_config"]["text_hidden_size"] = hidden_size | |
| config = Inkling2Config(**construction) | |
| set_seed(42) | |
| previous_dtype = torch.get_default_dtype() | |
| try: | |
| torch.set_default_dtype(torch.bfloat16) | |
| model = Inkling2ForConditionalGeneration(config) | |
| finally: | |
| torch.set_default_dtype(previous_dtype) | |
| with torch.no_grad(): | |
| for name, parameter in sorted(model.named_parameters()): | |
| if name.endswith("bias"): | |
| parameter.zero_() | |
| elif "norm" in name and parameter.ndim == 1: | |
| parameter.fill_(1) | |
| elif name.endswith(".scale"): | |
| parameter.fill_(1) | |
| else: | |
| parameter.normal_(0, 0.02) | |
| # Reverse Transformers' documented source-to-HF name conversion. Retain the | |
| # source's interleaved gate/up rows and original serialized precision. | |
| def original_name(name): | |
| name = name.replace("lm_head.weight", "model.llm.unembed.weight") | |
| name = name.replace("model.language_model.", "model.llm.") | |
| name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.") | |
| if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name): | |
| return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}" | |
| if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name): | |
| return f"model.llm.attn_res.mlp_queries.{match[1]}" | |
| name = name.replace("final_input.query", "attn_res.output_query") | |
| name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name | |
| for old, new in ( | |
| ("attn.q_proj", "attn.wq_du"), | |
| ("attn.k_proj", "attn.wk_dv"), | |
| ("attn.v_proj", "attn.wv_dv"), | |
| ("attn.r_proj", "attn.wr_du"), | |
| ("attn.o_proj", "attn.wo_ud"), | |
| ("attn.gate_proj", "attn.gate_attn"), | |
| ("attn.sink_proj", "attn.attn_sink_proj"), | |
| ("post_attention_convolution.conv1d", "attn_sconv"), | |
| ("post_mlp_convolution.conv1d", "mlp_sconv"), | |
| ("sconv.conv1d", "sconv"), | |
| ("sconv.static_weight", "sconv.static_w"), | |
| ("sconv.weight_proj.weight", "sconv.weight_proj"), | |
| ("post_attention_layernorm", "mlp_norm"), | |
| ("input_layernorm", "attn_norm"), | |
| ("q_attn_norm", "q_input_norm"), | |
| ("k_attn_norm", "k_input_norm"), | |
| ("v_attn_norm", "v_input_norm"), | |
| ("mlp.gate.e_score_correction_bias", "mlp.gate.bias"), | |
| ("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"), | |
| ("mlp.experts.down_proj", "mlp.experts.w2_weight"), | |
| ( | |
| "model.vision_tower.embeddings.position_embedding.weight", | |
| "model.visual.backbone.embeddings.position_embedding", | |
| ), | |
| ("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."), | |
| ("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."), | |
| ("model.vision_tower.merger.linear_1.", "model.visual.proj_in."), | |
| ("model.vision_tower.merger.linear_2.", "model.visual.proj_out."), | |
| ("model.vision_tower.layers.", "model.visual.backbone.layers."), | |
| (".layernorm_before.", ".ln_1."), | |
| (".layernorm_after.", ".ln_2."), | |
| (".mlp.fc1.", ".mlp.c_fc."), | |
| (".mlp.fc2.", ".mlp.c_proj."), | |
| (".lambda_1", ".ls_1.gamma"), | |
| (".lambda_2", ".ls_2.gamma"), | |
| ): | |
| if old.startswith("attn.") and not name.startswith("model.llm."): | |
| continue | |
| name = name.replace(old, new) | |
| return name | |
| def source_name(name): | |
| if match := re.search(r"model\.llm\.layers\.(\d+)\.", name): | |
| return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1) | |
| return name | |
| groups = precision["model"]["parameter_groups"] | |
| group_for = {name: group for group in groups for name in group["tensors"]} | |
| source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision) | |
| headers = { | |
| name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items() | |
| } | |
| state = {} | |
| for hf_name, value in sorted(model.state_dict().items()): | |
| name = original_name(hf_name) | |
| source = source_name(name) | |
| if source not in headers: | |
| raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}") | |
| group = group_for[source] | |
| value = value.detach().cpu().contiguous() | |
| if name.endswith("w13_weight"): | |
| value = ( | |
| value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape) | |
| ) | |
| compute = group["compute_dtype"] | |
| if compute == "nvfp4": | |
| # Source E2M1 nibbles use per-16-element E4M3 scales, with each | |
| # expert's outer scale stored as amax / denominator in FP32. | |
| value = value.float().contiguous() | |
| amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12) | |
| outer = amax[:, None, None, None] / 1536.0 | |
| blocks = value.reshape(*value.shape[:-1], -1, 16) | |
| scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn) | |
| normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12) | |
| levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6]) | |
| codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8) | |
| codes |= (normalized < 0).to(torch.uint8) * 8 | |
| codes = codes.reshape(value.shape) | |
| state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous() | |
| state[name + "_scale"] = scales.contiguous() | |
| state[name + "_amax"] = amax | |
| state[name + "_outer_scale_denominator"] = torch.tensor([1536.0]) | |
| elif compute == "mxfp8": | |
| # The pinned source stores UE8M0 scales per row/32 columns, even | |
| # though its computation descriptor says weight_block_shape=[32,32]. | |
| blocks = value.float().reshape(*value.shape[:-1], -1, 32) | |
| exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127) | |
| scales = torch.pow(2.0, exponents) | |
| state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn) | |
| state[name + "_scale"] = (exponents + 127).to(torch.uint8) | |
| else: | |
| state[name] = value.to(getattr(torch, group["output_dtype"])) | |
| # Build the independent DSpark draft using its authoritative BF16 header. | |
| draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision) | |
| draft = {} | |
| draft_module = torch.nn.Module() | |
| replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128} | |
| for name, tensor in sorted(draft_metadata.tensors.items()): | |
| if name == "fuse_weight": | |
| shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"])) | |
| elif name == "markov_head.markov_w1.weight": | |
| shape = (text["vocab_size"], mtp["markov_rank"]) | |
| elif name == "markov_head.hidden_out.weight": | |
| shape = (hidden_size, mtp["markov_rank"]) | |
| elif name == "markov_head.rnn_proj.weight": | |
| shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"]) | |
| elif name == "markov_head.rnn_proj.bias": | |
| shape = (3 * mtp["markov_rank"],) | |
| else: | |
| shape = tuple(replacements.get(n, n) for n in tensor.shape) | |
| value = torch.empty(shape, dtype=torch.bfloat16) | |
| if name.endswith("bias"): | |
| value.zero_() | |
| elif "norm" in name: | |
| value.fill_(1) | |
| else: | |
| value.normal_(0, 0.02) | |
| draft[name] = value | |
| module = draft_module | |
| parts = name.split(".") | |
| for part in parts[:-1]: | |
| if part not in module._modules: | |
| module.add_module(part, torch.nn.Module()) | |
| module = module._modules[part] | |
| module.register_parameter(parts[-1], torch.nn.Parameter(value)) | |
| model.add_module("mtp", draft_module) | |
| dtype_names = { | |
| torch.float32: "F32", | |
| torch.bfloat16: "BF16", | |
| torch.uint8: "U8", | |
| torch.float8_e4m3fn: "F8_E4M3", | |
| } | |
| for name, tensor in state.items(): | |
| source_tensor = headers[source_name(name)] | |
| if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape): | |
| raise ValueError(f"Source precision/rank mismatch: {name}") | |
| if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all(): | |
| raise ValueError(f"Nonfinite checkpoint tensor: {name}") | |
| save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"}) | |
| save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"}) | |
| # Remap every precision descriptor to exactly the retained source layers. | |
| remapped_groups = [] | |
| for group in groups: | |
| names = [name for name in state if source_name(name) in group["tensors"]] | |
| if names: | |
| remapped_groups.append({**group, "tensors": sorted(names)}) | |
| precision["model"]["parameter_groups"] = remapped_groups | |
| (folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n") | |
| (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n") | |
| for name in ( | |
| "tokenizer.json", | |
| "tokenizer_config.json", | |
| "special_tokens_map.json", | |
| "chat_template.jinja", | |
| "processor_config.json", | |
| "tiktoken/tokenizer.model", | |
| ): | |
| path = folder / name | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes()) | |
| return str(model) | |
| if __name__ == "__main__": | |
| create_model( | |
| source_model_id="thinkingmachines/Inkling-2-Small", | |
| source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce", | |
| save_folder="/tmp/inkling-2", | |
| ) | |