Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- .meta/create_model.py +561 -0
- .meta/env.json +57 -0
- .meta/model.log +400 -0
- README.md +85 -0
- chat_template.jinja +130 -0
- config.json +105 -0
- hf_quant_config.json +525 -0
- model.safetensors +3 -0
- modeling_inkling2_tiny.py +102 -0
- mtp.safetensors +3 -0
- processor_config.json +14 -0
- pyproject.toml +15 -0
- sglang_inkling2_tied.py +169 -0
- special_tokens_map.json +19 -0
- tiktoken/tokenizer.model +3 -0
- tokenizer.json +3 -0
- tokenizer_config.json +546 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
.meta/create_model.py
ADDED
|
@@ -0,0 +1,561 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
|
| 2 |
+
import json
|
| 3 |
+
import re
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata
|
| 8 |
+
from safetensors.torch import save_file
|
| 9 |
+
from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed
|
| 10 |
+
|
| 11 |
+
folder = Path(save_folder)
|
| 12 |
+
folder.mkdir(parents=True, exist_ok=True)
|
| 13 |
+
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
|
| 14 |
+
precision = json.loads(
|
| 15 |
+
Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
|
| 16 |
+
)
|
| 17 |
+
# Two full 5-SWA/1-global cycles, sampling both source expert precision regions.
|
| 18 |
+
source_layers = [*range(6), *range(42, 48)]
|
| 19 |
+
hidden_size = 32
|
| 20 |
+
intermediate_size = 32
|
| 21 |
+
text = raw["text_config"]
|
| 22 |
+
text.update(
|
| 23 |
+
hidden_size=hidden_size,
|
| 24 |
+
intermediate_size=intermediate_size,
|
| 25 |
+
num_hidden_layers=12,
|
| 26 |
+
num_attention_heads=24,
|
| 27 |
+
num_key_value_heads=4,
|
| 28 |
+
head_dim=32,
|
| 29 |
+
swa_num_attention_heads=24,
|
| 30 |
+
swa_num_key_value_heads=4,
|
| 31 |
+
swa_head_dim=32,
|
| 32 |
+
n_routed_experts=96,
|
| 33 |
+
tie_word_embeddings=True,
|
| 34 |
+
local_layer_ids=[i for i in range(12) if i % 6 != 5],
|
| 35 |
+
)
|
| 36 |
+
raw["tie_word_embeddings"] = True
|
| 37 |
+
raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
|
| 38 |
+
raw["vision_config"].update(
|
| 39 |
+
hidden_size=32,
|
| 40 |
+
num_hidden_layers=4,
|
| 41 |
+
num_attention_heads=4,
|
| 42 |
+
projection_hidden_dim=32,
|
| 43 |
+
text_hidden_size=hidden_size,
|
| 44 |
+
)
|
| 45 |
+
mtp = raw["mtp_config"]
|
| 46 |
+
mtp.update(
|
| 47 |
+
hidden_size=hidden_size,
|
| 48 |
+
intermediate_size=128,
|
| 49 |
+
num_attention_heads=24,
|
| 50 |
+
num_key_value_heads=4,
|
| 51 |
+
head_dim=32,
|
| 52 |
+
markov_rank=8,
|
| 53 |
+
target_layer_ids=[5, 12],
|
| 54 |
+
)
|
| 55 |
+
# from_dict adds an MXFP8-only quantizer to source configs. Construction must
|
| 56 |
+
# bypass it; the final artifact retains its authoritative hf_quant_config.
|
| 57 |
+
construction = json.loads(json.dumps(raw))
|
| 58 |
+
construction["vision_config"]["text_hidden_size"] = hidden_size
|
| 59 |
+
config = Inkling2Config(**construction)
|
| 60 |
+
set_seed(42)
|
| 61 |
+
previous_dtype = torch.get_default_dtype()
|
| 62 |
+
try:
|
| 63 |
+
torch.set_default_dtype(torch.bfloat16)
|
| 64 |
+
model = Inkling2ForConditionalGeneration(config)
|
| 65 |
+
finally:
|
| 66 |
+
torch.set_default_dtype(previous_dtype)
|
| 67 |
+
with torch.no_grad():
|
| 68 |
+
for name, parameter in sorted(model.named_parameters()):
|
| 69 |
+
if name.endswith("bias"):
|
| 70 |
+
parameter.zero_()
|
| 71 |
+
elif "norm" in name and parameter.ndim == 1:
|
| 72 |
+
parameter.fill_(1)
|
| 73 |
+
elif name.endswith(".scale"):
|
| 74 |
+
parameter.fill_(1)
|
| 75 |
+
else:
|
| 76 |
+
parameter.normal_(0, 0.02)
|
| 77 |
+
# Inkling 2's concrete Transformers class deliberately has no tied-weight
|
| 78 |
+
# mapping. Tie explicitly, then serialize the embedding only.
|
| 79 |
+
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 80 |
+
|
| 81 |
+
# Reverse Transformers' documented source-to-HF name conversion. Retain the
|
| 82 |
+
# source's interleaved gate/up rows and original serialized precision.
|
| 83 |
+
def original_name(name):
|
| 84 |
+
name = name.replace("lm_head.weight", "model.llm.unembed.weight")
|
| 85 |
+
name = name.replace("model.language_model.", "model.llm.")
|
| 86 |
+
name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.")
|
| 87 |
+
if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name):
|
| 88 |
+
return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}"
|
| 89 |
+
if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name):
|
| 90 |
+
return f"model.llm.attn_res.mlp_queries.{match[1]}"
|
| 91 |
+
name = name.replace("final_input.query", "attn_res.output_query")
|
| 92 |
+
name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name
|
| 93 |
+
for old, new in (
|
| 94 |
+
("attn.q_proj", "attn.wq_du"),
|
| 95 |
+
("attn.k_proj", "attn.wk_dv"),
|
| 96 |
+
("attn.v_proj", "attn.wv_dv"),
|
| 97 |
+
("attn.r_proj", "attn.wr_du"),
|
| 98 |
+
("attn.o_proj", "attn.wo_ud"),
|
| 99 |
+
("attn.gate_proj", "attn.gate_attn"),
|
| 100 |
+
("attn.sink_proj", "attn.attn_sink_proj"),
|
| 101 |
+
("post_attention_convolution.conv1d", "attn_sconv"),
|
| 102 |
+
("post_mlp_convolution.conv1d", "mlp_sconv"),
|
| 103 |
+
("sconv.conv1d", "sconv"),
|
| 104 |
+
("sconv.static_weight", "sconv.static_w"),
|
| 105 |
+
("sconv.weight_proj.weight", "sconv.weight_proj"),
|
| 106 |
+
("post_attention_layernorm", "mlp_norm"),
|
| 107 |
+
("input_layernorm", "attn_norm"),
|
| 108 |
+
("q_attn_norm", "q_input_norm"),
|
| 109 |
+
("k_attn_norm", "k_input_norm"),
|
| 110 |
+
("v_attn_norm", "v_input_norm"),
|
| 111 |
+
("mlp.gate.e_score_correction_bias", "mlp.gate.bias"),
|
| 112 |
+
("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"),
|
| 113 |
+
("mlp.experts.down_proj", "mlp.experts.w2_weight"),
|
| 114 |
+
(
|
| 115 |
+
"model.vision_tower.embeddings.position_embedding.weight",
|
| 116 |
+
"model.visual.backbone.embeddings.position_embedding",
|
| 117 |
+
),
|
| 118 |
+
("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."),
|
| 119 |
+
("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."),
|
| 120 |
+
("model.vision_tower.merger.linear_1.", "model.visual.proj_in."),
|
| 121 |
+
("model.vision_tower.merger.linear_2.", "model.visual.proj_out."),
|
| 122 |
+
("model.vision_tower.layers.", "model.visual.backbone.layers."),
|
| 123 |
+
(".layernorm_before.", ".ln_1."),
|
| 124 |
+
(".layernorm_after.", ".ln_2."),
|
| 125 |
+
(".mlp.fc1.", ".mlp.c_fc."),
|
| 126 |
+
(".mlp.fc2.", ".mlp.c_proj."),
|
| 127 |
+
(".lambda_1", ".ls_1.gamma"),
|
| 128 |
+
(".lambda_2", ".ls_2.gamma"),
|
| 129 |
+
):
|
| 130 |
+
if old.startswith("attn.") and not name.startswith("model.llm."):
|
| 131 |
+
continue
|
| 132 |
+
name = name.replace(old, new)
|
| 133 |
+
return name
|
| 134 |
+
|
| 135 |
+
def source_name(name):
|
| 136 |
+
if match := re.search(r"model\.llm\.layers\.(\d+)\.", name):
|
| 137 |
+
return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1)
|
| 138 |
+
return name
|
| 139 |
+
|
| 140 |
+
groups = precision["model"]["parameter_groups"]
|
| 141 |
+
group_for = {name: group for group in groups for name in group["tensors"]}
|
| 142 |
+
source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision)
|
| 143 |
+
headers = {
|
| 144 |
+
name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items()
|
| 145 |
+
}
|
| 146 |
+
state = {}
|
| 147 |
+
for hf_name, value in sorted(model.state_dict().items()):
|
| 148 |
+
if hf_name == "lm_head.weight":
|
| 149 |
+
continue
|
| 150 |
+
name = original_name(hf_name)
|
| 151 |
+
source = source_name(name)
|
| 152 |
+
if source not in headers:
|
| 153 |
+
raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}")
|
| 154 |
+
group = group_for[source]
|
| 155 |
+
value = value.detach().cpu().contiguous()
|
| 156 |
+
if name.endswith("w13_weight"):
|
| 157 |
+
value = (
|
| 158 |
+
value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape)
|
| 159 |
+
)
|
| 160 |
+
compute = group["compute_dtype"]
|
| 161 |
+
if compute == "nvfp4":
|
| 162 |
+
# Source E2M1 nibbles use per-16-element E4M3 scales, with each
|
| 163 |
+
# expert's outer scale stored as amax / denominator in FP32.
|
| 164 |
+
value = value.float().contiguous()
|
| 165 |
+
amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12)
|
| 166 |
+
outer = amax[:, None, None, None] / 1536.0
|
| 167 |
+
blocks = value.reshape(*value.shape[:-1], -1, 16)
|
| 168 |
+
scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn)
|
| 169 |
+
normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12)
|
| 170 |
+
levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6])
|
| 171 |
+
codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8)
|
| 172 |
+
codes |= (normalized < 0).to(torch.uint8) * 8
|
| 173 |
+
codes = codes.reshape(value.shape)
|
| 174 |
+
state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
|
| 175 |
+
state[name + "_scale"] = scales.contiguous()
|
| 176 |
+
state[name + "_amax"] = amax
|
| 177 |
+
state[name + "_outer_scale_denominator"] = torch.tensor([1536.0])
|
| 178 |
+
elif compute == "mxfp8":
|
| 179 |
+
# The pinned source stores UE8M0 scales per row/32 columns, even
|
| 180 |
+
# though its computation descriptor says weight_block_shape=[32,32].
|
| 181 |
+
blocks = value.float().reshape(*value.shape[:-1], -1, 32)
|
| 182 |
+
exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127)
|
| 183 |
+
scales = torch.pow(2.0, exponents)
|
| 184 |
+
state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn)
|
| 185 |
+
state[name + "_scale"] = (exponents + 127).to(torch.uint8)
|
| 186 |
+
else:
|
| 187 |
+
state[name] = value.to(getattr(torch, group["output_dtype"]))
|
| 188 |
+
|
| 189 |
+
# Build the independent DSpark draft using its authoritative BF16 header.
|
| 190 |
+
draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
|
| 191 |
+
draft = {}
|
| 192 |
+
draft_module = torch.nn.Module()
|
| 193 |
+
replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 24 * 32, 1024: 4 * 32, 128: 32}
|
| 194 |
+
for name, tensor in sorted(draft_metadata.tensors.items()):
|
| 195 |
+
if name == "fuse_weight":
|
| 196 |
+
shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
|
| 197 |
+
elif name == "markov_head.markov_w1.weight":
|
| 198 |
+
shape = (text["vocab_size"], mtp["markov_rank"])
|
| 199 |
+
elif name == "markov_head.hidden_out.weight":
|
| 200 |
+
shape = (hidden_size, mtp["markov_rank"])
|
| 201 |
+
elif name == "markov_head.rnn_proj.weight":
|
| 202 |
+
shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"])
|
| 203 |
+
elif name == "markov_head.rnn_proj.bias":
|
| 204 |
+
shape = (3 * mtp["markov_rank"],)
|
| 205 |
+
else:
|
| 206 |
+
shape = tuple(replacements.get(n, n) for n in tensor.shape)
|
| 207 |
+
value = torch.empty(shape, dtype=torch.bfloat16)
|
| 208 |
+
if name.endswith("bias"):
|
| 209 |
+
value.zero_()
|
| 210 |
+
elif "norm" in name:
|
| 211 |
+
value.fill_(1)
|
| 212 |
+
else:
|
| 213 |
+
value.normal_(0, 0.02)
|
| 214 |
+
draft[name] = value
|
| 215 |
+
module = draft_module
|
| 216 |
+
parts = name.split(".")
|
| 217 |
+
for part in parts[:-1]:
|
| 218 |
+
if part not in module._modules:
|
| 219 |
+
module.add_module(part, torch.nn.Module())
|
| 220 |
+
module = module._modules[part]
|
| 221 |
+
module.register_parameter(parts[-1], torch.nn.Parameter(value))
|
| 222 |
+
model.add_module("mtp", draft_module)
|
| 223 |
+
dtype_names = {
|
| 224 |
+
torch.float32: "F32",
|
| 225 |
+
torch.bfloat16: "BF16",
|
| 226 |
+
torch.uint8: "U8",
|
| 227 |
+
torch.float8_e4m3fn: "F8_E4M3",
|
| 228 |
+
}
|
| 229 |
+
for name, tensor in state.items():
|
| 230 |
+
source_tensor = headers[source_name(name)]
|
| 231 |
+
if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape):
|
| 232 |
+
raise ValueError(f"Source precision/rank mismatch: {name}")
|
| 233 |
+
if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all():
|
| 234 |
+
raise ValueError(f"Nonfinite checkpoint tensor: {name}")
|
| 235 |
+
save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
|
| 236 |
+
save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"})
|
| 237 |
+
|
| 238 |
+
# Remap every precision descriptor to exactly the retained source layers.
|
| 239 |
+
remapped_groups = []
|
| 240 |
+
for group in groups:
|
| 241 |
+
names = [name for name in state if source_name(name) in group["tensors"]]
|
| 242 |
+
if "model.llm.unembed.weight" in group["tensors"]:
|
| 243 |
+
# The adapter supplies this virtual tensor to the native loader;
|
| 244 |
+
# keep its FP32-logit computation descriptor even without a file copy.
|
| 245 |
+
names.append("model.llm.unembed.weight")
|
| 246 |
+
if names:
|
| 247 |
+
remapped_groups.append({**group, "tensors": sorted(names)})
|
| 248 |
+
precision["model"]["parameter_groups"] = remapped_groups
|
| 249 |
+
(folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n")
|
| 250 |
+
(folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
|
| 251 |
+
for name in (
|
| 252 |
+
"tokenizer.json",
|
| 253 |
+
"tokenizer_config.json",
|
| 254 |
+
"special_tokens_map.json",
|
| 255 |
+
"chat_template.jinja",
|
| 256 |
+
"processor_config.json",
|
| 257 |
+
"tiktoken/tokenizer.model",
|
| 258 |
+
):
|
| 259 |
+
path = folder / name
|
| 260 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 261 |
+
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
| 262 |
+
plugin_code = '''"""Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
|
| 263 |
+
|
| 264 |
+
import logging
|
| 265 |
+
|
| 266 |
+
logger = logging.getLogger(__name__)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _config_init(original, config, *args, **kwargs):
|
| 270 |
+
tied = kwargs.get("tie_word_embeddings", False)
|
| 271 |
+
if tied:
|
| 272 |
+
kwargs = {**kwargs, "tie_word_embeddings": False}
|
| 273 |
+
original(config, *args, **kwargs)
|
| 274 |
+
if tied:
|
| 275 |
+
config.tie_word_embeddings = True
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _load_weights(original, model, weights):
|
| 279 |
+
if not model.text_config.tie_word_embeddings or model.llm is None:
|
| 280 |
+
return original(model, weights)
|
| 281 |
+
|
| 282 |
+
def with_head():
|
| 283 |
+
embedding = None
|
| 284 |
+
for name, tensor in weights:
|
| 285 |
+
if name == "model.llm.unembed.weight":
|
| 286 |
+
raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
|
| 287 |
+
if name == "model.llm.embed.weight":
|
| 288 |
+
embedding = tensor
|
| 289 |
+
yield name, tensor
|
| 290 |
+
if embedding is None:
|
| 291 |
+
raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
|
| 292 |
+
yield "model.llm.unembed.weight", embedding
|
| 293 |
+
|
| 294 |
+
loaded = original(model, with_head())
|
| 295 |
+
embedding, head = model.llm.embed_tokens, model.llm.lm_head
|
| 296 |
+
if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
|
| 297 |
+
raise ValueError("Tied embedding/head tensor-parallel partitions must match")
|
| 298 |
+
head.weight = embedding.weight
|
| 299 |
+
assert head.weight is embedding.weight
|
| 300 |
+
logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
|
| 301 |
+
return loaded
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
class _PaddedMarlin(tuple):
|
| 305 |
+
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
| 306 |
+
value = super().__new__(cls, packed)
|
| 307 |
+
value.input_width = input_width
|
| 308 |
+
value.output_width = output_width
|
| 309 |
+
value.padded_input = padded_input
|
| 310 |
+
value.padded_output = padded_output
|
| 311 |
+
return value
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def _pad_matrix(tensor, rows, columns, fill=0):
|
| 315 |
+
output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
|
| 316 |
+
output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
|
| 317 |
+
return output
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def _repack_marlin(original, weight, scales):
|
| 321 |
+
import torch
|
| 322 |
+
|
| 323 |
+
nvfp4 = weight.dtype == torch.uint8
|
| 324 |
+
packing = 2 if nvfp4 else 1
|
| 325 |
+
block = 16 if nvfp4 else 32
|
| 326 |
+
n, k = weight.shape[-2], weight.shape[-1] * packing
|
| 327 |
+
padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
|
| 328 |
+
if (padded_k, padded_n) == (k, n):
|
| 329 |
+
return original(weight, scales)
|
| 330 |
+
weight = _pad_matrix(weight, padded_n, padded_k // packing)
|
| 331 |
+
# Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
|
| 332 |
+
# Zero weight bytes make padded products zero in both precision families.
|
| 333 |
+
scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
|
| 334 |
+
packed = original(weight, scales)
|
| 335 |
+
return _PaddedMarlin(packed, k, n, padded_k, padded_n)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
|
| 339 |
+
if not isinstance(packed, _PaddedMarlin):
|
| 340 |
+
return original(x, packed, out_size, *args, **kwargs)
|
| 341 |
+
if x.shape[-1] != packed.input_width or out_size != packed.output_width:
|
| 342 |
+
raise ValueError("Marlin padding metadata does not match the expert projection")
|
| 343 |
+
padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
|
| 344 |
+
padded[..., :x.shape[-1]].copy_(x)
|
| 345 |
+
if original.__name__ == "gemv" and packed.padded_input == 128:
|
| 346 |
+
# Eight K tiles fit one split of eight warps. The large-model heuristic
|
| 347 |
+
# otherwise chooses four splits/four warps for short decode batches.
|
| 348 |
+
kwargs = {**kwargs, "splits": 1, "warps": 8}
|
| 349 |
+
result = original(padded, packed, packed.padded_output, *args, **kwargs)
|
| 350 |
+
return result[..., :out_size].contiguous()
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _relative_fa4(original, *args, **kwargs):
|
| 354 |
+
q = args[0] if args else kwargs.get("q")
|
| 355 |
+
if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
|
| 356 |
+
and kwargs.get("tile_mn") is None):
|
| 357 |
+
# The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
|
| 358 |
+
# relative-bias kernel explicitly requires M=N=128. Use its tile API.
|
| 359 |
+
kwargs = {**kwargs, "tile_mn": (128, 128)}
|
| 360 |
+
return original(*args, **kwargs)
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def _attn_res_config(original, width, *args, **kwargs):
|
| 364 |
+
threads, stages, vector, cluster = original(width, *args, **kwargs)
|
| 365 |
+
if width == 32:
|
| 366 |
+
# One compute warp covers the original 32 residual coordinates.
|
| 367 |
+
# Keep the same CuTe kernel, RMS normalization, and source weighting.
|
| 368 |
+
return 32, stages, 1, 1
|
| 369 |
+
return threads, stages, vector, cluster
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def _small_all_reduce(original, x, group, **kwargs):
|
| 373 |
+
if x.ndim == 2 and x.shape[-1] == 32:
|
| 374 |
+
return group.all_reduce(x.clone())
|
| 375 |
+
return original(x, group, **kwargs)
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def _small_multicast_input(original, group, shape, dtype):
|
| 379 |
+
if len(shape) == 2 and shape[-1] == 32:
|
| 380 |
+
import torch
|
| 381 |
+
|
| 382 |
+
return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
|
| 383 |
+
return original(group, shape, dtype)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _small_multicast_reduce(original, x, group):
|
| 387 |
+
if x.ndim == 2 and x.shape[-1] == 32:
|
| 388 |
+
x.copy_(group.all_reduce(x))
|
| 389 |
+
return x
|
| 390 |
+
return original(x, group)
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
| 394 |
+
if cols == 32:
|
| 395 |
+
# Use the existing FP32 combine path instead of the GB300-tuned
|
| 396 |
+
# symmetric-memory one-shot collective for these tiny projections.
|
| 397 |
+
return None
|
| 398 |
+
return original(group, rows, cols, dtype, tag=tag)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def register():
|
| 402 |
+
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 403 |
+
|
| 404 |
+
HookRegistry.register(
|
| 405 |
+
"sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
|
| 406 |
+
)
|
| 407 |
+
HookRegistry.register(
|
| 408 |
+
"sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
|
| 409 |
+
)
|
| 410 |
+
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 411 |
+
for name in ("repack", "repack_mxfp8"):
|
| 412 |
+
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
| 413 |
+
for name in ("gemm", "gemv"):
|
| 414 |
+
HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
|
| 415 |
+
HookRegistry.register(
|
| 416 |
+
"sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
|
| 417 |
+
_relative_fa4, HookType.AROUND,
|
| 418 |
+
)
|
| 419 |
+
HookRegistry.register(
|
| 420 |
+
"sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
|
| 421 |
+
_attn_res_config, HookType.AROUND,
|
| 422 |
+
)
|
| 423 |
+
prefix = "sglang.kernels.ops.inkling2.comm"
|
| 424 |
+
for name, hook in (
|
| 425 |
+
("all_reduce", _small_all_reduce),
|
| 426 |
+
("multicast_all_reduce_input", _small_multicast_input),
|
| 427 |
+
("multicast_all_reduce", _small_multicast_reduce),
|
| 428 |
+
("oneshot_all_reduce_input", _small_oneshot_input),
|
| 429 |
+
):
|
| 430 |
+
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
| 431 |
+
'''
|
| 432 |
+
(folder / "sglang_inkling2_tied.py").write_text(plugin_code)
|
| 433 |
+
hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 434 |
+
|
| 435 |
+
from pathlib import Path
|
| 436 |
+
|
| 437 |
+
import torch
|
| 438 |
+
from huggingface_hub import hf_hub_download
|
| 439 |
+
from safetensors.torch import load_file
|
| 440 |
+
from transformers import Inkling2Config, Inkling2ForConditionalGeneration
|
| 441 |
+
from transformers.conversion_mapping import get_model_conversion_mapping
|
| 442 |
+
from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def dequantize(weight, scale, amax=None, denominator=None):
|
| 446 |
+
if weight.dtype == torch.uint8:
|
| 447 |
+
if amax is None or denominator is None:
|
| 448 |
+
raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
|
| 449 |
+
codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
|
| 450 |
+
levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
|
| 451 |
+
values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
|
| 452 |
+
return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
|
| 453 |
+
if weight.dtype == torch.float8_e4m3fn:
|
| 454 |
+
scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
|
| 455 |
+
return weight.float() * scales
|
| 456 |
+
raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
| 460 |
+
@classmethod
|
| 461 |
+
def from_pretrained(
|
| 462 |
+
cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
|
| 463 |
+
revision=None, cache_dir=None, local_files_only=False, token=None,
|
| 464 |
+
attn_implementation="eager", **kwargs,
|
| 465 |
+
):
|
| 466 |
+
if model_args:
|
| 467 |
+
raise ValueError("Inkling 2 does not take positional construction arguments")
|
| 468 |
+
options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
|
| 469 |
+
root = Path(pretrained_model_name_or_path)
|
| 470 |
+
|
| 471 |
+
def resolve(filename):
|
| 472 |
+
if root.is_dir():
|
| 473 |
+
return root / filename
|
| 474 |
+
return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
|
| 475 |
+
|
| 476 |
+
import json
|
| 477 |
+
|
| 478 |
+
raw = json.loads(resolve("config.json").read_text())
|
| 479 |
+
# The native config otherwise assumes the all-MXFP8 sibling. This
|
| 480 |
+
# adapter dequantizes each tensor according to its actual stored dtype.
|
| 481 |
+
raw.pop("quantization_config", None)
|
| 482 |
+
config = Inkling2Config(**raw)
|
| 483 |
+
config._attn_implementation = attn_implementation
|
| 484 |
+
dtype = kwargs.pop("torch_dtype", dtype)
|
| 485 |
+
if dtype in (None, "auto"):
|
| 486 |
+
dtype = torch.bfloat16
|
| 487 |
+
elif isinstance(dtype, str):
|
| 488 |
+
dtype = getattr(torch, dtype)
|
| 489 |
+
if dtype not in (torch.float32, torch.bfloat16, torch.float16):
|
| 490 |
+
raise ValueError("The Transformers adapter requires a floating compute dtype")
|
| 491 |
+
device_map = kwargs.pop("device_map", None)
|
| 492 |
+
if device_map == "auto":
|
| 493 |
+
device_map = "cuda" if torch.cuda.is_available() else "cpu"
|
| 494 |
+
if isinstance(device_map, dict):
|
| 495 |
+
if set(device_map) != {""}:
|
| 496 |
+
raise ValueError("Use a single device for this tiny Transformers adapter")
|
| 497 |
+
device_map = device_map[""]
|
| 498 |
+
allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
|
| 499 |
+
unexpected = set(kwargs) - allowed
|
| 500 |
+
if unexpected:
|
| 501 |
+
raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
|
| 502 |
+
model = Inkling2ForConditionalGeneration(config)
|
| 503 |
+
transforms = get_model_conversion_mapping(model)
|
| 504 |
+
renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
|
| 505 |
+
converters = [t for t in transforms if isinstance(t, WeightConverter)]
|
| 506 |
+
source = load_file(str(resolve("model.safetensors")))
|
| 507 |
+
state = {}
|
| 508 |
+
for name, weight in source.items():
|
| 509 |
+
if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
|
| 510 |
+
continue
|
| 511 |
+
if name + "_scale" in source:
|
| 512 |
+
weight = dequantize(
|
| 513 |
+
weight, source[name + "_scale"], source.get(name + "_amax"),
|
| 514 |
+
source.get(name + "_outer_scale_denominator"),
|
| 515 |
+
)
|
| 516 |
+
target, _ = rename_source_key(name, renamings, converters)
|
| 517 |
+
if name.endswith("w13_weight"):
|
| 518 |
+
# The source interleaves gate and up; Transformers chunks them.
|
| 519 |
+
e, rows, columns = weight.shape
|
| 520 |
+
weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
|
| 521 |
+
if target in state:
|
| 522 |
+
raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
|
| 523 |
+
keep_float32 = source[name].dtype == torch.float32 or any(
|
| 524 |
+
part in target for part in model._keep_in_fp32_modules_strict
|
| 525 |
+
)
|
| 526 |
+
target_dtype = torch.float32 if keep_float32 else dtype
|
| 527 |
+
state[target] = weight.to(target_dtype).contiguous()
|
| 528 |
+
state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
|
| 529 |
+
model.load_state_dict(state, strict=True, assign=True)
|
| 530 |
+
model.eval()
|
| 531 |
+
if device_map is not None:
|
| 532 |
+
model.to(device_map)
|
| 533 |
+
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 534 |
+
return model
|
| 535 |
+
'''
|
| 536 |
+
(folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
|
| 537 |
+
(folder / "pyproject.toml").write_text("""[build-system]
|
| 538 |
+
requires = ["setuptools>=68"]
|
| 539 |
+
build-backend = "setuptools.build_meta"
|
| 540 |
+
|
| 541 |
+
[project]
|
| 542 |
+
name = "sglang-inkling2-tied"
|
| 543 |
+
version = "0.1.0"
|
| 544 |
+
description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
|
| 545 |
+
requires-python = ">=3.10"
|
| 546 |
+
|
| 547 |
+
[project.entry-points."sglang.srt.plugins"]
|
| 548 |
+
inkling2_tied = "sglang_inkling2_tied:register"
|
| 549 |
+
|
| 550 |
+
[tool.setuptools]
|
| 551 |
+
py-modules = ["sglang_inkling2_tied"]
|
| 552 |
+
""")
|
| 553 |
+
return str(model)
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
if __name__ == "__main__":
|
| 557 |
+
create_model(
|
| 558 |
+
source_model_id="thinkingmachines/Inkling-2-Small",
|
| 559 |
+
source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
|
| 560 |
+
save_folder="/tmp/inkling-2-tiny-random",
|
| 561 |
+
)
|
.meta/env.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source_model": {
|
| 3 |
+
"name": "thinkingmachines/Inkling-2-Small",
|
| 4 |
+
"commit_id": "aac974122acd33ddb0d3e1f54433a9c5dd2249ce"
|
| 5 |
+
},
|
| 6 |
+
"environments": {
|
| 7 |
+
"creation": {
|
| 8 |
+
"python": "3.12.13",
|
| 9 |
+
"packages": {
|
| 10 |
+
"huggingface_hub": "2.3.0",
|
| 11 |
+
"safetensors": "0.8.0",
|
| 12 |
+
"tokenizers": "0.23.3",
|
| 13 |
+
"torch": "2.13.0+cu126",
|
| 14 |
+
"transformers": "5.23.0.dev0"
|
| 15 |
+
},
|
| 16 |
+
"cuda": "12.6"
|
| 17 |
+
},
|
| 18 |
+
"sglang": {
|
| 19 |
+
"python": "3.12.13",
|
| 20 |
+
"packages": {
|
| 21 |
+
"Pillow": "12.3.0",
|
| 22 |
+
"flashinfer-python": "0.7.4",
|
| 23 |
+
"huggingface_hub": "1.33.0",
|
| 24 |
+
"nvidia-cutlass-dsl": "4.8.0",
|
| 25 |
+
"safetensors": "0.8.0",
|
| 26 |
+
"sglang": "0.0.0.dev0",
|
| 27 |
+
"sglang-inkling2-tied": "0.1.0",
|
| 28 |
+
"sglang-kernel": "0.4.10",
|
| 29 |
+
"tiktoken": "0.14.0",
|
| 30 |
+
"tml-renderers": "0.4.1",
|
| 31 |
+
"tokenizers": "0.23.2",
|
| 32 |
+
"torch": "2.14.1+cu130",
|
| 33 |
+
"transformers": "5.19.0",
|
| 34 |
+
"triton": "3.8.0"
|
| 35 |
+
},
|
| 36 |
+
"cuda": "13.0",
|
| 37 |
+
"hardware": {
|
| 38 |
+
"gpu_model": "NVIDIA H20"
|
| 39 |
+
}
|
| 40 |
+
},
|
| 41 |
+
"transformers": {
|
| 42 |
+
"python": "3.12.13",
|
| 43 |
+
"packages": {
|
| 44 |
+
"Pillow": "12.3.0",
|
| 45 |
+
"huggingface_hub": "2.3.0",
|
| 46 |
+
"safetensors": "0.8.0",
|
| 47 |
+
"tokenizers": "0.23.3",
|
| 48 |
+
"torch": "2.13.0+cu126",
|
| 49 |
+
"transformers": "5.23.0.dev0"
|
| 50 |
+
},
|
| 51 |
+
"cuda": "12.6",
|
| 52 |
+
"hardware": {
|
| 53 |
+
"gpu_model": "NVIDIA H20"
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
}
|
.meta/model.log
ADDED
|
@@ -0,0 +1,400 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Inkling2ForConditionalGeneration(
|
| 2 |
+
(model): Inkling2Model(
|
| 3 |
+
(language_model): Inkling2TextModel(
|
| 4 |
+
(embed_tokens): Inkling2NormedEmbedding(
|
| 5 |
+
201024, 32
|
| 6 |
+
(embed_norm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 7 |
+
)
|
| 8 |
+
(layers): ModuleList(
|
| 9 |
+
(0): Inkling2SlidingDecoderLayer(
|
| 10 |
+
(mlp_input): Inkling2BlockAttentionResidual(
|
| 11 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 12 |
+
)
|
| 13 |
+
(self_attn): Inkling2SlidingAttention(
|
| 14 |
+
(q_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 15 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 16 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 17 |
+
(r_proj): Linear(in_features=32, out_features=384, bias=False)
|
| 18 |
+
(o_proj): Linear(in_features=768, out_features=32, bias=False)
|
| 19 |
+
(k_sconv): Inkling2ShortConvolution(
|
| 20 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 21 |
+
)
|
| 22 |
+
(v_sconv): Inkling2ShortConvolution(
|
| 23 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 24 |
+
)
|
| 25 |
+
(rel_logits_proj): Inkling2RelativeLogits()
|
| 26 |
+
(gate_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 27 |
+
(sink_proj): Linear(in_features=32, out_features=24, bias=False)
|
| 28 |
+
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 29 |
+
(k_l2_clip): Inkling2L2Clip()
|
| 30 |
+
(rel_l2_clip): Inkling2ScaledL2Clip()
|
| 31 |
+
(v_norm): Inkling2UnweightedRMSNorm()
|
| 32 |
+
)
|
| 33 |
+
(mlp): Inkling2SparseMoeBlock(
|
| 34 |
+
(experts): Inkling2Experts(
|
| 35 |
+
(act_fn): Inkling2SoftcappedSwiGLU(
|
| 36 |
+
(gate_softcap): Inkling2Softcap()
|
| 37 |
+
(up_softcap): Inkling2Softcap()
|
| 38 |
+
)
|
| 39 |
+
)
|
| 40 |
+
(gate): Inkling2TopKRouter(
|
| 41 |
+
(score_fn): Sigmoid()
|
| 42 |
+
)
|
| 43 |
+
)
|
| 44 |
+
(input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 45 |
+
(post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 46 |
+
(post_attention_convolution): Inkling2ShortConvolution(
|
| 47 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 48 |
+
)
|
| 49 |
+
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 50 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 51 |
+
)
|
| 52 |
+
)
|
| 53 |
+
(1-4): 4 x Inkling2SlidingDecoderLayer(
|
| 54 |
+
(attention_input): Inkling2BlockAttentionResidual(
|
| 55 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 56 |
+
)
|
| 57 |
+
(mlp_input): Inkling2BlockAttentionResidual(
|
| 58 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 59 |
+
)
|
| 60 |
+
(self_attn): Inkling2SlidingAttention(
|
| 61 |
+
(q_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 62 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 63 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 64 |
+
(r_proj): Linear(in_features=32, out_features=384, bias=False)
|
| 65 |
+
(o_proj): Linear(in_features=768, out_features=32, bias=False)
|
| 66 |
+
(k_sconv): Inkling2ShortConvolution(
|
| 67 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 68 |
+
)
|
| 69 |
+
(v_sconv): Inkling2ShortConvolution(
|
| 70 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 71 |
+
)
|
| 72 |
+
(rel_logits_proj): Inkling2RelativeLogits()
|
| 73 |
+
(gate_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 74 |
+
(sink_proj): Linear(in_features=32, out_features=24, bias=False)
|
| 75 |
+
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 76 |
+
(k_l2_clip): Inkling2L2Clip()
|
| 77 |
+
(rel_l2_clip): Inkling2ScaledL2Clip()
|
| 78 |
+
(v_norm): Inkling2UnweightedRMSNorm()
|
| 79 |
+
)
|
| 80 |
+
(mlp): Inkling2SparseMoeBlock(
|
| 81 |
+
(experts): Inkling2Experts(
|
| 82 |
+
(act_fn): Inkling2SoftcappedSwiGLU(
|
| 83 |
+
(gate_softcap): Inkling2Softcap()
|
| 84 |
+
(up_softcap): Inkling2Softcap()
|
| 85 |
+
)
|
| 86 |
+
)
|
| 87 |
+
(gate): Inkling2TopKRouter(
|
| 88 |
+
(score_fn): Sigmoid()
|
| 89 |
+
)
|
| 90 |
+
)
|
| 91 |
+
(input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 92 |
+
(post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 93 |
+
(post_attention_convolution): Inkling2ShortConvolution(
|
| 94 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 95 |
+
)
|
| 96 |
+
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 97 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 98 |
+
)
|
| 99 |
+
)
|
| 100 |
+
(5): Inkling2FullDecoderLayer(
|
| 101 |
+
(attention_input): Inkling2BlockAttentionResidual(
|
| 102 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 103 |
+
)
|
| 104 |
+
(mlp_input): Inkling2BlockAttentionResidual(
|
| 105 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 106 |
+
)
|
| 107 |
+
(self_attn): Inkling2FullAttention(
|
| 108 |
+
(q_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 109 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 110 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 111 |
+
(r_proj): Linear(in_features=32, out_features=384, bias=False)
|
| 112 |
+
(o_proj): Linear(in_features=768, out_features=32, bias=False)
|
| 113 |
+
(k_sconv): Inkling2DynamicShortConvolution(
|
| 114 |
+
(weight_proj): Linear(in_features=32, out_features=16, bias=False)
|
| 115 |
+
)
|
| 116 |
+
(v_sconv): Inkling2DynamicShortConvolution(
|
| 117 |
+
(weight_proj): Linear(in_features=32, out_features=16, bias=False)
|
| 118 |
+
)
|
| 119 |
+
(rel_logits_proj): Inkling2RelativeLogits()
|
| 120 |
+
(gate_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 121 |
+
(sink_proj): Linear(in_features=32, out_features=24, bias=False)
|
| 122 |
+
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 123 |
+
(k_l2_clip): Inkling2L2Clip()
|
| 124 |
+
(rel_l2_clip): Inkling2ScaledL2Clip()
|
| 125 |
+
(v_norm): Inkling2UnweightedRMSNorm()
|
| 126 |
+
)
|
| 127 |
+
(mlp): Inkling2SparseMoeBlock(
|
| 128 |
+
(experts): Inkling2Experts(
|
| 129 |
+
(act_fn): Inkling2SoftcappedSwiGLU(
|
| 130 |
+
(gate_softcap): Inkling2Softcap()
|
| 131 |
+
(up_softcap): Inkling2Softcap()
|
| 132 |
+
)
|
| 133 |
+
)
|
| 134 |
+
(gate): Inkling2TopKRouter(
|
| 135 |
+
(score_fn): Sigmoid()
|
| 136 |
+
)
|
| 137 |
+
)
|
| 138 |
+
(q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 139 |
+
(k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 140 |
+
(v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 141 |
+
(post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 142 |
+
(post_attention_convolution): Inkling2ShortConvolution(
|
| 143 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 144 |
+
)
|
| 145 |
+
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 146 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 147 |
+
)
|
| 148 |
+
)
|
| 149 |
+
(6-10): 5 x Inkling2SlidingDecoderLayer(
|
| 150 |
+
(attention_input): Inkling2BlockAttentionResidual(
|
| 151 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 152 |
+
)
|
| 153 |
+
(mlp_input): Inkling2BlockAttentionResidual(
|
| 154 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 155 |
+
)
|
| 156 |
+
(self_attn): Inkling2SlidingAttention(
|
| 157 |
+
(q_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 158 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 159 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 160 |
+
(r_proj): Linear(in_features=32, out_features=384, bias=False)
|
| 161 |
+
(o_proj): Linear(in_features=768, out_features=32, bias=False)
|
| 162 |
+
(k_sconv): Inkling2ShortConvolution(
|
| 163 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 164 |
+
)
|
| 165 |
+
(v_sconv): Inkling2ShortConvolution(
|
| 166 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 167 |
+
)
|
| 168 |
+
(rel_logits_proj): Inkling2RelativeLogits()
|
| 169 |
+
(gate_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 170 |
+
(sink_proj): Linear(in_features=32, out_features=24, bias=False)
|
| 171 |
+
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 172 |
+
(k_l2_clip): Inkling2L2Clip()
|
| 173 |
+
(rel_l2_clip): Inkling2ScaledL2Clip()
|
| 174 |
+
(v_norm): Inkling2UnweightedRMSNorm()
|
| 175 |
+
)
|
| 176 |
+
(mlp): Inkling2SparseMoeBlock(
|
| 177 |
+
(experts): Inkling2Experts(
|
| 178 |
+
(act_fn): Inkling2SoftcappedSwiGLU(
|
| 179 |
+
(gate_softcap): Inkling2Softcap()
|
| 180 |
+
(up_softcap): Inkling2Softcap()
|
| 181 |
+
)
|
| 182 |
+
)
|
| 183 |
+
(gate): Inkling2TopKRouter(
|
| 184 |
+
(score_fn): Sigmoid()
|
| 185 |
+
)
|
| 186 |
+
)
|
| 187 |
+
(input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 188 |
+
(post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 189 |
+
(post_attention_convolution): Inkling2ShortConvolution(
|
| 190 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 191 |
+
)
|
| 192 |
+
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 193 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 194 |
+
)
|
| 195 |
+
)
|
| 196 |
+
(11): Inkling2FullDecoderLayer(
|
| 197 |
+
(attention_input): Inkling2BlockAttentionResidual(
|
| 198 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 199 |
+
)
|
| 200 |
+
(mlp_input): Inkling2BlockAttentionResidual(
|
| 201 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 202 |
+
)
|
| 203 |
+
(self_attn): Inkling2FullAttention(
|
| 204 |
+
(q_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 205 |
+
(k_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 206 |
+
(v_proj): Linear(in_features=32, out_features=128, bias=False)
|
| 207 |
+
(r_proj): Linear(in_features=32, out_features=384, bias=False)
|
| 208 |
+
(o_proj): Linear(in_features=768, out_features=32, bias=False)
|
| 209 |
+
(k_sconv): Inkling2DynamicShortConvolution(
|
| 210 |
+
(weight_proj): Linear(in_features=32, out_features=16, bias=False)
|
| 211 |
+
)
|
| 212 |
+
(v_sconv): Inkling2DynamicShortConvolution(
|
| 213 |
+
(weight_proj): Linear(in_features=32, out_features=16, bias=False)
|
| 214 |
+
)
|
| 215 |
+
(rel_logits_proj): Inkling2RelativeLogits()
|
| 216 |
+
(gate_proj): Linear(in_features=32, out_features=768, bias=False)
|
| 217 |
+
(sink_proj): Linear(in_features=32, out_features=24, bias=False)
|
| 218 |
+
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 219 |
+
(k_l2_clip): Inkling2L2Clip()
|
| 220 |
+
(rel_l2_clip): Inkling2ScaledL2Clip()
|
| 221 |
+
(v_norm): Inkling2UnweightedRMSNorm()
|
| 222 |
+
)
|
| 223 |
+
(mlp): Inkling2SparseMoeBlock(
|
| 224 |
+
(experts): Inkling2Experts(
|
| 225 |
+
(act_fn): Inkling2SoftcappedSwiGLU(
|
| 226 |
+
(gate_softcap): Inkling2Softcap()
|
| 227 |
+
(up_softcap): Inkling2Softcap()
|
| 228 |
+
)
|
| 229 |
+
)
|
| 230 |
+
(gate): Inkling2TopKRouter(
|
| 231 |
+
(score_fn): Sigmoid()
|
| 232 |
+
)
|
| 233 |
+
)
|
| 234 |
+
(q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 235 |
+
(k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 236 |
+
(v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 237 |
+
(post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 238 |
+
(post_attention_convolution): Inkling2ShortConvolution(
|
| 239 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 240 |
+
)
|
| 241 |
+
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 242 |
+
(conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
|
| 243 |
+
)
|
| 244 |
+
)
|
| 245 |
+
)
|
| 246 |
+
(norm): Inkling2RMSNorm((32,), eps=1e-06)
|
| 247 |
+
(final_input): Inkling2BlockAttentionResidual(
|
| 248 |
+
(norm): Inkling2UnweightedRMSNorm()
|
| 249 |
+
)
|
| 250 |
+
)
|
| 251 |
+
(vision_tower): Inkling2VisionModel(
|
| 252 |
+
(embeddings): Inkling2VisionEmbeddings(
|
| 253 |
+
(patch_embedding): Linear(in_features=1200, out_features=32, bias=False)
|
| 254 |
+
(position_embedding): Embedding(1024, 32)
|
| 255 |
+
)
|
| 256 |
+
(pre_layernorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 257 |
+
(rotary_emb): Inkling2VisionRotaryEmbedding()
|
| 258 |
+
(layers): ModuleList(
|
| 259 |
+
(0-3): 4 x Inkling2VisionEncoderLayer(
|
| 260 |
+
(self_attn): Inkling2VisionAttention(
|
| 261 |
+
(k_proj): Linear(in_features=32, out_features=32, bias=True)
|
| 262 |
+
(v_proj): Linear(in_features=32, out_features=32, bias=True)
|
| 263 |
+
(q_proj): Linear(in_features=32, out_features=32, bias=True)
|
| 264 |
+
(out_proj): Linear(in_features=32, out_features=32, bias=True)
|
| 265 |
+
)
|
| 266 |
+
(mlp): Inkling2VisionMLP(
|
| 267 |
+
(activation_fn): GELUActivation()
|
| 268 |
+
(fc1): Linear(in_features=32, out_features=128, bias=True)
|
| 269 |
+
(fc2): Linear(in_features=128, out_features=32, bias=True)
|
| 270 |
+
)
|
| 271 |
+
(layernorm_before): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 272 |
+
(layernorm_after): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
|
| 273 |
+
)
|
| 274 |
+
)
|
| 275 |
+
(merger): Inkling2VisionMerger(
|
| 276 |
+
(linear_1): Linear(in_features=128, out_features=32, bias=False)
|
| 277 |
+
(act): GELUActivation()
|
| 278 |
+
(linear_2): Linear(in_features=32, out_features=32, bias=False)
|
| 279 |
+
)
|
| 280 |
+
)
|
| 281 |
+
)
|
| 282 |
+
(lm_head): Linear(in_features=32, out_features=201024, bias=False)
|
| 283 |
+
(logit_softcap): Inkling2Softcap()
|
| 284 |
+
(mtp): Module(
|
| 285 |
+
(layers): Module(
|
| 286 |
+
(0): Module(
|
| 287 |
+
(dflare_norm): Module()
|
| 288 |
+
(input_layernorm): Module()
|
| 289 |
+
(mlp): Module(
|
| 290 |
+
(down_proj): Module()
|
| 291 |
+
(gate_proj): Module()
|
| 292 |
+
(up_proj): Module()
|
| 293 |
+
)
|
| 294 |
+
(post_attention_layernorm): Module()
|
| 295 |
+
(self_attn): Module(
|
| 296 |
+
(k_norm): Module()
|
| 297 |
+
(k_norm_ctx): Module()
|
| 298 |
+
(k_proj): Module()
|
| 299 |
+
(k_proj_ctx): Module()
|
| 300 |
+
(o_proj): Module()
|
| 301 |
+
(q_norm): Module()
|
| 302 |
+
(q_proj): Module()
|
| 303 |
+
(v_proj): Module()
|
| 304 |
+
(v_proj_ctx): Module()
|
| 305 |
+
)
|
| 306 |
+
)
|
| 307 |
+
(1): Module(
|
| 308 |
+
(dflare_norm): Module()
|
| 309 |
+
(input_layernorm): Module()
|
| 310 |
+
(mlp): Module(
|
| 311 |
+
(down_proj): Module()
|
| 312 |
+
(gate_proj): Module()
|
| 313 |
+
(up_proj): Module()
|
| 314 |
+
)
|
| 315 |
+
(post_attention_layernorm): Module()
|
| 316 |
+
(self_attn): Module(
|
| 317 |
+
(k_norm): Module()
|
| 318 |
+
(k_norm_ctx): Module()
|
| 319 |
+
(k_proj): Module()
|
| 320 |
+
(k_proj_ctx): Module()
|
| 321 |
+
(o_proj): Module()
|
| 322 |
+
(q_norm): Module()
|
| 323 |
+
(q_proj): Module()
|
| 324 |
+
(v_proj): Module()
|
| 325 |
+
(v_proj_ctx): Module()
|
| 326 |
+
)
|
| 327 |
+
)
|
| 328 |
+
(2): Module(
|
| 329 |
+
(dflare_norm): Module()
|
| 330 |
+
(input_layernorm): Module()
|
| 331 |
+
(mlp): Module(
|
| 332 |
+
(down_proj): Module()
|
| 333 |
+
(gate_proj): Module()
|
| 334 |
+
(up_proj): Module()
|
| 335 |
+
)
|
| 336 |
+
(post_attention_layernorm): Module()
|
| 337 |
+
(self_attn): Module(
|
| 338 |
+
(k_norm): Module()
|
| 339 |
+
(k_norm_ctx): Module()
|
| 340 |
+
(k_proj): Module()
|
| 341 |
+
(k_proj_ctx): Module()
|
| 342 |
+
(o_proj): Module()
|
| 343 |
+
(q_norm): Module()
|
| 344 |
+
(q_proj): Module()
|
| 345 |
+
(v_proj): Module()
|
| 346 |
+
(v_proj_ctx): Module()
|
| 347 |
+
)
|
| 348 |
+
)
|
| 349 |
+
(3): Module(
|
| 350 |
+
(dflare_norm): Module()
|
| 351 |
+
(input_layernorm): Module()
|
| 352 |
+
(mlp): Module(
|
| 353 |
+
(down_proj): Module()
|
| 354 |
+
(gate_proj): Module()
|
| 355 |
+
(up_proj): Module()
|
| 356 |
+
)
|
| 357 |
+
(post_attention_layernorm): Module()
|
| 358 |
+
(self_attn): Module(
|
| 359 |
+
(k_norm): Module()
|
| 360 |
+
(k_norm_ctx): Module()
|
| 361 |
+
(k_proj): Module()
|
| 362 |
+
(k_proj_ctx): Module()
|
| 363 |
+
(o_proj): Module()
|
| 364 |
+
(q_norm): Module()
|
| 365 |
+
(q_proj): Module()
|
| 366 |
+
(v_proj): Module()
|
| 367 |
+
(v_proj_ctx): Module()
|
| 368 |
+
)
|
| 369 |
+
)
|
| 370 |
+
(4): Module(
|
| 371 |
+
(dflare_norm): Module()
|
| 372 |
+
(input_layernorm): Module()
|
| 373 |
+
(mlp): Module(
|
| 374 |
+
(down_proj): Module()
|
| 375 |
+
(gate_proj): Module()
|
| 376 |
+
(up_proj): Module()
|
| 377 |
+
)
|
| 378 |
+
(post_attention_layernorm): Module()
|
| 379 |
+
(self_attn): Module(
|
| 380 |
+
(k_norm): Module()
|
| 381 |
+
(k_norm_ctx): Module()
|
| 382 |
+
(k_proj): Module()
|
| 383 |
+
(k_proj_ctx): Module()
|
| 384 |
+
(o_proj): Module()
|
| 385 |
+
(q_norm): Module()
|
| 386 |
+
(q_proj): Module()
|
| 387 |
+
(v_proj): Module()
|
| 388 |
+
(v_proj_ctx): Module()
|
| 389 |
+
)
|
| 390 |
+
)
|
| 391 |
+
)
|
| 392 |
+
(markov_head): Module(
|
| 393 |
+
(dflare_norm): Module()
|
| 394 |
+
(hidden_out): Module()
|
| 395 |
+
(markov_w1): Module()
|
| 396 |
+
(rnn_proj): Module()
|
| 397 |
+
)
|
| 398 |
+
(norm): Module()
|
| 399 |
+
)
|
| 400 |
+
)
|
README.md
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: transformers
|
| 3 |
+
pipeline_tag: image-text-to-text
|
| 4 |
+
base_model:
|
| 5 |
+
- thinkingmachines/Inkling-2-Small
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-Small](https://huggingface.co/thinkingmachines/Inkling-2-Small) for testing and debugging.
|
| 9 |
+
|
| 10 |
+
| Feature | Source | Tiny |
|
| 11 |
+
|---|---|---|
|
| 12 |
+
| Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 9.97M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
|
| 13 |
+
| Safetensors weight files (including MTP) | 393.48 GB (25 files) | 22.82 MB (2 files) |
|
| 14 |
+
| Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
|
| 15 |
+
| Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
|
| 16 |
+
| Routed experts / experts per token | 384 / 8 | 96 / 8 |
|
| 17 |
+
| Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
|
| 18 |
+
| Vocabulary / tied embeddings | 201,024 / false | 201,024 / true |
|
| 19 |
+
| Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
|
| 20 |
+
| Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
|
| 21 |
+
| Vision width / layers | 1,024 / 23 | 32 / 4 |
|
| 22 |
+
| MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
|
| 23 |
+
| MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
|
| 24 |
+
|
| 25 |
+
| File path | Size |
|
| 26 |
+
|------|------|
|
| 27 |
+
| model.safetensors | 18.8MB |
|
| 28 |
+
| mtp.safetensors | 4.0MB |
|
| 29 |
+
|
| 30 |
+
### Example usage
|
| 31 |
+
|
| 32 |
+
#### Transformers (✅ verified)
|
| 33 |
+
|
| 34 |
+
Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
|
| 35 |
+
|
| 36 |
+
```python
|
| 37 |
+
def run_transformers(model_id: str):
|
| 38 |
+
import torch
|
| 39 |
+
from PIL import Image
|
| 40 |
+
from transformers import AutoModelForImageTextToText, AutoProcessor
|
| 41 |
+
|
| 42 |
+
processor = AutoProcessor.from_pretrained(model_id)
|
| 43 |
+
model = AutoModelForImageTextToText.from_pretrained(
|
| 44 |
+
model_id, trust_remote_code=True, dtype=torch.bfloat16, device_map="cuda"
|
| 45 |
+
)
|
| 46 |
+
image = Image.new("RGB", (80, 80), (80, 120, 160))
|
| 47 |
+
inputs = processor(text=f"{processor.image_token}Describe this image.", images=image, return_tensors="pt").to(
|
| 48 |
+
model.device
|
| 49 |
+
)
|
| 50 |
+
with torch.inference_mode():
|
| 51 |
+
output = model.generate(**inputs, max_new_tokens=10, min_new_tokens=10, do_sample=False)
|
| 52 |
+
print(processor.batch_decode(output[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
if __name__ == "__main__":
|
| 56 |
+
run_transformers(model_id="tiny-random/inkling-2")
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
#### SGLang serving (✅ verified)
|
| 60 |
+
|
| 61 |
+
Use the Inkling 2 support branch of SGLang; the general release does not contain this architecture.
|
| 62 |
+
|
| 63 |
+
```bash
|
| 64 |
+
SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
|
| 65 |
+
hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
|
| 66 |
+
pip install --no-deps ./inkling-2-tiny
|
| 67 |
+
sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
|
| 68 |
+
--dtype bfloat16 --attention-backend fa4 --enable-multimodal \
|
| 69 |
+
--context-length 2048 --mem-fraction-static 0.2 \
|
| 70 |
+
--disable-custom-all-reduce \
|
| 71 |
+
--cuda-graph-config '{"decode": {"bs": [1, 2, 4]}, "prefill": {"bs": [128, 2048]}}' \
|
| 72 |
+
--reasoning-parser inkling --tool-call-parser inkling2
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
### Creating this repo
|
| 76 |
+
|
| 77 |
+
- Code: [.meta/create_model.py](.meta/create_model.py)
|
| 78 |
+
- Model structure: [.meta/model.log](.meta/model.log)
|
| 79 |
+
- Environment: [.meta/env.json](.meta/env.json)
|
| 80 |
+
|
| 81 |
+
### Change log
|
| 82 |
+
|
| 83 |
+
- 2026-11-13: Created tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, tied token embeddings, and BF16 MTP.
|
| 84 |
+
|
| 85 |
+
*This repo is generated by GPT-6.1-Sol*
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "xhigh": 0.99, "max": 0.99} -%}
|
| 2 |
+
{%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
|
| 3 |
+
|
| 4 |
+
{%- macro emit_thinking_effort() -%}
|
| 5 |
+
{%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
|
| 6 |
+
{%- if eff is string -%}
|
| 7 |
+
{%- set key = eff | trim -%}
|
| 8 |
+
{%- if key not in effort_map -%}
|
| 9 |
+
{{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
|
| 10 |
+
{%- endif -%}
|
| 11 |
+
{%- set num = effort_map[key] -%}
|
| 12 |
+
{%- else -%}
|
| 13 |
+
{%- set num = eff | float -%}
|
| 14 |
+
{%- endif -%}
|
| 15 |
+
{%- if num < 0.0 or num > 0.99 -%}
|
| 16 |
+
{{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
|
| 17 |
+
{%- endif -%}
|
| 18 |
+
{{- "<|message_system|><|content_text|>Thinking effort level: " -}}
|
| 19 |
+
{%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
|
| 20 |
+
{{- "<|end_message|>" -}}
|
| 21 |
+
{%- endmacro -%}
|
| 22 |
+
|
| 23 |
+
{%- if tools -%}
|
| 24 |
+
{%- set tool_state = namespace(specs=[]) -%}
|
| 25 |
+
{%- for tool in tools -%}
|
| 26 |
+
{%- set fn = tool.function if tool.function is defined else tool -%}
|
| 27 |
+
{%- set spec = {
|
| 28 |
+
"description": (fn.description if fn.description is defined and fn.description else ""),
|
| 29 |
+
"name": fn.name,
|
| 30 |
+
"parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
|
| 31 |
+
"type": (tool.type if tool.type is defined and tool.type else "function"),
|
| 32 |
+
} -%}
|
| 33 |
+
{%- set tool_state.specs = tool_state.specs + [spec] -%}
|
| 34 |
+
{%- endfor -%}
|
| 35 |
+
{{- "<|message_system|>tool_declare<|content_xml|>" -}}
|
| 36 |
+
{{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
|
| 37 |
+
{{- "<|end_message|>" -}}
|
| 38 |
+
{%- endif -%}
|
| 39 |
+
|
| 40 |
+
{%- set state = namespace(effort_emitted=false) -%}
|
| 41 |
+
{%- for message in messages -%}
|
| 42 |
+
{%- if message.role not in role_token -%}
|
| 43 |
+
{{- raise_exception("Unknown message role: " ~ message.role) -}}
|
| 44 |
+
{%- endif -%}
|
| 45 |
+
{%- if not state.effort_emitted and message.role != "system" -%}
|
| 46 |
+
{{- emit_thinking_effort() -}}
|
| 47 |
+
{%- set state.effort_emitted = true -%}
|
| 48 |
+
{%- endif -%}
|
| 49 |
+
|
| 50 |
+
{%- set header = namespace(text=role_token[message.role]) -%}
|
| 51 |
+
{%- if message.role == "tool" -%}
|
| 52 |
+
{%- if message.name is defined and message.name -%}
|
| 53 |
+
{%- set header.text = header.text ~ message.name -%}
|
| 54 |
+
{%- elif message.tool_call_id is defined and message.tool_call_id -%}
|
| 55 |
+
{%- for prev in messages -%}
|
| 56 |
+
{%- if prev.role == "assistant" and prev.tool_calls -%}
|
| 57 |
+
{%- for tc in prev.tool_calls -%}
|
| 58 |
+
{%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
|
| 59 |
+
{%- set header.text = header.text ~ tc.function.name -%}
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
{%- endfor -%}
|
| 62 |
+
{%- endif -%}
|
| 63 |
+
{%- endfor -%}
|
| 64 |
+
{%- endif -%}
|
| 65 |
+
{%- elif message.role == "assistant" and message.name is defined and message.name -%}
|
| 66 |
+
{%- set header.text = header.text ~ message.name -%}
|
| 67 |
+
{%- endif -%}
|
| 68 |
+
{%- set rtok = header.text -%}
|
| 69 |
+
|
| 70 |
+
{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
|
| 71 |
+
{{- rtok ~ "<|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
|
| 72 |
+
{%- endif -%}
|
| 73 |
+
|
| 74 |
+
{%- if message.content is string and (message.content or not (message.role == "assistant" and (message.tool_calls or message.reasoning_content))) -%}
|
| 75 |
+
{{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
|
| 76 |
+
{%- elif message.content -%}
|
| 77 |
+
{%- for part in message.content -%}
|
| 78 |
+
{%- if part is string -%}
|
| 79 |
+
{{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
|
| 80 |
+
{%- elif part.type is not defined or part.type in ("text", "input_text") -%}
|
| 81 |
+
{%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
|
| 82 |
+
{{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
|
| 83 |
+
{%- elif part.type in ("image", "input_image", "image_url") -%}
|
| 84 |
+
{%- if message.role not in ("user", "tool") -%}
|
| 85 |
+
{{- raise_exception("Image content is only supported for user and tool messages") -}}
|
| 86 |
+
{%- endif -%}
|
| 87 |
+
{{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
|
| 88 |
+
{%- elif part.type in ("audio", "input_audio", "audio_url") -%}
|
| 89 |
+
{{- raise_exception("Inkling 2 does not accept audio input") -}}
|
| 90 |
+
{%- else -%}
|
| 91 |
+
{{- raise_exception("Unsupported content part type: " ~ part.type) -}}
|
| 92 |
+
{%- endif -%}
|
| 93 |
+
{%- endfor -%}
|
| 94 |
+
{%- endif -%}
|
| 95 |
+
|
| 96 |
+
{%- if message.role == "assistant" and message.tool_calls -%}
|
| 97 |
+
{%- for tc in message.tool_calls -%}
|
| 98 |
+
{%- set fn = tc.function -%}
|
| 99 |
+
{%- if fn.name is not defined or fn.name is not string -%}
|
| 100 |
+
{{- raise_exception("tool call function name must be a string") -}}
|
| 101 |
+
{%- endif -%}
|
| 102 |
+
{%- if message.name is defined and message.name and message.name != fn.name -%}
|
| 103 |
+
{{- raise_exception("assistant name " ~ message.name ~ " does not match its tool call " ~ fn.name) -}}
|
| 104 |
+
{%- endif -%}
|
| 105 |
+
{%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
|
| 106 |
+
{%- if args is string -%}
|
| 107 |
+
{{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
|
| 108 |
+
{%- endif -%}
|
| 109 |
+
{%- if args is not mapping -%}
|
| 110 |
+
{{- raise_exception("tool call arguments must be an object") -}}
|
| 111 |
+
{%- endif -%}
|
| 112 |
+
{{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
|
| 113 |
+
{{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
|
| 114 |
+
{{- "{" -}}
|
| 115 |
+
{%- for key, value in args | dictsort(case_sensitive=true) -%}
|
| 116 |
+
{{- (key | tojson) ~ ":" ~ (value | tojson(separators=(",", ":"))) -}}
|
| 117 |
+
{%- if not loop.last -%},{%- endif -%}
|
| 118 |
+
{%- endfor -%}
|
| 119 |
+
{{- "}}<|end_message|>" -}}
|
| 120 |
+
{%- endfor -%}
|
| 121 |
+
{%- endif -%}
|
| 122 |
+
|
| 123 |
+
{%- if message.role == "assistant" -%}
|
| 124 |
+
{{- "<|content_model_end_sampling|>" -}}
|
| 125 |
+
{%- endif -%}
|
| 126 |
+
{%- endfor -%}
|
| 127 |
+
|
| 128 |
+
{%- if not state.effort_emitted -%}
|
| 129 |
+
{{- emit_thinking_effort() -}}
|
| 130 |
+
{%- endif -%}
|
config.json
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Inkling2ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"model_type": "inkling2",
|
| 6 |
+
"eos_token_id": 200006,
|
| 7 |
+
"text_config": {
|
| 8 |
+
"model_max_length": 1048576,
|
| 9 |
+
"torch_dtype": "bfloat16",
|
| 10 |
+
"hidden_size": 32,
|
| 11 |
+
"num_hidden_layers": 12,
|
| 12 |
+
"vocab_size": 201024,
|
| 13 |
+
"num_attention_heads": 24,
|
| 14 |
+
"num_key_value_heads": 4,
|
| 15 |
+
"head_dim": 32,
|
| 16 |
+
"d_rel": 16,
|
| 17 |
+
"rel_extent": 1024,
|
| 18 |
+
"q_bias": false,
|
| 19 |
+
"o_bias": false,
|
| 20 |
+
"log_scaling_n_floor": 128000,
|
| 21 |
+
"log_scaling_alpha": 0.1,
|
| 22 |
+
"rms_norm_eps": 1e-06,
|
| 23 |
+
"pre_norm_pre_affine": false,
|
| 24 |
+
"final_norm_pre_affine": false,
|
| 25 |
+
"use_value_norm_no_gamma": true,
|
| 26 |
+
"qk_l2_clip_threshold": 22.627416997969522,
|
| 27 |
+
"rel_l2_clip_threshold": 8.0,
|
| 28 |
+
"local_layer_ids": [
|
| 29 |
+
0,
|
| 30 |
+
1,
|
| 31 |
+
2,
|
| 32 |
+
3,
|
| 33 |
+
4,
|
| 34 |
+
6,
|
| 35 |
+
7,
|
| 36 |
+
8,
|
| 37 |
+
9,
|
| 38 |
+
10
|
| 39 |
+
],
|
| 40 |
+
"sconv_kernel_size": 4,
|
| 41 |
+
"unpadded_vocab_size": 200058,
|
| 42 |
+
"final_logit_softcapping": 30.0,
|
| 43 |
+
"swa_head_dim": 32,
|
| 44 |
+
"swa_num_attention_heads": 24,
|
| 45 |
+
"swa_num_key_value_heads": 4,
|
| 46 |
+
"sliding_window_size": 1024,
|
| 47 |
+
"n_layers_per_attn_res_block": 6,
|
| 48 |
+
"attn_res_normalize_values": true,
|
| 49 |
+
"n_routed_experts": 96,
|
| 50 |
+
"num_experts_per_tok": 8,
|
| 51 |
+
"n_shared_experts": 0,
|
| 52 |
+
"intermediate_size": 32,
|
| 53 |
+
"mlp_gate_softcap": 4.0,
|
| 54 |
+
"mlp_up_softcap": 25.0,
|
| 55 |
+
"route_scale": 8.0,
|
| 56 |
+
"use_gate_bias": true,
|
| 57 |
+
"tie_word_embeddings": true
|
| 58 |
+
},
|
| 59 |
+
"vision_config": {
|
| 60 |
+
"temporal_patch_size": 2,
|
| 61 |
+
"native_patch_size": 20,
|
| 62 |
+
"adapter_merge_size": 2,
|
| 63 |
+
"projection_hidden_dim": 32,
|
| 64 |
+
"projector_bias": false,
|
| 65 |
+
"hidden_size": 32,
|
| 66 |
+
"num_hidden_layers": 4,
|
| 67 |
+
"num_attention_heads": 4,
|
| 68 |
+
"mlp_ratio": 4.0,
|
| 69 |
+
"posemb_grid_size": 32,
|
| 70 |
+
"layer_norm_eps": 1e-05,
|
| 71 |
+
"text_hidden_size": 32
|
| 72 |
+
},
|
| 73 |
+
"mtp_config": {
|
| 74 |
+
"hidden_size": 32,
|
| 75 |
+
"num_hidden_layers": 5,
|
| 76 |
+
"num_attention_heads": 24,
|
| 77 |
+
"num_key_value_heads": 4,
|
| 78 |
+
"head_dim": 32,
|
| 79 |
+
"intermediate_size": 128,
|
| 80 |
+
"max_position_embeddings": 1048576,
|
| 81 |
+
"torch_dtype": "bfloat16",
|
| 82 |
+
"q_bias": false,
|
| 83 |
+
"o_bias": false,
|
| 84 |
+
"rms_norm_eps": 1e-06,
|
| 85 |
+
"rope_theta": 1000000.0,
|
| 86 |
+
"local_layer_ids": [
|
| 87 |
+
0,
|
| 88 |
+
1,
|
| 89 |
+
3,
|
| 90 |
+
4
|
| 91 |
+
],
|
| 92 |
+
"sliding_window_size": 1024,
|
| 93 |
+
"block_size": 8,
|
| 94 |
+
"markov_rank": 8,
|
| 95 |
+
"markov_head_type": "rnn_hidden",
|
| 96 |
+
"target_layer_ids": [
|
| 97 |
+
5,
|
| 98 |
+
12
|
| 99 |
+
]
|
| 100 |
+
},
|
| 101 |
+
"tie_word_embeddings": true,
|
| 102 |
+
"auto_map": {
|
| 103 |
+
"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
|
| 104 |
+
}
|
| 105 |
+
}
|
hf_quant_config.json
ADDED
|
@@ -0,0 +1,525 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": {
|
| 3 |
+
"parameter_groups": [
|
| 4 |
+
{
|
| 5 |
+
"compute_dtype": "float32",
|
| 6 |
+
"output_dtype": "float32",
|
| 7 |
+
"required_activation_quantization": null,
|
| 8 |
+
"tensors": [
|
| 9 |
+
"model.llm.attn_res.attn_queries.0",
|
| 10 |
+
"model.llm.attn_res.attn_queries.1",
|
| 11 |
+
"model.llm.attn_res.attn_queries.10",
|
| 12 |
+
"model.llm.attn_res.attn_queries.2",
|
| 13 |
+
"model.llm.attn_res.attn_queries.3",
|
| 14 |
+
"model.llm.attn_res.attn_queries.4",
|
| 15 |
+
"model.llm.attn_res.attn_queries.5",
|
| 16 |
+
"model.llm.attn_res.attn_queries.6",
|
| 17 |
+
"model.llm.attn_res.attn_queries.7",
|
| 18 |
+
"model.llm.attn_res.attn_queries.8",
|
| 19 |
+
"model.llm.attn_res.attn_queries.9",
|
| 20 |
+
"model.llm.attn_res.mlp_queries.0",
|
| 21 |
+
"model.llm.attn_res.mlp_queries.1",
|
| 22 |
+
"model.llm.attn_res.mlp_queries.10",
|
| 23 |
+
"model.llm.attn_res.mlp_queries.11",
|
| 24 |
+
"model.llm.attn_res.mlp_queries.2",
|
| 25 |
+
"model.llm.attn_res.mlp_queries.3",
|
| 26 |
+
"model.llm.attn_res.mlp_queries.4",
|
| 27 |
+
"model.llm.attn_res.mlp_queries.5",
|
| 28 |
+
"model.llm.attn_res.mlp_queries.6",
|
| 29 |
+
"model.llm.attn_res.mlp_queries.7",
|
| 30 |
+
"model.llm.attn_res.mlp_queries.8",
|
| 31 |
+
"model.llm.attn_res.mlp_queries.9",
|
| 32 |
+
"model.llm.attn_res.output_query",
|
| 33 |
+
"model.llm.layers.0.mlp.gate.bias",
|
| 34 |
+
"model.llm.layers.1.mlp.gate.bias",
|
| 35 |
+
"model.llm.layers.10.mlp.gate.bias",
|
| 36 |
+
"model.llm.layers.11.mlp.gate.bias",
|
| 37 |
+
"model.llm.layers.2.mlp.gate.bias",
|
| 38 |
+
"model.llm.layers.3.mlp.gate.bias",
|
| 39 |
+
"model.llm.layers.4.mlp.gate.bias",
|
| 40 |
+
"model.llm.layers.5.mlp.gate.bias",
|
| 41 |
+
"model.llm.layers.6.mlp.gate.bias",
|
| 42 |
+
"model.llm.layers.7.mlp.gate.bias",
|
| 43 |
+
"model.llm.layers.8.mlp.gate.bias",
|
| 44 |
+
"model.llm.layers.9.mlp.gate.bias"
|
| 45 |
+
],
|
| 46 |
+
"weight_block_shape": [],
|
| 47 |
+
"weight_transform": "cast"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"compute_dtype": "bfloat16",
|
| 51 |
+
"output_dtype": "bfloat16",
|
| 52 |
+
"required_activation_quantization": null,
|
| 53 |
+
"tensors": [
|
| 54 |
+
"model.llm.embed.weight",
|
| 55 |
+
"model.llm.embed_norm.weight",
|
| 56 |
+
"model.llm.layers.0.attn.attn_sink_proj.weight",
|
| 57 |
+
"model.llm.layers.0.attn.gate_attn.weight",
|
| 58 |
+
"model.llm.layers.0.attn.k_sconv.weight",
|
| 59 |
+
"model.llm.layers.0.attn.q_l2_clip.scale",
|
| 60 |
+
"model.llm.layers.0.attn.rel_l2_clip.scale",
|
| 61 |
+
"model.llm.layers.0.attn.rel_logits_proj.proj",
|
| 62 |
+
"model.llm.layers.0.attn.v_sconv.weight",
|
| 63 |
+
"model.llm.layers.0.attn.wk_dv.weight",
|
| 64 |
+
"model.llm.layers.0.attn.wo_ud.weight",
|
| 65 |
+
"model.llm.layers.0.attn.wq_du.weight",
|
| 66 |
+
"model.llm.layers.0.attn.wr_du.weight",
|
| 67 |
+
"model.llm.layers.0.attn.wv_dv.weight",
|
| 68 |
+
"model.llm.layers.0.attn_norm.weight",
|
| 69 |
+
"model.llm.layers.0.attn_sconv.weight",
|
| 70 |
+
"model.llm.layers.0.mlp.gate.weight",
|
| 71 |
+
"model.llm.layers.0.mlp_norm.weight",
|
| 72 |
+
"model.llm.layers.0.mlp_sconv.weight",
|
| 73 |
+
"model.llm.layers.1.attn.attn_sink_proj.weight",
|
| 74 |
+
"model.llm.layers.1.attn.gate_attn.weight",
|
| 75 |
+
"model.llm.layers.1.attn.k_sconv.weight",
|
| 76 |
+
"model.llm.layers.1.attn.q_l2_clip.scale",
|
| 77 |
+
"model.llm.layers.1.attn.rel_l2_clip.scale",
|
| 78 |
+
"model.llm.layers.1.attn.rel_logits_proj.proj",
|
| 79 |
+
"model.llm.layers.1.attn.v_sconv.weight",
|
| 80 |
+
"model.llm.layers.1.attn.wk_dv.weight",
|
| 81 |
+
"model.llm.layers.1.attn.wo_ud.weight",
|
| 82 |
+
"model.llm.layers.1.attn.wq_du.weight",
|
| 83 |
+
"model.llm.layers.1.attn.wr_du.weight",
|
| 84 |
+
"model.llm.layers.1.attn.wv_dv.weight",
|
| 85 |
+
"model.llm.layers.1.attn_norm.weight",
|
| 86 |
+
"model.llm.layers.1.attn_sconv.weight",
|
| 87 |
+
"model.llm.layers.1.mlp.gate.weight",
|
| 88 |
+
"model.llm.layers.1.mlp_norm.weight",
|
| 89 |
+
"model.llm.layers.1.mlp_sconv.weight",
|
| 90 |
+
"model.llm.layers.10.attn.attn_sink_proj.weight",
|
| 91 |
+
"model.llm.layers.10.attn.gate_attn.weight",
|
| 92 |
+
"model.llm.layers.10.attn.k_sconv.weight",
|
| 93 |
+
"model.llm.layers.10.attn.q_l2_clip.scale",
|
| 94 |
+
"model.llm.layers.10.attn.rel_l2_clip.scale",
|
| 95 |
+
"model.llm.layers.10.attn.rel_logits_proj.proj",
|
| 96 |
+
"model.llm.layers.10.attn.v_sconv.weight",
|
| 97 |
+
"model.llm.layers.10.attn.wk_dv.weight",
|
| 98 |
+
"model.llm.layers.10.attn.wo_ud.weight",
|
| 99 |
+
"model.llm.layers.10.attn.wq_du.weight",
|
| 100 |
+
"model.llm.layers.10.attn.wr_du.weight",
|
| 101 |
+
"model.llm.layers.10.attn.wv_dv.weight",
|
| 102 |
+
"model.llm.layers.10.attn_norm.weight",
|
| 103 |
+
"model.llm.layers.10.attn_sconv.weight",
|
| 104 |
+
"model.llm.layers.10.mlp.gate.weight",
|
| 105 |
+
"model.llm.layers.10.mlp_norm.weight",
|
| 106 |
+
"model.llm.layers.10.mlp_sconv.weight",
|
| 107 |
+
"model.llm.layers.11.attn.attn_sink_proj.weight",
|
| 108 |
+
"model.llm.layers.11.attn.gate_attn.weight",
|
| 109 |
+
"model.llm.layers.11.attn.k_sconv.static_w",
|
| 110 |
+
"model.llm.layers.11.attn.k_sconv.weight_proj",
|
| 111 |
+
"model.llm.layers.11.attn.q_l2_clip.scale",
|
| 112 |
+
"model.llm.layers.11.attn.rel_l2_clip.scale",
|
| 113 |
+
"model.llm.layers.11.attn.rel_logits_proj.proj",
|
| 114 |
+
"model.llm.layers.11.attn.v_sconv.static_w",
|
| 115 |
+
"model.llm.layers.11.attn.v_sconv.weight_proj",
|
| 116 |
+
"model.llm.layers.11.attn.wk_dv.weight",
|
| 117 |
+
"model.llm.layers.11.attn.wo_ud.weight",
|
| 118 |
+
"model.llm.layers.11.attn.wq_du.weight",
|
| 119 |
+
"model.llm.layers.11.attn.wr_du.weight",
|
| 120 |
+
"model.llm.layers.11.attn.wv_dv.weight",
|
| 121 |
+
"model.llm.layers.11.attn_sconv.weight",
|
| 122 |
+
"model.llm.layers.11.k_input_norm.weight",
|
| 123 |
+
"model.llm.layers.11.mlp.gate.weight",
|
| 124 |
+
"model.llm.layers.11.mlp_norm.weight",
|
| 125 |
+
"model.llm.layers.11.mlp_sconv.weight",
|
| 126 |
+
"model.llm.layers.11.q_input_norm.weight",
|
| 127 |
+
"model.llm.layers.11.v_input_norm.weight",
|
| 128 |
+
"model.llm.layers.2.attn.attn_sink_proj.weight",
|
| 129 |
+
"model.llm.layers.2.attn.gate_attn.weight",
|
| 130 |
+
"model.llm.layers.2.attn.k_sconv.weight",
|
| 131 |
+
"model.llm.layers.2.attn.q_l2_clip.scale",
|
| 132 |
+
"model.llm.layers.2.attn.rel_l2_clip.scale",
|
| 133 |
+
"model.llm.layers.2.attn.rel_logits_proj.proj",
|
| 134 |
+
"model.llm.layers.2.attn.v_sconv.weight",
|
| 135 |
+
"model.llm.layers.2.attn.wk_dv.weight",
|
| 136 |
+
"model.llm.layers.2.attn.wo_ud.weight",
|
| 137 |
+
"model.llm.layers.2.attn.wq_du.weight",
|
| 138 |
+
"model.llm.layers.2.attn.wr_du.weight",
|
| 139 |
+
"model.llm.layers.2.attn.wv_dv.weight",
|
| 140 |
+
"model.llm.layers.2.attn_norm.weight",
|
| 141 |
+
"model.llm.layers.2.attn_sconv.weight",
|
| 142 |
+
"model.llm.layers.2.mlp.gate.weight",
|
| 143 |
+
"model.llm.layers.2.mlp_norm.weight",
|
| 144 |
+
"model.llm.layers.2.mlp_sconv.weight",
|
| 145 |
+
"model.llm.layers.3.attn.attn_sink_proj.weight",
|
| 146 |
+
"model.llm.layers.3.attn.gate_attn.weight",
|
| 147 |
+
"model.llm.layers.3.attn.k_sconv.weight",
|
| 148 |
+
"model.llm.layers.3.attn.q_l2_clip.scale",
|
| 149 |
+
"model.llm.layers.3.attn.rel_l2_clip.scale",
|
| 150 |
+
"model.llm.layers.3.attn.rel_logits_proj.proj",
|
| 151 |
+
"model.llm.layers.3.attn.v_sconv.weight",
|
| 152 |
+
"model.llm.layers.3.attn.wk_dv.weight",
|
| 153 |
+
"model.llm.layers.3.attn.wo_ud.weight",
|
| 154 |
+
"model.llm.layers.3.attn.wq_du.weight",
|
| 155 |
+
"model.llm.layers.3.attn.wr_du.weight",
|
| 156 |
+
"model.llm.layers.3.attn.wv_dv.weight",
|
| 157 |
+
"model.llm.layers.3.attn_norm.weight",
|
| 158 |
+
"model.llm.layers.3.attn_sconv.weight",
|
| 159 |
+
"model.llm.layers.3.mlp.gate.weight",
|
| 160 |
+
"model.llm.layers.3.mlp_norm.weight",
|
| 161 |
+
"model.llm.layers.3.mlp_sconv.weight",
|
| 162 |
+
"model.llm.layers.4.attn.attn_sink_proj.weight",
|
| 163 |
+
"model.llm.layers.4.attn.gate_attn.weight",
|
| 164 |
+
"model.llm.layers.4.attn.k_sconv.weight",
|
| 165 |
+
"model.llm.layers.4.attn.q_l2_clip.scale",
|
| 166 |
+
"model.llm.layers.4.attn.rel_l2_clip.scale",
|
| 167 |
+
"model.llm.layers.4.attn.rel_logits_proj.proj",
|
| 168 |
+
"model.llm.layers.4.attn.v_sconv.weight",
|
| 169 |
+
"model.llm.layers.4.attn.wk_dv.weight",
|
| 170 |
+
"model.llm.layers.4.attn.wo_ud.weight",
|
| 171 |
+
"model.llm.layers.4.attn.wq_du.weight",
|
| 172 |
+
"model.llm.layers.4.attn.wr_du.weight",
|
| 173 |
+
"model.llm.layers.4.attn.wv_dv.weight",
|
| 174 |
+
"model.llm.layers.4.attn_norm.weight",
|
| 175 |
+
"model.llm.layers.4.attn_sconv.weight",
|
| 176 |
+
"model.llm.layers.4.mlp.gate.weight",
|
| 177 |
+
"model.llm.layers.4.mlp_norm.weight",
|
| 178 |
+
"model.llm.layers.4.mlp_sconv.weight",
|
| 179 |
+
"model.llm.layers.5.attn.attn_sink_proj.weight",
|
| 180 |
+
"model.llm.layers.5.attn.gate_attn.weight",
|
| 181 |
+
"model.llm.layers.5.attn.k_sconv.static_w",
|
| 182 |
+
"model.llm.layers.5.attn.k_sconv.weight_proj",
|
| 183 |
+
"model.llm.layers.5.attn.q_l2_clip.scale",
|
| 184 |
+
"model.llm.layers.5.attn.rel_l2_clip.scale",
|
| 185 |
+
"model.llm.layers.5.attn.rel_logits_proj.proj",
|
| 186 |
+
"model.llm.layers.5.attn.v_sconv.static_w",
|
| 187 |
+
"model.llm.layers.5.attn.v_sconv.weight_proj",
|
| 188 |
+
"model.llm.layers.5.attn.wk_dv.weight",
|
| 189 |
+
"model.llm.layers.5.attn.wo_ud.weight",
|
| 190 |
+
"model.llm.layers.5.attn.wq_du.weight",
|
| 191 |
+
"model.llm.layers.5.attn.wr_du.weight",
|
| 192 |
+
"model.llm.layers.5.attn.wv_dv.weight",
|
| 193 |
+
"model.llm.layers.5.attn_sconv.weight",
|
| 194 |
+
"model.llm.layers.5.k_input_norm.weight",
|
| 195 |
+
"model.llm.layers.5.mlp.gate.weight",
|
| 196 |
+
"model.llm.layers.5.mlp_norm.weight",
|
| 197 |
+
"model.llm.layers.5.mlp_sconv.weight",
|
| 198 |
+
"model.llm.layers.5.q_input_norm.weight",
|
| 199 |
+
"model.llm.layers.5.v_input_norm.weight",
|
| 200 |
+
"model.llm.layers.6.attn.attn_sink_proj.weight",
|
| 201 |
+
"model.llm.layers.6.attn.gate_attn.weight",
|
| 202 |
+
"model.llm.layers.6.attn.k_sconv.weight",
|
| 203 |
+
"model.llm.layers.6.attn.q_l2_clip.scale",
|
| 204 |
+
"model.llm.layers.6.attn.rel_l2_clip.scale",
|
| 205 |
+
"model.llm.layers.6.attn.rel_logits_proj.proj",
|
| 206 |
+
"model.llm.layers.6.attn.v_sconv.weight",
|
| 207 |
+
"model.llm.layers.6.attn.wk_dv.weight",
|
| 208 |
+
"model.llm.layers.6.attn.wo_ud.weight",
|
| 209 |
+
"model.llm.layers.6.attn.wq_du.weight",
|
| 210 |
+
"model.llm.layers.6.attn.wr_du.weight",
|
| 211 |
+
"model.llm.layers.6.attn.wv_dv.weight",
|
| 212 |
+
"model.llm.layers.6.attn_norm.weight",
|
| 213 |
+
"model.llm.layers.6.attn_sconv.weight",
|
| 214 |
+
"model.llm.layers.6.mlp.gate.weight",
|
| 215 |
+
"model.llm.layers.6.mlp_norm.weight",
|
| 216 |
+
"model.llm.layers.6.mlp_sconv.weight",
|
| 217 |
+
"model.llm.layers.7.attn.attn_sink_proj.weight",
|
| 218 |
+
"model.llm.layers.7.attn.gate_attn.weight",
|
| 219 |
+
"model.llm.layers.7.attn.k_sconv.weight",
|
| 220 |
+
"model.llm.layers.7.attn.q_l2_clip.scale",
|
| 221 |
+
"model.llm.layers.7.attn.rel_l2_clip.scale",
|
| 222 |
+
"model.llm.layers.7.attn.rel_logits_proj.proj",
|
| 223 |
+
"model.llm.layers.7.attn.v_sconv.weight",
|
| 224 |
+
"model.llm.layers.7.attn.wk_dv.weight",
|
| 225 |
+
"model.llm.layers.7.attn.wo_ud.weight",
|
| 226 |
+
"model.llm.layers.7.attn.wq_du.weight",
|
| 227 |
+
"model.llm.layers.7.attn.wr_du.weight",
|
| 228 |
+
"model.llm.layers.7.attn.wv_dv.weight",
|
| 229 |
+
"model.llm.layers.7.attn_norm.weight",
|
| 230 |
+
"model.llm.layers.7.attn_sconv.weight",
|
| 231 |
+
"model.llm.layers.7.mlp.gate.weight",
|
| 232 |
+
"model.llm.layers.7.mlp_norm.weight",
|
| 233 |
+
"model.llm.layers.7.mlp_sconv.weight",
|
| 234 |
+
"model.llm.layers.8.attn.attn_sink_proj.weight",
|
| 235 |
+
"model.llm.layers.8.attn.gate_attn.weight",
|
| 236 |
+
"model.llm.layers.8.attn.k_sconv.weight",
|
| 237 |
+
"model.llm.layers.8.attn.q_l2_clip.scale",
|
| 238 |
+
"model.llm.layers.8.attn.rel_l2_clip.scale",
|
| 239 |
+
"model.llm.layers.8.attn.rel_logits_proj.proj",
|
| 240 |
+
"model.llm.layers.8.attn.v_sconv.weight",
|
| 241 |
+
"model.llm.layers.8.attn.wk_dv.weight",
|
| 242 |
+
"model.llm.layers.8.attn.wo_ud.weight",
|
| 243 |
+
"model.llm.layers.8.attn.wq_du.weight",
|
| 244 |
+
"model.llm.layers.8.attn.wr_du.weight",
|
| 245 |
+
"model.llm.layers.8.attn.wv_dv.weight",
|
| 246 |
+
"model.llm.layers.8.attn_norm.weight",
|
| 247 |
+
"model.llm.layers.8.attn_sconv.weight",
|
| 248 |
+
"model.llm.layers.8.mlp.gate.weight",
|
| 249 |
+
"model.llm.layers.8.mlp_norm.weight",
|
| 250 |
+
"model.llm.layers.8.mlp_sconv.weight",
|
| 251 |
+
"model.llm.layers.9.attn.attn_sink_proj.weight",
|
| 252 |
+
"model.llm.layers.9.attn.gate_attn.weight",
|
| 253 |
+
"model.llm.layers.9.attn.k_sconv.weight",
|
| 254 |
+
"model.llm.layers.9.attn.q_l2_clip.scale",
|
| 255 |
+
"model.llm.layers.9.attn.rel_l2_clip.scale",
|
| 256 |
+
"model.llm.layers.9.attn.rel_logits_proj.proj",
|
| 257 |
+
"model.llm.layers.9.attn.v_sconv.weight",
|
| 258 |
+
"model.llm.layers.9.attn.wk_dv.weight",
|
| 259 |
+
"model.llm.layers.9.attn.wo_ud.weight",
|
| 260 |
+
"model.llm.layers.9.attn.wq_du.weight",
|
| 261 |
+
"model.llm.layers.9.attn.wr_du.weight",
|
| 262 |
+
"model.llm.layers.9.attn.wv_dv.weight",
|
| 263 |
+
"model.llm.layers.9.attn_norm.weight",
|
| 264 |
+
"model.llm.layers.9.attn_sconv.weight",
|
| 265 |
+
"model.llm.layers.9.mlp.gate.weight",
|
| 266 |
+
"model.llm.layers.9.mlp_norm.weight",
|
| 267 |
+
"model.llm.layers.9.mlp_sconv.weight",
|
| 268 |
+
"model.llm.norm.weight",
|
| 269 |
+
"model.visual.backbone.embeddings.class_embedding",
|
| 270 |
+
"model.visual.backbone.embeddings.cls_position_embedding",
|
| 271 |
+
"model.visual.backbone.embeddings.patch_embedding.weight",
|
| 272 |
+
"model.visual.backbone.embeddings.position_embedding",
|
| 273 |
+
"model.visual.backbone.layers.0.ln_1.bias",
|
| 274 |
+
"model.visual.backbone.layers.0.ln_1.weight",
|
| 275 |
+
"model.visual.backbone.layers.0.ln_2.bias",
|
| 276 |
+
"model.visual.backbone.layers.0.ln_2.weight",
|
| 277 |
+
"model.visual.backbone.layers.0.ls_1.gamma",
|
| 278 |
+
"model.visual.backbone.layers.0.ls_2.gamma",
|
| 279 |
+
"model.visual.backbone.layers.0.mlp.c_fc.bias",
|
| 280 |
+
"model.visual.backbone.layers.0.mlp.c_fc.weight",
|
| 281 |
+
"model.visual.backbone.layers.0.mlp.c_proj.bias",
|
| 282 |
+
"model.visual.backbone.layers.0.mlp.c_proj.weight",
|
| 283 |
+
"model.visual.backbone.layers.0.self_attn.k_proj.bias",
|
| 284 |
+
"model.visual.backbone.layers.0.self_attn.k_proj.weight",
|
| 285 |
+
"model.visual.backbone.layers.0.self_attn.out_proj.bias",
|
| 286 |
+
"model.visual.backbone.layers.0.self_attn.out_proj.weight",
|
| 287 |
+
"model.visual.backbone.layers.0.self_attn.q_proj.bias",
|
| 288 |
+
"model.visual.backbone.layers.0.self_attn.q_proj.weight",
|
| 289 |
+
"model.visual.backbone.layers.0.self_attn.v_proj.bias",
|
| 290 |
+
"model.visual.backbone.layers.0.self_attn.v_proj.weight",
|
| 291 |
+
"model.visual.backbone.layers.1.ln_1.bias",
|
| 292 |
+
"model.visual.backbone.layers.1.ln_1.weight",
|
| 293 |
+
"model.visual.backbone.layers.1.ln_2.bias",
|
| 294 |
+
"model.visual.backbone.layers.1.ln_2.weight",
|
| 295 |
+
"model.visual.backbone.layers.1.ls_1.gamma",
|
| 296 |
+
"model.visual.backbone.layers.1.ls_2.gamma",
|
| 297 |
+
"model.visual.backbone.layers.1.mlp.c_fc.bias",
|
| 298 |
+
"model.visual.backbone.layers.1.mlp.c_fc.weight",
|
| 299 |
+
"model.visual.backbone.layers.1.mlp.c_proj.bias",
|
| 300 |
+
"model.visual.backbone.layers.1.mlp.c_proj.weight",
|
| 301 |
+
"model.visual.backbone.layers.1.self_attn.k_proj.bias",
|
| 302 |
+
"model.visual.backbone.layers.1.self_attn.k_proj.weight",
|
| 303 |
+
"model.visual.backbone.layers.1.self_attn.out_proj.bias",
|
| 304 |
+
"model.visual.backbone.layers.1.self_attn.out_proj.weight",
|
| 305 |
+
"model.visual.backbone.layers.1.self_attn.q_proj.bias",
|
| 306 |
+
"model.visual.backbone.layers.1.self_attn.q_proj.weight",
|
| 307 |
+
"model.visual.backbone.layers.1.self_attn.v_proj.bias",
|
| 308 |
+
"model.visual.backbone.layers.1.self_attn.v_proj.weight",
|
| 309 |
+
"model.visual.backbone.layers.2.ln_1.bias",
|
| 310 |
+
"model.visual.backbone.layers.2.ln_1.weight",
|
| 311 |
+
"model.visual.backbone.layers.2.ln_2.bias",
|
| 312 |
+
"model.visual.backbone.layers.2.ln_2.weight",
|
| 313 |
+
"model.visual.backbone.layers.2.ls_1.gamma",
|
| 314 |
+
"model.visual.backbone.layers.2.ls_2.gamma",
|
| 315 |
+
"model.visual.backbone.layers.2.mlp.c_fc.bias",
|
| 316 |
+
"model.visual.backbone.layers.2.mlp.c_fc.weight",
|
| 317 |
+
"model.visual.backbone.layers.2.mlp.c_proj.bias",
|
| 318 |
+
"model.visual.backbone.layers.2.mlp.c_proj.weight",
|
| 319 |
+
"model.visual.backbone.layers.2.self_attn.k_proj.bias",
|
| 320 |
+
"model.visual.backbone.layers.2.self_attn.k_proj.weight",
|
| 321 |
+
"model.visual.backbone.layers.2.self_attn.out_proj.bias",
|
| 322 |
+
"model.visual.backbone.layers.2.self_attn.out_proj.weight",
|
| 323 |
+
"model.visual.backbone.layers.2.self_attn.q_proj.bias",
|
| 324 |
+
"model.visual.backbone.layers.2.self_attn.q_proj.weight",
|
| 325 |
+
"model.visual.backbone.layers.2.self_attn.v_proj.bias",
|
| 326 |
+
"model.visual.backbone.layers.2.self_attn.v_proj.weight",
|
| 327 |
+
"model.visual.backbone.layers.3.ln_1.bias",
|
| 328 |
+
"model.visual.backbone.layers.3.ln_1.weight",
|
| 329 |
+
"model.visual.backbone.layers.3.ln_2.bias",
|
| 330 |
+
"model.visual.backbone.layers.3.ln_2.weight",
|
| 331 |
+
"model.visual.backbone.layers.3.ls_1.gamma",
|
| 332 |
+
"model.visual.backbone.layers.3.ls_2.gamma",
|
| 333 |
+
"model.visual.backbone.layers.3.mlp.c_fc.bias",
|
| 334 |
+
"model.visual.backbone.layers.3.mlp.c_fc.weight",
|
| 335 |
+
"model.visual.backbone.layers.3.mlp.c_proj.bias",
|
| 336 |
+
"model.visual.backbone.layers.3.mlp.c_proj.weight",
|
| 337 |
+
"model.visual.backbone.layers.3.self_attn.k_proj.bias",
|
| 338 |
+
"model.visual.backbone.layers.3.self_attn.k_proj.weight",
|
| 339 |
+
"model.visual.backbone.layers.3.self_attn.out_proj.bias",
|
| 340 |
+
"model.visual.backbone.layers.3.self_attn.out_proj.weight",
|
| 341 |
+
"model.visual.backbone.layers.3.self_attn.q_proj.bias",
|
| 342 |
+
"model.visual.backbone.layers.3.self_attn.q_proj.weight",
|
| 343 |
+
"model.visual.backbone.layers.3.self_attn.v_proj.bias",
|
| 344 |
+
"model.visual.backbone.layers.3.self_attn.v_proj.weight",
|
| 345 |
+
"model.visual.backbone.ln_pre.bias",
|
| 346 |
+
"model.visual.backbone.ln_pre.weight",
|
| 347 |
+
"model.visual.proj_in.weight",
|
| 348 |
+
"model.visual.proj_out.weight"
|
| 349 |
+
],
|
| 350 |
+
"weight_block_shape": [],
|
| 351 |
+
"weight_transform": "cast"
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"compute_dtype": "nvfp4",
|
| 355 |
+
"output_dtype": "uint8",
|
| 356 |
+
"required_activation_quantization": {
|
| 357 |
+
"amax_type": "per_token",
|
| 358 |
+
"block_size": 16,
|
| 359 |
+
"format": "nvfp4_e2m1",
|
| 360 |
+
"rounding": "nearest",
|
| 361 |
+
"scale_dtype": "float8_e4m3fn",
|
| 362 |
+
"scale_selection_rule": "fouroversix_mae"
|
| 363 |
+
},
|
| 364 |
+
"tensors": [
|
| 365 |
+
"model.llm.layers.0.mlp.experts.w13_weight",
|
| 366 |
+
"model.llm.layers.0.mlp.experts.w2_weight",
|
| 367 |
+
"model.llm.layers.1.mlp.experts.w13_weight",
|
| 368 |
+
"model.llm.layers.1.mlp.experts.w2_weight",
|
| 369 |
+
"model.llm.layers.2.mlp.experts.w13_weight",
|
| 370 |
+
"model.llm.layers.2.mlp.experts.w2_weight",
|
| 371 |
+
"model.llm.layers.3.mlp.experts.w13_weight",
|
| 372 |
+
"model.llm.layers.3.mlp.experts.w2_weight",
|
| 373 |
+
"model.llm.layers.4.mlp.experts.w13_weight",
|
| 374 |
+
"model.llm.layers.4.mlp.experts.w2_weight",
|
| 375 |
+
"model.llm.layers.5.mlp.experts.w13_weight",
|
| 376 |
+
"model.llm.layers.5.mlp.experts.w2_weight"
|
| 377 |
+
],
|
| 378 |
+
"weight_block_shape": [
|
| 379 |
+
1,
|
| 380 |
+
16
|
| 381 |
+
],
|
| 382 |
+
"weight_scale_format": "amax_denominator",
|
| 383 |
+
"weight_transform": "nvfp4_nearest"
|
| 384 |
+
},
|
| 385 |
+
{
|
| 386 |
+
"compute_dtype": "mxfp8",
|
| 387 |
+
"output_dtype": "float8_e4m3fn",
|
| 388 |
+
"required_activation_quantization": {
|
| 389 |
+
"block_size": 32,
|
| 390 |
+
"format": "mxfp8_e4m3",
|
| 391 |
+
"rounding": "nearest",
|
| 392 |
+
"scale_dtype": "ue8m0"
|
| 393 |
+
},
|
| 394 |
+
"tensors": [
|
| 395 |
+
"model.llm.layers.10.mlp.experts.w13_weight",
|
| 396 |
+
"model.llm.layers.10.mlp.experts.w2_weight",
|
| 397 |
+
"model.llm.layers.11.mlp.experts.w13_weight",
|
| 398 |
+
"model.llm.layers.11.mlp.experts.w2_weight",
|
| 399 |
+
"model.llm.layers.6.mlp.experts.w13_weight",
|
| 400 |
+
"model.llm.layers.6.mlp.experts.w2_weight",
|
| 401 |
+
"model.llm.layers.7.mlp.experts.w13_weight",
|
| 402 |
+
"model.llm.layers.7.mlp.experts.w2_weight",
|
| 403 |
+
"model.llm.layers.8.mlp.experts.w13_weight",
|
| 404 |
+
"model.llm.layers.8.mlp.experts.w2_weight",
|
| 405 |
+
"model.llm.layers.9.mlp.experts.w13_weight",
|
| 406 |
+
"model.llm.layers.9.mlp.experts.w2_weight"
|
| 407 |
+
],
|
| 408 |
+
"weight_block_shape": [
|
| 409 |
+
32,
|
| 410 |
+
32
|
| 411 |
+
],
|
| 412 |
+
"weight_transform": "mxfp8_nearest"
|
| 413 |
+
},
|
| 414 |
+
{
|
| 415 |
+
"compute_dtype": "bfloat16",
|
| 416 |
+
"fwd_output_dtype": "float32",
|
| 417 |
+
"output_dtype": "bfloat16",
|
| 418 |
+
"required_activation_quantization": null,
|
| 419 |
+
"tensors": [
|
| 420 |
+
"model.llm.unembed.weight"
|
| 421 |
+
],
|
| 422 |
+
"weight_block_shape": [],
|
| 423 |
+
"weight_transform": "cast"
|
| 424 |
+
}
|
| 425 |
+
]
|
| 426 |
+
},
|
| 427 |
+
"mtp": {
|
| 428 |
+
"parameter_groups": [
|
| 429 |
+
{
|
| 430 |
+
"compute_dtype": "bfloat16",
|
| 431 |
+
"output_dtype": "bfloat16",
|
| 432 |
+
"required_activation_quantization": null,
|
| 433 |
+
"tensors": [
|
| 434 |
+
"fuse_weight",
|
| 435 |
+
"layers.0.dflare_norm.weight",
|
| 436 |
+
"layers.0.input_layernorm.weight",
|
| 437 |
+
"layers.0.mlp.down_proj.weight",
|
| 438 |
+
"layers.0.mlp.gate_proj.weight",
|
| 439 |
+
"layers.0.mlp.up_proj.weight",
|
| 440 |
+
"layers.0.post_attention_layernorm.weight",
|
| 441 |
+
"layers.0.self_attn.k_norm.weight",
|
| 442 |
+
"layers.0.self_attn.k_norm_ctx.weight",
|
| 443 |
+
"layers.0.self_attn.k_proj.weight",
|
| 444 |
+
"layers.0.self_attn.k_proj_ctx.weight",
|
| 445 |
+
"layers.0.self_attn.o_proj.weight",
|
| 446 |
+
"layers.0.self_attn.q_norm.weight",
|
| 447 |
+
"layers.0.self_attn.q_proj.weight",
|
| 448 |
+
"layers.0.self_attn.v_proj.weight",
|
| 449 |
+
"layers.0.self_attn.v_proj_ctx.weight",
|
| 450 |
+
"layers.1.dflare_norm.weight",
|
| 451 |
+
"layers.1.input_layernorm.weight",
|
| 452 |
+
"layers.1.mlp.down_proj.weight",
|
| 453 |
+
"layers.1.mlp.gate_proj.weight",
|
| 454 |
+
"layers.1.mlp.up_proj.weight",
|
| 455 |
+
"layers.1.post_attention_layernorm.weight",
|
| 456 |
+
"layers.1.self_attn.k_norm.weight",
|
| 457 |
+
"layers.1.self_attn.k_norm_ctx.weight",
|
| 458 |
+
"layers.1.self_attn.k_proj.weight",
|
| 459 |
+
"layers.1.self_attn.k_proj_ctx.weight",
|
| 460 |
+
"layers.1.self_attn.o_proj.weight",
|
| 461 |
+
"layers.1.self_attn.q_norm.weight",
|
| 462 |
+
"layers.1.self_attn.q_proj.weight",
|
| 463 |
+
"layers.1.self_attn.v_proj.weight",
|
| 464 |
+
"layers.1.self_attn.v_proj_ctx.weight",
|
| 465 |
+
"layers.2.dflare_norm.weight",
|
| 466 |
+
"layers.2.input_layernorm.weight",
|
| 467 |
+
"layers.2.mlp.down_proj.weight",
|
| 468 |
+
"layers.2.mlp.gate_proj.weight",
|
| 469 |
+
"layers.2.mlp.up_proj.weight",
|
| 470 |
+
"layers.2.post_attention_layernorm.weight",
|
| 471 |
+
"layers.2.self_attn.k_norm.weight",
|
| 472 |
+
"layers.2.self_attn.k_norm_ctx.weight",
|
| 473 |
+
"layers.2.self_attn.k_proj.weight",
|
| 474 |
+
"layers.2.self_attn.k_proj_ctx.weight",
|
| 475 |
+
"layers.2.self_attn.o_proj.weight",
|
| 476 |
+
"layers.2.self_attn.q_norm.weight",
|
| 477 |
+
"layers.2.self_attn.q_proj.weight",
|
| 478 |
+
"layers.2.self_attn.v_proj.weight",
|
| 479 |
+
"layers.2.self_attn.v_proj_ctx.weight",
|
| 480 |
+
"layers.3.dflare_norm.weight",
|
| 481 |
+
"layers.3.input_layernorm.weight",
|
| 482 |
+
"layers.3.mlp.down_proj.weight",
|
| 483 |
+
"layers.3.mlp.gate_proj.weight",
|
| 484 |
+
"layers.3.mlp.up_proj.weight",
|
| 485 |
+
"layers.3.post_attention_layernorm.weight",
|
| 486 |
+
"layers.3.self_attn.k_norm.weight",
|
| 487 |
+
"layers.3.self_attn.k_norm_ctx.weight",
|
| 488 |
+
"layers.3.self_attn.k_proj.weight",
|
| 489 |
+
"layers.3.self_attn.k_proj_ctx.weight",
|
| 490 |
+
"layers.3.self_attn.o_proj.weight",
|
| 491 |
+
"layers.3.self_attn.q_norm.weight",
|
| 492 |
+
"layers.3.self_attn.q_proj.weight",
|
| 493 |
+
"layers.3.self_attn.v_proj.weight",
|
| 494 |
+
"layers.3.self_attn.v_proj_ctx.weight",
|
| 495 |
+
"layers.4.dflare_norm.weight",
|
| 496 |
+
"layers.4.input_layernorm.weight",
|
| 497 |
+
"layers.4.mlp.down_proj.weight",
|
| 498 |
+
"layers.4.mlp.gate_proj.weight",
|
| 499 |
+
"layers.4.mlp.up_proj.weight",
|
| 500 |
+
"layers.4.post_attention_layernorm.weight",
|
| 501 |
+
"layers.4.self_attn.k_norm.weight",
|
| 502 |
+
"layers.4.self_attn.k_norm_ctx.weight",
|
| 503 |
+
"layers.4.self_attn.k_proj.weight",
|
| 504 |
+
"layers.4.self_attn.k_proj_ctx.weight",
|
| 505 |
+
"layers.4.self_attn.o_proj.weight",
|
| 506 |
+
"layers.4.self_attn.q_norm.weight",
|
| 507 |
+
"layers.4.self_attn.q_proj.weight",
|
| 508 |
+
"layers.4.self_attn.v_proj.weight",
|
| 509 |
+
"layers.4.self_attn.v_proj_ctx.weight",
|
| 510 |
+
"markov_head.dflare_norm.weight",
|
| 511 |
+
"markov_head.hidden_out.bias",
|
| 512 |
+
"markov_head.hidden_out.weight",
|
| 513 |
+
"markov_head.markov_w1.weight",
|
| 514 |
+
"markov_head.rnn_proj.bias",
|
| 515 |
+
"markov_head.rnn_proj.weight",
|
| 516 |
+
"mask_embed",
|
| 517 |
+
"norm.weight"
|
| 518 |
+
],
|
| 519 |
+
"weight_block_shape": [],
|
| 520 |
+
"weight_transform": "cast"
|
| 521 |
+
}
|
| 522 |
+
]
|
| 523 |
+
},
|
| 524 |
+
"quant_method": "inkling2"
|
| 525 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9c24250280ea240d1957cb6b92fb597d1a6c2beca4c926ef0d10d8342d669657
|
| 3 |
+
size 18808680
|
modeling_inkling2_tiny.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from huggingface_hub import hf_hub_download
|
| 7 |
+
from safetensors.torch import load_file
|
| 8 |
+
from transformers import Inkling2Config, Inkling2ForConditionalGeneration
|
| 9 |
+
from transformers.conversion_mapping import get_model_conversion_mapping
|
| 10 |
+
from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def dequantize(weight, scale, amax=None, denominator=None):
|
| 14 |
+
if weight.dtype == torch.uint8:
|
| 15 |
+
if amax is None or denominator is None:
|
| 16 |
+
raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
|
| 17 |
+
codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
|
| 18 |
+
levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
|
| 19 |
+
values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
|
| 20 |
+
return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
|
| 21 |
+
if weight.dtype == torch.float8_e4m3fn:
|
| 22 |
+
scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
|
| 23 |
+
return weight.float() * scales
|
| 24 |
+
raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
| 28 |
+
@classmethod
|
| 29 |
+
def from_pretrained(
|
| 30 |
+
cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
|
| 31 |
+
revision=None, cache_dir=None, local_files_only=False, token=None,
|
| 32 |
+
attn_implementation="eager", **kwargs,
|
| 33 |
+
):
|
| 34 |
+
if model_args:
|
| 35 |
+
raise ValueError("Inkling 2 does not take positional construction arguments")
|
| 36 |
+
options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
|
| 37 |
+
root = Path(pretrained_model_name_or_path)
|
| 38 |
+
|
| 39 |
+
def resolve(filename):
|
| 40 |
+
if root.is_dir():
|
| 41 |
+
return root / filename
|
| 42 |
+
return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
|
| 43 |
+
|
| 44 |
+
import json
|
| 45 |
+
|
| 46 |
+
raw = json.loads(resolve("config.json").read_text())
|
| 47 |
+
# The native config otherwise assumes the all-MXFP8 sibling. This
|
| 48 |
+
# adapter dequantizes each tensor according to its actual stored dtype.
|
| 49 |
+
raw.pop("quantization_config", None)
|
| 50 |
+
config = Inkling2Config(**raw)
|
| 51 |
+
config._attn_implementation = attn_implementation
|
| 52 |
+
dtype = kwargs.pop("torch_dtype", dtype)
|
| 53 |
+
if dtype in (None, "auto"):
|
| 54 |
+
dtype = torch.bfloat16
|
| 55 |
+
elif isinstance(dtype, str):
|
| 56 |
+
dtype = getattr(torch, dtype)
|
| 57 |
+
if dtype not in (torch.float32, torch.bfloat16, torch.float16):
|
| 58 |
+
raise ValueError("The Transformers adapter requires a floating compute dtype")
|
| 59 |
+
device_map = kwargs.pop("device_map", None)
|
| 60 |
+
if device_map == "auto":
|
| 61 |
+
device_map = "cuda" if torch.cuda.is_available() else "cpu"
|
| 62 |
+
if isinstance(device_map, dict):
|
| 63 |
+
if set(device_map) != {""}:
|
| 64 |
+
raise ValueError("Use a single device for this tiny Transformers adapter")
|
| 65 |
+
device_map = device_map[""]
|
| 66 |
+
allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
|
| 67 |
+
unexpected = set(kwargs) - allowed
|
| 68 |
+
if unexpected:
|
| 69 |
+
raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
|
| 70 |
+
model = Inkling2ForConditionalGeneration(config)
|
| 71 |
+
transforms = get_model_conversion_mapping(model)
|
| 72 |
+
renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
|
| 73 |
+
converters = [t for t in transforms if isinstance(t, WeightConverter)]
|
| 74 |
+
source = load_file(str(resolve("model.safetensors")))
|
| 75 |
+
state = {}
|
| 76 |
+
for name, weight in source.items():
|
| 77 |
+
if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
|
| 78 |
+
continue
|
| 79 |
+
if name + "_scale" in source:
|
| 80 |
+
weight = dequantize(
|
| 81 |
+
weight, source[name + "_scale"], source.get(name + "_amax"),
|
| 82 |
+
source.get(name + "_outer_scale_denominator"),
|
| 83 |
+
)
|
| 84 |
+
target, _ = rename_source_key(name, renamings, converters)
|
| 85 |
+
if name.endswith("w13_weight"):
|
| 86 |
+
# The source interleaves gate and up; Transformers chunks them.
|
| 87 |
+
e, rows, columns = weight.shape
|
| 88 |
+
weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
|
| 89 |
+
if target in state:
|
| 90 |
+
raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
|
| 91 |
+
keep_float32 = source[name].dtype == torch.float32 or any(
|
| 92 |
+
part in target for part in model._keep_in_fp32_modules_strict
|
| 93 |
+
)
|
| 94 |
+
target_dtype = torch.float32 if keep_float32 else dtype
|
| 95 |
+
state[target] = weight.to(target_dtype).contiguous()
|
| 96 |
+
state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
|
| 97 |
+
model.load_state_dict(state, strict=True, assign=True)
|
| 98 |
+
model.eval()
|
| 99 |
+
if device_map is not None:
|
| 100 |
+
model.to(device_map)
|
| 101 |
+
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 102 |
+
return model
|
mtp.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:194ed2549514367c14ba8e3df0a1f1e0a7397bb9ceaf1971b647535f52531663
|
| 3 |
+
size 4009408
|
processor_config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"image_processor": {
|
| 3 |
+
"do_convert_rgb": true,
|
| 4 |
+
"do_normalize": true,
|
| 5 |
+
"do_rescale": true,
|
| 6 |
+
"image_mean": [0.5, 0.5, 0.5],
|
| 7 |
+
"image_processor_type": "Inkling2ImageProcessor",
|
| 8 |
+
"image_std": [0.5, 0.5, 0.5],
|
| 9 |
+
"rescale_factor": 0.00392156862745098,
|
| 10 |
+
"size": {"height": 40, "width": 40}
|
| 11 |
+
},
|
| 12 |
+
"image_token": "<|unused_200054|>",
|
| 13 |
+
"processor_class": "Inkling2Processor"
|
| 14 |
+
}
|
pyproject.toml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=68"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "sglang-inkling2-tied"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
|
| 9 |
+
requires-python = ">=3.10"
|
| 10 |
+
|
| 11 |
+
[project.entry-points."sglang.srt.plugins"]
|
| 12 |
+
inkling2_tied = "sglang_inkling2_tied:register"
|
| 13 |
+
|
| 14 |
+
[tool.setuptools]
|
| 15 |
+
py-modules = ["sglang_inkling2_tied"]
|
sglang_inkling2_tied.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
|
| 2 |
+
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _config_init(original, config, *args, **kwargs):
|
| 9 |
+
tied = kwargs.get("tie_word_embeddings", False)
|
| 10 |
+
if tied:
|
| 11 |
+
kwargs = {**kwargs, "tie_word_embeddings": False}
|
| 12 |
+
original(config, *args, **kwargs)
|
| 13 |
+
if tied:
|
| 14 |
+
config.tie_word_embeddings = True
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _load_weights(original, model, weights):
|
| 18 |
+
if not model.text_config.tie_word_embeddings or model.llm is None:
|
| 19 |
+
return original(model, weights)
|
| 20 |
+
|
| 21 |
+
def with_head():
|
| 22 |
+
embedding = None
|
| 23 |
+
for name, tensor in weights:
|
| 24 |
+
if name == "model.llm.unembed.weight":
|
| 25 |
+
raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
|
| 26 |
+
if name == "model.llm.embed.weight":
|
| 27 |
+
embedding = tensor
|
| 28 |
+
yield name, tensor
|
| 29 |
+
if embedding is None:
|
| 30 |
+
raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
|
| 31 |
+
yield "model.llm.unembed.weight", embedding
|
| 32 |
+
|
| 33 |
+
loaded = original(model, with_head())
|
| 34 |
+
embedding, head = model.llm.embed_tokens, model.llm.lm_head
|
| 35 |
+
if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
|
| 36 |
+
raise ValueError("Tied embedding/head tensor-parallel partitions must match")
|
| 37 |
+
head.weight = embedding.weight
|
| 38 |
+
assert head.weight is embedding.weight
|
| 39 |
+
logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
|
| 40 |
+
return loaded
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class _PaddedMarlin(tuple):
|
| 44 |
+
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
| 45 |
+
value = super().__new__(cls, packed)
|
| 46 |
+
value.input_width = input_width
|
| 47 |
+
value.output_width = output_width
|
| 48 |
+
value.padded_input = padded_input
|
| 49 |
+
value.padded_output = padded_output
|
| 50 |
+
return value
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _pad_matrix(tensor, rows, columns, fill=0):
|
| 54 |
+
output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
|
| 55 |
+
output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
|
| 56 |
+
return output
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _repack_marlin(original, weight, scales):
|
| 60 |
+
import torch
|
| 61 |
+
|
| 62 |
+
nvfp4 = weight.dtype == torch.uint8
|
| 63 |
+
packing = 2 if nvfp4 else 1
|
| 64 |
+
block = 16 if nvfp4 else 32
|
| 65 |
+
n, k = weight.shape[-2], weight.shape[-1] * packing
|
| 66 |
+
padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
|
| 67 |
+
if (padded_k, padded_n) == (k, n):
|
| 68 |
+
return original(weight, scales)
|
| 69 |
+
weight = _pad_matrix(weight, padded_n, padded_k // packing)
|
| 70 |
+
# Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
|
| 71 |
+
# Zero weight bytes make padded products zero in both precision families.
|
| 72 |
+
scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
|
| 73 |
+
packed = original(weight, scales)
|
| 74 |
+
return _PaddedMarlin(packed, k, n, padded_k, padded_n)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
|
| 78 |
+
if not isinstance(packed, _PaddedMarlin):
|
| 79 |
+
return original(x, packed, out_size, *args, **kwargs)
|
| 80 |
+
if x.shape[-1] != packed.input_width or out_size != packed.output_width:
|
| 81 |
+
raise ValueError("Marlin padding metadata does not match the expert projection")
|
| 82 |
+
padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
|
| 83 |
+
padded[..., :x.shape[-1]].copy_(x)
|
| 84 |
+
if original.__name__ == "gemv" and packed.padded_input == 128:
|
| 85 |
+
# Eight K tiles fit one split of eight warps. The large-model heuristic
|
| 86 |
+
# otherwise chooses four splits/four warps for short decode batches.
|
| 87 |
+
kwargs = {**kwargs, "splits": 1, "warps": 8}
|
| 88 |
+
result = original(padded, packed, packed.padded_output, *args, **kwargs)
|
| 89 |
+
return result[..., :out_size].contiguous()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _relative_fa4(original, *args, **kwargs):
|
| 93 |
+
q = args[0] if args else kwargs.get("q")
|
| 94 |
+
if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
|
| 95 |
+
and kwargs.get("tile_mn") is None):
|
| 96 |
+
# The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
|
| 97 |
+
# relative-bias kernel explicitly requires M=N=128. Use its tile API.
|
| 98 |
+
kwargs = {**kwargs, "tile_mn": (128, 128)}
|
| 99 |
+
return original(*args, **kwargs)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _attn_res_config(original, width, *args, **kwargs):
|
| 103 |
+
threads, stages, vector, cluster = original(width, *args, **kwargs)
|
| 104 |
+
if width == 32:
|
| 105 |
+
# One compute warp covers the original 32 residual coordinates.
|
| 106 |
+
# Keep the same CuTe kernel, RMS normalization, and source weighting.
|
| 107 |
+
return 32, stages, 1, 1
|
| 108 |
+
return threads, stages, vector, cluster
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _small_all_reduce(original, x, group, **kwargs):
|
| 112 |
+
if x.ndim == 2 and x.shape[-1] == 32:
|
| 113 |
+
return group.all_reduce(x.clone())
|
| 114 |
+
return original(x, group, **kwargs)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _small_multicast_input(original, group, shape, dtype):
|
| 118 |
+
if len(shape) == 2 and shape[-1] == 32:
|
| 119 |
+
import torch
|
| 120 |
+
|
| 121 |
+
return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
|
| 122 |
+
return original(group, shape, dtype)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _small_multicast_reduce(original, x, group):
|
| 126 |
+
if x.ndim == 2 and x.shape[-1] == 32:
|
| 127 |
+
x.copy_(group.all_reduce(x))
|
| 128 |
+
return x
|
| 129 |
+
return original(x, group)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
| 133 |
+
if cols == 32:
|
| 134 |
+
# Use the existing FP32 combine path instead of the GB300-tuned
|
| 135 |
+
# symmetric-memory one-shot collective for these tiny projections.
|
| 136 |
+
return None
|
| 137 |
+
return original(group, rows, cols, dtype, tag=tag)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def register():
|
| 141 |
+
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 142 |
+
|
| 143 |
+
HookRegistry.register(
|
| 144 |
+
"sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
|
| 145 |
+
)
|
| 146 |
+
HookRegistry.register(
|
| 147 |
+
"sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
|
| 148 |
+
)
|
| 149 |
+
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 150 |
+
for name in ("repack", "repack_mxfp8"):
|
| 151 |
+
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
| 152 |
+
for name in ("gemm", "gemv"):
|
| 153 |
+
HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
|
| 154 |
+
HookRegistry.register(
|
| 155 |
+
"sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
|
| 156 |
+
_relative_fa4, HookType.AROUND,
|
| 157 |
+
)
|
| 158 |
+
HookRegistry.register(
|
| 159 |
+
"sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
|
| 160 |
+
_attn_res_config, HookType.AROUND,
|
| 161 |
+
)
|
| 162 |
+
prefix = "sglang.kernels.ops.inkling2.comm"
|
| 163 |
+
for name, hook in (
|
| 164 |
+
("all_reduce", _small_all_reduce),
|
| 165 |
+
("multicast_all_reduce_input", _small_multicast_input),
|
| 166 |
+
("multicast_all_reduce", _small_multicast_reduce),
|
| 167 |
+
("oneshot_all_reduce_input", _small_oneshot_input),
|
| 168 |
+
):
|
| 169 |
+
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|endoftext|>",
|
| 4 |
+
"<|message_user|>",
|
| 5 |
+
"<|message_model|>",
|
| 6 |
+
"<|message_system|>",
|
| 7 |
+
"<|message_tool|>",
|
| 8 |
+
"<|content_text|>",
|
| 9 |
+
"<|content_image|>",
|
| 10 |
+
"<|content_model_end_sampling|>",
|
| 11 |
+
"<|content_thinking|>",
|
| 12 |
+
"<|end_message|>",
|
| 13 |
+
"<|content_tool_error|>",
|
| 14 |
+
"<|content_xml|>",
|
| 15 |
+
"<|begin_of_text|>",
|
| 16 |
+
"<|content_invoke_tool_json|>",
|
| 17 |
+
"<|content_invoke_tool_text|>"
|
| 18 |
+
]
|
| 19 |
+
}
|
tiktoken/tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f170372dd655728909228220d77cf6b8104a59f38a91262a7d2c4027906ec0d8
|
| 3 |
+
size 3386494
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7f332727f9a3f093dd452a39cc978e6757627e5dad1b0673aa4e75174f236dd6
|
| 3 |
+
size 26178678
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 3 |
+
"model_max_length": 1048576,
|
| 4 |
+
"clean_up_tokenization_spaces": false,
|
| 5 |
+
"fix_mistral_regex": false,
|
| 6 |
+
"extra_special_tokens": {
|
| 7 |
+
"endoftext": "<|endoftext|>",
|
| 8 |
+
"message_user": "<|message_user|>",
|
| 9 |
+
"message_model": "<|message_model|>",
|
| 10 |
+
"message_system": "<|message_system|>",
|
| 11 |
+
"message_tool": "<|message_tool|>",
|
| 12 |
+
"content_text": "<|content_text|>",
|
| 13 |
+
"content_image": "<|content_image|>",
|
| 14 |
+
"content_model_end_sampling": "<|content_model_end_sampling|>",
|
| 15 |
+
"content_thinking": "<|content_thinking|>",
|
| 16 |
+
"end_message": "<|end_message|>",
|
| 17 |
+
"content_tool_error": "<|content_tool_error|>",
|
| 18 |
+
"content_xml": "<|content_xml|>",
|
| 19 |
+
"begin_of_text": "<|begin_of_text|>",
|
| 20 |
+
"content_invoke_tool_json": "<|content_invoke_tool_json|>",
|
| 21 |
+
"content_invoke_tool_text": "<|content_invoke_tool_text|>"
|
| 22 |
+
},
|
| 23 |
+
"added_tokens_decoder": {
|
| 24 |
+
"199998": {
|
| 25 |
+
"content": "<|unused|>",
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"lstrip": false,
|
| 28 |
+
"rstrip": false,
|
| 29 |
+
"normalized": false,
|
| 30 |
+
"special": true
|
| 31 |
+
},
|
| 32 |
+
"199999": {
|
| 33 |
+
"content": "<|endoftext|>",
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"lstrip": false,
|
| 36 |
+
"rstrip": false,
|
| 37 |
+
"normalized": false,
|
| 38 |
+
"special": true
|
| 39 |
+
},
|
| 40 |
+
"200000": {
|
| 41 |
+
"content": "<|message_user|>",
|
| 42 |
+
"single_word": false,
|
| 43 |
+
"lstrip": false,
|
| 44 |
+
"rstrip": false,
|
| 45 |
+
"normalized": false,
|
| 46 |
+
"special": true
|
| 47 |
+
},
|
| 48 |
+
"200001": {
|
| 49 |
+
"content": "<|message_model|>",
|
| 50 |
+
"single_word": false,
|
| 51 |
+
"lstrip": false,
|
| 52 |
+
"rstrip": false,
|
| 53 |
+
"normalized": false,
|
| 54 |
+
"special": true
|
| 55 |
+
},
|
| 56 |
+
"200002": {
|
| 57 |
+
"content": "<|message_system|>",
|
| 58 |
+
"single_word": false,
|
| 59 |
+
"lstrip": false,
|
| 60 |
+
"rstrip": false,
|
| 61 |
+
"normalized": false,
|
| 62 |
+
"special": true
|
| 63 |
+
},
|
| 64 |
+
"200003": {
|
| 65 |
+
"content": "<|message_tool|>",
|
| 66 |
+
"single_word": false,
|
| 67 |
+
"lstrip": false,
|
| 68 |
+
"rstrip": false,
|
| 69 |
+
"normalized": false,
|
| 70 |
+
"special": true
|
| 71 |
+
},
|
| 72 |
+
"200004": {
|
| 73 |
+
"content": "<|content_text|>",
|
| 74 |
+
"single_word": false,
|
| 75 |
+
"lstrip": false,
|
| 76 |
+
"rstrip": false,
|
| 77 |
+
"normalized": false,
|
| 78 |
+
"special": true
|
| 79 |
+
},
|
| 80 |
+
"200005": {
|
| 81 |
+
"content": "<|content_image|>",
|
| 82 |
+
"single_word": false,
|
| 83 |
+
"lstrip": false,
|
| 84 |
+
"rstrip": false,
|
| 85 |
+
"normalized": false,
|
| 86 |
+
"special": true
|
| 87 |
+
},
|
| 88 |
+
"200006": {
|
| 89 |
+
"content": "<|content_model_end_sampling|>",
|
| 90 |
+
"single_word": false,
|
| 91 |
+
"lstrip": false,
|
| 92 |
+
"rstrip": false,
|
| 93 |
+
"normalized": false,
|
| 94 |
+
"special": true
|
| 95 |
+
},
|
| 96 |
+
"200007": {
|
| 97 |
+
"content": "<|unused_200007|>",
|
| 98 |
+
"single_word": false,
|
| 99 |
+
"lstrip": false,
|
| 100 |
+
"rstrip": false,
|
| 101 |
+
"normalized": false,
|
| 102 |
+
"special": true
|
| 103 |
+
},
|
| 104 |
+
"200008": {
|
| 105 |
+
"content": "<|content_thinking|>",
|
| 106 |
+
"single_word": false,
|
| 107 |
+
"lstrip": false,
|
| 108 |
+
"rstrip": false,
|
| 109 |
+
"normalized": false,
|
| 110 |
+
"special": true
|
| 111 |
+
},
|
| 112 |
+
"200009": {
|
| 113 |
+
"content": "<|unused_200009|>",
|
| 114 |
+
"single_word": false,
|
| 115 |
+
"lstrip": false,
|
| 116 |
+
"rstrip": false,
|
| 117 |
+
"normalized": false,
|
| 118 |
+
"special": true
|
| 119 |
+
},
|
| 120 |
+
"200010": {
|
| 121 |
+
"content": "<|end_message|>",
|
| 122 |
+
"single_word": false,
|
| 123 |
+
"lstrip": false,
|
| 124 |
+
"rstrip": false,
|
| 125 |
+
"normalized": false,
|
| 126 |
+
"special": true
|
| 127 |
+
},
|
| 128 |
+
"200011": {
|
| 129 |
+
"content": "<|unused_200011|>",
|
| 130 |
+
"single_word": false,
|
| 131 |
+
"lstrip": false,
|
| 132 |
+
"rstrip": false,
|
| 133 |
+
"normalized": false,
|
| 134 |
+
"special": true
|
| 135 |
+
},
|
| 136 |
+
"200012": {
|
| 137 |
+
"content": "<|unused_200012|>",
|
| 138 |
+
"single_word": false,
|
| 139 |
+
"lstrip": false,
|
| 140 |
+
"rstrip": false,
|
| 141 |
+
"normalized": false,
|
| 142 |
+
"special": true
|
| 143 |
+
},
|
| 144 |
+
"200013": {
|
| 145 |
+
"content": "<|unused_200013|>",
|
| 146 |
+
"single_word": false,
|
| 147 |
+
"lstrip": false,
|
| 148 |
+
"rstrip": false,
|
| 149 |
+
"normalized": false,
|
| 150 |
+
"special": true
|
| 151 |
+
},
|
| 152 |
+
"200014": {
|
| 153 |
+
"content": "<|unused_200014|>",
|
| 154 |
+
"single_word": false,
|
| 155 |
+
"lstrip": false,
|
| 156 |
+
"rstrip": false,
|
| 157 |
+
"normalized": false,
|
| 158 |
+
"special": true
|
| 159 |
+
},
|
| 160 |
+
"200015": {
|
| 161 |
+
"content": "<|unused_200015|>",
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"lstrip": false,
|
| 164 |
+
"rstrip": false,
|
| 165 |
+
"normalized": false,
|
| 166 |
+
"special": true
|
| 167 |
+
},
|
| 168 |
+
"200016": {
|
| 169 |
+
"content": "<|unused_200016|>",
|
| 170 |
+
"single_word": false,
|
| 171 |
+
"lstrip": false,
|
| 172 |
+
"rstrip": false,
|
| 173 |
+
"normalized": false,
|
| 174 |
+
"special": true
|
| 175 |
+
},
|
| 176 |
+
"200017": {
|
| 177 |
+
"content": "<|unused_200017|>",
|
| 178 |
+
"single_word": false,
|
| 179 |
+
"lstrip": false,
|
| 180 |
+
"rstrip": false,
|
| 181 |
+
"normalized": false,
|
| 182 |
+
"special": true
|
| 183 |
+
},
|
| 184 |
+
"200018": {
|
| 185 |
+
"content": "<|unused_200018|>",
|
| 186 |
+
"single_word": false,
|
| 187 |
+
"lstrip": false,
|
| 188 |
+
"rstrip": false,
|
| 189 |
+
"normalized": false,
|
| 190 |
+
"special": true
|
| 191 |
+
},
|
| 192 |
+
"200019": {
|
| 193 |
+
"content": "<|unused_200019|>",
|
| 194 |
+
"single_word": false,
|
| 195 |
+
"lstrip": false,
|
| 196 |
+
"rstrip": false,
|
| 197 |
+
"normalized": false,
|
| 198 |
+
"special": true
|
| 199 |
+
},
|
| 200 |
+
"200020": {
|
| 201 |
+
"content": "<|unused_200020|>",
|
| 202 |
+
"single_word": false,
|
| 203 |
+
"lstrip": false,
|
| 204 |
+
"rstrip": false,
|
| 205 |
+
"normalized": false,
|
| 206 |
+
"special": true
|
| 207 |
+
},
|
| 208 |
+
"200021": {
|
| 209 |
+
"content": "<|unused_200021|>",
|
| 210 |
+
"single_word": false,
|
| 211 |
+
"lstrip": false,
|
| 212 |
+
"rstrip": false,
|
| 213 |
+
"normalized": false,
|
| 214 |
+
"special": true
|
| 215 |
+
},
|
| 216 |
+
"200022": {
|
| 217 |
+
"content": "<|content_tool_error|>",
|
| 218 |
+
"single_word": false,
|
| 219 |
+
"lstrip": false,
|
| 220 |
+
"rstrip": false,
|
| 221 |
+
"normalized": false,
|
| 222 |
+
"special": true
|
| 223 |
+
},
|
| 224 |
+
"200023": {
|
| 225 |
+
"content": "<|unused_200023|>",
|
| 226 |
+
"single_word": false,
|
| 227 |
+
"lstrip": false,
|
| 228 |
+
"rstrip": false,
|
| 229 |
+
"normalized": false,
|
| 230 |
+
"special": true
|
| 231 |
+
},
|
| 232 |
+
"200024": {
|
| 233 |
+
"content": "<|content_xml|>",
|
| 234 |
+
"single_word": false,
|
| 235 |
+
"lstrip": false,
|
| 236 |
+
"rstrip": false,
|
| 237 |
+
"normalized": false,
|
| 238 |
+
"special": true
|
| 239 |
+
},
|
| 240 |
+
"200025": {
|
| 241 |
+
"content": "<|unused_200025|>",
|
| 242 |
+
"single_word": false,
|
| 243 |
+
"lstrip": false,
|
| 244 |
+
"rstrip": false,
|
| 245 |
+
"normalized": false,
|
| 246 |
+
"special": true
|
| 247 |
+
},
|
| 248 |
+
"200026": {
|
| 249 |
+
"content": "<|unused_200026|>",
|
| 250 |
+
"single_word": false,
|
| 251 |
+
"lstrip": false,
|
| 252 |
+
"rstrip": false,
|
| 253 |
+
"normalized": false,
|
| 254 |
+
"special": true
|
| 255 |
+
},
|
| 256 |
+
"200027": {
|
| 257 |
+
"content": "<|unused_200027|>",
|
| 258 |
+
"single_word": false,
|
| 259 |
+
"lstrip": false,
|
| 260 |
+
"rstrip": false,
|
| 261 |
+
"normalized": false,
|
| 262 |
+
"special": true
|
| 263 |
+
},
|
| 264 |
+
"200028": {
|
| 265 |
+
"content": "<|begin_of_text|>",
|
| 266 |
+
"single_word": false,
|
| 267 |
+
"lstrip": false,
|
| 268 |
+
"rstrip": false,
|
| 269 |
+
"normalized": false,
|
| 270 |
+
"special": true
|
| 271 |
+
},
|
| 272 |
+
"200029": {
|
| 273 |
+
"content": "<|unused_200029|>",
|
| 274 |
+
"single_word": false,
|
| 275 |
+
"lstrip": false,
|
| 276 |
+
"rstrip": false,
|
| 277 |
+
"normalized": false,
|
| 278 |
+
"special": true
|
| 279 |
+
},
|
| 280 |
+
"200030": {
|
| 281 |
+
"content": "<|unused_200030|>",
|
| 282 |
+
"single_word": false,
|
| 283 |
+
"lstrip": false,
|
| 284 |
+
"rstrip": false,
|
| 285 |
+
"normalized": false,
|
| 286 |
+
"special": true
|
| 287 |
+
},
|
| 288 |
+
"200031": {
|
| 289 |
+
"content": "<|unused_200031|>",
|
| 290 |
+
"single_word": false,
|
| 291 |
+
"lstrip": false,
|
| 292 |
+
"rstrip": false,
|
| 293 |
+
"normalized": false,
|
| 294 |
+
"special": true
|
| 295 |
+
},
|
| 296 |
+
"200032": {
|
| 297 |
+
"content": "<|unused_200032|>",
|
| 298 |
+
"single_word": false,
|
| 299 |
+
"lstrip": false,
|
| 300 |
+
"rstrip": false,
|
| 301 |
+
"normalized": false,
|
| 302 |
+
"special": true
|
| 303 |
+
},
|
| 304 |
+
"200033": {
|
| 305 |
+
"content": "<|unused_200033|>",
|
| 306 |
+
"single_word": false,
|
| 307 |
+
"lstrip": false,
|
| 308 |
+
"rstrip": false,
|
| 309 |
+
"normalized": false,
|
| 310 |
+
"special": true
|
| 311 |
+
},
|
| 312 |
+
"200034": {
|
| 313 |
+
"content": "<|unused_200034|>",
|
| 314 |
+
"single_word": false,
|
| 315 |
+
"lstrip": false,
|
| 316 |
+
"rstrip": false,
|
| 317 |
+
"normalized": false,
|
| 318 |
+
"special": true
|
| 319 |
+
},
|
| 320 |
+
"200035": {
|
| 321 |
+
"content": "<|unused_200035|>",
|
| 322 |
+
"single_word": false,
|
| 323 |
+
"lstrip": false,
|
| 324 |
+
"rstrip": false,
|
| 325 |
+
"normalized": false,
|
| 326 |
+
"special": true
|
| 327 |
+
},
|
| 328 |
+
"200036": {
|
| 329 |
+
"content": "<|unused_200036|>",
|
| 330 |
+
"single_word": false,
|
| 331 |
+
"lstrip": false,
|
| 332 |
+
"rstrip": false,
|
| 333 |
+
"normalized": false,
|
| 334 |
+
"special": true
|
| 335 |
+
},
|
| 336 |
+
"200037": {
|
| 337 |
+
"content": "<|unused_200037|>",
|
| 338 |
+
"single_word": false,
|
| 339 |
+
"lstrip": false,
|
| 340 |
+
"rstrip": false,
|
| 341 |
+
"normalized": false,
|
| 342 |
+
"special": true
|
| 343 |
+
},
|
| 344 |
+
"200038": {
|
| 345 |
+
"content": "<|unused_200038|>",
|
| 346 |
+
"single_word": false,
|
| 347 |
+
"lstrip": false,
|
| 348 |
+
"rstrip": false,
|
| 349 |
+
"normalized": false,
|
| 350 |
+
"special": true
|
| 351 |
+
},
|
| 352 |
+
"200039": {
|
| 353 |
+
"content": "<|unused_200039|>",
|
| 354 |
+
"single_word": false,
|
| 355 |
+
"lstrip": false,
|
| 356 |
+
"rstrip": false,
|
| 357 |
+
"normalized": false,
|
| 358 |
+
"special": true
|
| 359 |
+
},
|
| 360 |
+
"200040": {
|
| 361 |
+
"content": "<|unused_200040|>",
|
| 362 |
+
"single_word": false,
|
| 363 |
+
"lstrip": false,
|
| 364 |
+
"rstrip": false,
|
| 365 |
+
"normalized": false,
|
| 366 |
+
"special": true
|
| 367 |
+
},
|
| 368 |
+
"200041": {
|
| 369 |
+
"content": "<|unused_200041|>",
|
| 370 |
+
"single_word": false,
|
| 371 |
+
"lstrip": false,
|
| 372 |
+
"rstrip": false,
|
| 373 |
+
"normalized": false,
|
| 374 |
+
"special": true
|
| 375 |
+
},
|
| 376 |
+
"200042": {
|
| 377 |
+
"content": "<|unused_200042|>",
|
| 378 |
+
"single_word": false,
|
| 379 |
+
"lstrip": false,
|
| 380 |
+
"rstrip": false,
|
| 381 |
+
"normalized": false,
|
| 382 |
+
"special": true
|
| 383 |
+
},
|
| 384 |
+
"200043": {
|
| 385 |
+
"content": "<|unused_200043|>",
|
| 386 |
+
"single_word": false,
|
| 387 |
+
"lstrip": false,
|
| 388 |
+
"rstrip": false,
|
| 389 |
+
"normalized": false,
|
| 390 |
+
"special": true
|
| 391 |
+
},
|
| 392 |
+
"200044": {
|
| 393 |
+
"content": "<|unused_200044|>",
|
| 394 |
+
"single_word": false,
|
| 395 |
+
"lstrip": false,
|
| 396 |
+
"rstrip": false,
|
| 397 |
+
"normalized": false,
|
| 398 |
+
"special": true
|
| 399 |
+
},
|
| 400 |
+
"200045": {
|
| 401 |
+
"content": "<|unused_200045|>",
|
| 402 |
+
"single_word": false,
|
| 403 |
+
"lstrip": false,
|
| 404 |
+
"rstrip": false,
|
| 405 |
+
"normalized": false,
|
| 406 |
+
"special": true
|
| 407 |
+
},
|
| 408 |
+
"200046": {
|
| 409 |
+
"content": "<|unused_200046|>",
|
| 410 |
+
"single_word": false,
|
| 411 |
+
"lstrip": false,
|
| 412 |
+
"rstrip": false,
|
| 413 |
+
"normalized": false,
|
| 414 |
+
"special": true
|
| 415 |
+
},
|
| 416 |
+
"200047": {
|
| 417 |
+
"content": "<|unused_200047|>",
|
| 418 |
+
"single_word": false,
|
| 419 |
+
"lstrip": false,
|
| 420 |
+
"rstrip": false,
|
| 421 |
+
"normalized": false,
|
| 422 |
+
"special": true
|
| 423 |
+
},
|
| 424 |
+
"200048": {
|
| 425 |
+
"content": "<|unused_200048|>",
|
| 426 |
+
"single_word": false,
|
| 427 |
+
"lstrip": false,
|
| 428 |
+
"rstrip": false,
|
| 429 |
+
"normalized": false,
|
| 430 |
+
"special": true
|
| 431 |
+
},
|
| 432 |
+
"200049": {
|
| 433 |
+
"content": "<|content_invoke_tool_json|>",
|
| 434 |
+
"single_word": false,
|
| 435 |
+
"lstrip": false,
|
| 436 |
+
"rstrip": false,
|
| 437 |
+
"normalized": false,
|
| 438 |
+
"special": true
|
| 439 |
+
},
|
| 440 |
+
"200050": {
|
| 441 |
+
"content": "<|unused_200050|>",
|
| 442 |
+
"single_word": false,
|
| 443 |
+
"lstrip": false,
|
| 444 |
+
"rstrip": false,
|
| 445 |
+
"normalized": false,
|
| 446 |
+
"special": true
|
| 447 |
+
},
|
| 448 |
+
"200051": {
|
| 449 |
+
"content": "<|unused_200051|>",
|
| 450 |
+
"single_word": false,
|
| 451 |
+
"lstrip": false,
|
| 452 |
+
"rstrip": false,
|
| 453 |
+
"normalized": false,
|
| 454 |
+
"special": true
|
| 455 |
+
},
|
| 456 |
+
"200052": {
|
| 457 |
+
"content": "<|unused_200052|>",
|
| 458 |
+
"single_word": false,
|
| 459 |
+
"lstrip": false,
|
| 460 |
+
"rstrip": false,
|
| 461 |
+
"normalized": false,
|
| 462 |
+
"special": true
|
| 463 |
+
},
|
| 464 |
+
"200053": {
|
| 465 |
+
"content": "<|unused_200053|>",
|
| 466 |
+
"single_word": false,
|
| 467 |
+
"lstrip": false,
|
| 468 |
+
"rstrip": false,
|
| 469 |
+
"normalized": false,
|
| 470 |
+
"special": true
|
| 471 |
+
},
|
| 472 |
+
"200054": {
|
| 473 |
+
"content": "<|unused_200054|>",
|
| 474 |
+
"single_word": false,
|
| 475 |
+
"lstrip": false,
|
| 476 |
+
"rstrip": false,
|
| 477 |
+
"normalized": false,
|
| 478 |
+
"special": true
|
| 479 |
+
},
|
| 480 |
+
"200055": {
|
| 481 |
+
"content": "<|unused_200055|>",
|
| 482 |
+
"single_word": false,
|
| 483 |
+
"lstrip": false,
|
| 484 |
+
"rstrip": false,
|
| 485 |
+
"normalized": false,
|
| 486 |
+
"special": true
|
| 487 |
+
},
|
| 488 |
+
"200056": {
|
| 489 |
+
"content": "<|unused_200056|>",
|
| 490 |
+
"single_word": false,
|
| 491 |
+
"lstrip": false,
|
| 492 |
+
"rstrip": false,
|
| 493 |
+
"normalized": false,
|
| 494 |
+
"special": true
|
| 495 |
+
},
|
| 496 |
+
"200057": {
|
| 497 |
+
"content": "<|content_invoke_tool_text|>",
|
| 498 |
+
"single_word": false,
|
| 499 |
+
"lstrip": false,
|
| 500 |
+
"rstrip": false,
|
| 501 |
+
"normalized": false,
|
| 502 |
+
"special": true
|
| 503 |
+
}
|
| 504 |
+
},
|
| 505 |
+
"response_template": {
|
| 506 |
+
"defaults": {
|
| 507 |
+
"role": "assistant"
|
| 508 |
+
},
|
| 509 |
+
"start_anchor_pattern": "<\\|end_message\\|>|<\\|content_model_end_sampling\\|>",
|
| 510 |
+
"fields": {
|
| 511 |
+
"reasoning_content": {
|
| 512 |
+
"open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_thinking\\|>",
|
| 513 |
+
"close": "<|end_message|>",
|
| 514 |
+
"repeats": true,
|
| 515 |
+
"join": "",
|
| 516 |
+
"content": "text",
|
| 517 |
+
"content_args": {
|
| 518 |
+
"strip": false
|
| 519 |
+
}
|
| 520 |
+
},
|
| 521 |
+
"content": {
|
| 522 |
+
"open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_(?:text|tool_error|invoke_tool_text)\\|>",
|
| 523 |
+
"close": "<|end_message|>",
|
| 524 |
+
"repeats": true,
|
| 525 |
+
"join": "",
|
| 526 |
+
"content": "text",
|
| 527 |
+
"content_args": {
|
| 528 |
+
"strip": false
|
| 529 |
+
}
|
| 530 |
+
},
|
| 531 |
+
"tool_calls": {
|
| 532 |
+
"open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_invoke_tool_json\\|>",
|
| 533 |
+
"close": "<|end_message|>",
|
| 534 |
+
"repeats": true,
|
| 535 |
+
"content": "json",
|
| 536 |
+
"transform": {
|
| 537 |
+
"type": "function",
|
| 538 |
+
"function": {
|
| 539 |
+
"name": "{content.name}",
|
| 540 |
+
"arguments": "{content.args}"
|
| 541 |
+
}
|
| 542 |
+
}
|
| 543 |
+
}
|
| 544 |
+
}
|
| 545 |
+
}
|
| 546 |
+
}
|