yujiepan commited on
Commit
08c77f0
·
verified ·
1 Parent(s): fab7aa4

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
.meta/create_model.py ADDED
@@ -0,0 +1,561 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
2
+ import json
3
+ import re
4
+ from pathlib import Path
5
+
6
+ import torch
7
+ from huggingface_hub import get_safetensors_metadata, hf_hub_download, parse_safetensors_file_metadata
8
+ from safetensors.torch import save_file
9
+ from transformers import Inkling2Config, Inkling2ForConditionalGeneration, set_seed
10
+
11
+ folder = Path(save_folder)
12
+ folder.mkdir(parents=True, exist_ok=True)
13
+ raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
14
+ precision = json.loads(
15
+ Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
16
+ )
17
+ # Two full 5-SWA/1-global cycles, sampling both source expert precision regions.
18
+ source_layers = [*range(6), *range(42, 48)]
19
+ hidden_size = 32
20
+ intermediate_size = 32
21
+ text = raw["text_config"]
22
+ text.update(
23
+ hidden_size=hidden_size,
24
+ intermediate_size=intermediate_size,
25
+ num_hidden_layers=12,
26
+ num_attention_heads=24,
27
+ num_key_value_heads=4,
28
+ head_dim=32,
29
+ swa_num_attention_heads=24,
30
+ swa_num_key_value_heads=4,
31
+ swa_head_dim=32,
32
+ n_routed_experts=96,
33
+ tie_word_embeddings=True,
34
+ local_layer_ids=[i for i in range(12) if i % 6 != 5],
35
+ )
36
+ raw["tie_word_embeddings"] = True
37
+ raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
38
+ raw["vision_config"].update(
39
+ hidden_size=32,
40
+ num_hidden_layers=4,
41
+ num_attention_heads=4,
42
+ projection_hidden_dim=32,
43
+ text_hidden_size=hidden_size,
44
+ )
45
+ mtp = raw["mtp_config"]
46
+ mtp.update(
47
+ hidden_size=hidden_size,
48
+ intermediate_size=128,
49
+ num_attention_heads=24,
50
+ num_key_value_heads=4,
51
+ head_dim=32,
52
+ markov_rank=8,
53
+ target_layer_ids=[5, 12],
54
+ )
55
+ # from_dict adds an MXFP8-only quantizer to source configs. Construction must
56
+ # bypass it; the final artifact retains its authoritative hf_quant_config.
57
+ construction = json.loads(json.dumps(raw))
58
+ construction["vision_config"]["text_hidden_size"] = hidden_size
59
+ config = Inkling2Config(**construction)
60
+ set_seed(42)
61
+ previous_dtype = torch.get_default_dtype()
62
+ try:
63
+ torch.set_default_dtype(torch.bfloat16)
64
+ model = Inkling2ForConditionalGeneration(config)
65
+ finally:
66
+ torch.set_default_dtype(previous_dtype)
67
+ with torch.no_grad():
68
+ for name, parameter in sorted(model.named_parameters()):
69
+ if name.endswith("bias"):
70
+ parameter.zero_()
71
+ elif "norm" in name and parameter.ndim == 1:
72
+ parameter.fill_(1)
73
+ elif name.endswith(".scale"):
74
+ parameter.fill_(1)
75
+ else:
76
+ parameter.normal_(0, 0.02)
77
+ # Inkling 2's concrete Transformers class deliberately has no tied-weight
78
+ # mapping. Tie explicitly, then serialize the embedding only.
79
+ model.lm_head.weight = model.model.language_model.embed_tokens.weight
80
+
81
+ # Reverse Transformers' documented source-to-HF name conversion. Retain the
82
+ # source's interleaved gate/up rows and original serialized precision.
83
+ def original_name(name):
84
+ name = name.replace("lm_head.weight", "model.llm.unembed.weight")
85
+ name = name.replace("model.language_model.", "model.llm.")
86
+ name = name.replace("embed_tokens.embed_norm.", "embed_norm.").replace("embed_tokens.", "embed.")
87
+ if match := re.search(r"layers\.(\d+)\.attention_input\.query$", name):
88
+ return f"model.llm.attn_res.attn_queries.{int(match[1]) - 1}"
89
+ if match := re.search(r"layers\.(\d+)\.mlp_input\.query$", name):
90
+ return f"model.llm.attn_res.mlp_queries.{match[1]}"
91
+ name = name.replace("final_input.query", "attn_res.output_query")
92
+ name = name.replace("self_attn.", "attn.") if name.startswith("model.llm.") else name
93
+ for old, new in (
94
+ ("attn.q_proj", "attn.wq_du"),
95
+ ("attn.k_proj", "attn.wk_dv"),
96
+ ("attn.v_proj", "attn.wv_dv"),
97
+ ("attn.r_proj", "attn.wr_du"),
98
+ ("attn.o_proj", "attn.wo_ud"),
99
+ ("attn.gate_proj", "attn.gate_attn"),
100
+ ("attn.sink_proj", "attn.attn_sink_proj"),
101
+ ("post_attention_convolution.conv1d", "attn_sconv"),
102
+ ("post_mlp_convolution.conv1d", "mlp_sconv"),
103
+ ("sconv.conv1d", "sconv"),
104
+ ("sconv.static_weight", "sconv.static_w"),
105
+ ("sconv.weight_proj.weight", "sconv.weight_proj"),
106
+ ("post_attention_layernorm", "mlp_norm"),
107
+ ("input_layernorm", "attn_norm"),
108
+ ("q_attn_norm", "q_input_norm"),
109
+ ("k_attn_norm", "k_input_norm"),
110
+ ("v_attn_norm", "v_input_norm"),
111
+ ("mlp.gate.e_score_correction_bias", "mlp.gate.bias"),
112
+ ("mlp.experts.gate_up_proj", "mlp.experts.w13_weight"),
113
+ ("mlp.experts.down_proj", "mlp.experts.w2_weight"),
114
+ (
115
+ "model.vision_tower.embeddings.position_embedding.weight",
116
+ "model.visual.backbone.embeddings.position_embedding",
117
+ ),
118
+ ("model.vision_tower.embeddings.", "model.visual.backbone.embeddings."),
119
+ ("model.vision_tower.pre_layernorm.", "model.visual.backbone.ln_pre."),
120
+ ("model.vision_tower.merger.linear_1.", "model.visual.proj_in."),
121
+ ("model.vision_tower.merger.linear_2.", "model.visual.proj_out."),
122
+ ("model.vision_tower.layers.", "model.visual.backbone.layers."),
123
+ (".layernorm_before.", ".ln_1."),
124
+ (".layernorm_after.", ".ln_2."),
125
+ (".mlp.fc1.", ".mlp.c_fc."),
126
+ (".mlp.fc2.", ".mlp.c_proj."),
127
+ (".lambda_1", ".ls_1.gamma"),
128
+ (".lambda_2", ".ls_2.gamma"),
129
+ ):
130
+ if old.startswith("attn.") and not name.startswith("model.llm."):
131
+ continue
132
+ name = name.replace(old, new)
133
+ return name
134
+
135
+ def source_name(name):
136
+ if match := re.search(r"model\.llm\.layers\.(\d+)\.", name):
137
+ return name.replace(match[0], f"model.llm.layers.{source_layers[int(match[1])] }.", 1)
138
+ return name
139
+
140
+ groups = precision["model"]["parameter_groups"]
141
+ group_for = {name: group for group in groups for name in group["tensors"]}
142
+ source_metadata = get_safetensors_metadata(source_model_id, revision=source_revision)
143
+ headers = {
144
+ name: tensor for file in source_metadata.files_metadata.values() for name, tensor in file.tensors.items()
145
+ }
146
+ state = {}
147
+ for hf_name, value in sorted(model.state_dict().items()):
148
+ if hf_name == "lm_head.weight":
149
+ continue
150
+ name = original_name(hf_name)
151
+ source = source_name(name)
152
+ if source not in headers:
153
+ raise ValueError(f"Constructed tensor has no source counterpart: {hf_name} -> {source}")
154
+ group = group_for[source]
155
+ value = value.detach().cpu().contiguous()
156
+ if name.endswith("w13_weight"):
157
+ value = (
158
+ value.reshape(value.shape[0], 2, intermediate_size, hidden_size).transpose(1, 2).reshape(value.shape)
159
+ )
160
+ compute = group["compute_dtype"]
161
+ if compute == "nvfp4":
162
+ # Source E2M1 nibbles use per-16-element E4M3 scales, with each
163
+ # expert's outer scale stored as amax / denominator in FP32.
164
+ value = value.float().contiguous()
165
+ amax = value.abs().amax(dim=(1, 2)).clamp_min(1e-12)
166
+ outer = amax[:, None, None, None] / 1536.0
167
+ blocks = value.reshape(*value.shape[:-1], -1, 16)
168
+ scales = (blocks.abs().amax(-1) / (6 * outer.squeeze(-1))).clamp_max(448).to(torch.float8_e4m3fn)
169
+ normalized = blocks / (scales.float().unsqueeze(-1) * outer).clamp_min(1e-12)
170
+ levels = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6])
171
+ codes = (normalized.abs().unsqueeze(-1) - levels).abs().argmin(-1).to(torch.uint8)
172
+ codes |= (normalized < 0).to(torch.uint8) * 8
173
+ codes = codes.reshape(value.shape)
174
+ state[name] = (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous()
175
+ state[name + "_scale"] = scales.contiguous()
176
+ state[name + "_amax"] = amax
177
+ state[name + "_outer_scale_denominator"] = torch.tensor([1536.0])
178
+ elif compute == "mxfp8":
179
+ # The pinned source stores UE8M0 scales per row/32 columns, even
180
+ # though its computation descriptor says weight_block_shape=[32,32].
181
+ blocks = value.float().reshape(*value.shape[:-1], -1, 32)
182
+ exponents = torch.ceil(torch.log2(blocks.abs().amax(-1).clamp_min(1e-12) / 448)).clamp(-127, 127)
183
+ scales = torch.pow(2.0, exponents)
184
+ state[name] = (blocks / scales.unsqueeze(-1)).reshape(value.shape).to(torch.float8_e4m3fn)
185
+ state[name + "_scale"] = (exponents + 127).to(torch.uint8)
186
+ else:
187
+ state[name] = value.to(getattr(torch, group["output_dtype"]))
188
+
189
+ # Build the independent DSpark draft using its authoritative BF16 header.
190
+ draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
191
+ draft = {}
192
+ draft_module = torch.nn.Module()
193
+ replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 24 * 32, 1024: 4 * 32, 128: 32}
194
+ for name, tensor in sorted(draft_metadata.tensors.items()):
195
+ if name == "fuse_weight":
196
+ shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
197
+ elif name == "markov_head.markov_w1.weight":
198
+ shape = (text["vocab_size"], mtp["markov_rank"])
199
+ elif name == "markov_head.hidden_out.weight":
200
+ shape = (hidden_size, mtp["markov_rank"])
201
+ elif name == "markov_head.rnn_proj.weight":
202
+ shape = (3 * mtp["markov_rank"], 2 * hidden_size + 2 * mtp["markov_rank"])
203
+ elif name == "markov_head.rnn_proj.bias":
204
+ shape = (3 * mtp["markov_rank"],)
205
+ else:
206
+ shape = tuple(replacements.get(n, n) for n in tensor.shape)
207
+ value = torch.empty(shape, dtype=torch.bfloat16)
208
+ if name.endswith("bias"):
209
+ value.zero_()
210
+ elif "norm" in name:
211
+ value.fill_(1)
212
+ else:
213
+ value.normal_(0, 0.02)
214
+ draft[name] = value
215
+ module = draft_module
216
+ parts = name.split(".")
217
+ for part in parts[:-1]:
218
+ if part not in module._modules:
219
+ module.add_module(part, torch.nn.Module())
220
+ module = module._modules[part]
221
+ module.register_parameter(parts[-1], torch.nn.Parameter(value))
222
+ model.add_module("mtp", draft_module)
223
+ dtype_names = {
224
+ torch.float32: "F32",
225
+ torch.bfloat16: "BF16",
226
+ torch.uint8: "U8",
227
+ torch.float8_e4m3fn: "F8_E4M3",
228
+ }
229
+ for name, tensor in state.items():
230
+ source_tensor = headers[source_name(name)]
231
+ if dtype_names[tensor.dtype] != source_tensor.dtype or tensor.ndim != len(source_tensor.shape):
232
+ raise ValueError(f"Source precision/rank mismatch: {name}")
233
+ if tensor.is_floating_point() and not torch.isfinite(tensor.float()).all():
234
+ raise ValueError(f"Nonfinite checkpoint tensor: {name}")
235
+ save_file(state, str(folder / "model.safetensors"), metadata={"format": "pt"})
236
+ save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"})
237
+
238
+ # Remap every precision descriptor to exactly the retained source layers.
239
+ remapped_groups = []
240
+ for group in groups:
241
+ names = [name for name in state if source_name(name) in group["tensors"]]
242
+ if "model.llm.unembed.weight" in group["tensors"]:
243
+ # The adapter supplies this virtual tensor to the native loader;
244
+ # keep its FP32-logit computation descriptor even without a file copy.
245
+ names.append("model.llm.unembed.weight")
246
+ if names:
247
+ remapped_groups.append({**group, "tensors": sorted(names)})
248
+ precision["model"]["parameter_groups"] = remapped_groups
249
+ (folder / "hf_quant_config.json").write_text(json.dumps(precision, indent=2) + "\n")
250
+ (folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n")
251
+ for name in (
252
+ "tokenizer.json",
253
+ "tokenizer_config.json",
254
+ "special_tokens_map.json",
255
+ "chat_template.jinja",
256
+ "processor_config.json",
257
+ "tiktoken/tokenizer.model",
258
+ ):
259
+ path = folder / name
260
+ path.parent.mkdir(parents=True, exist_ok=True)
261
+ path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
262
+ plugin_code = '''"""Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
263
+
264
+ import logging
265
+
266
+ logger = logging.getLogger(__name__)
267
+
268
+
269
+ def _config_init(original, config, *args, **kwargs):
270
+ tied = kwargs.get("tie_word_embeddings", False)
271
+ if tied:
272
+ kwargs = {**kwargs, "tie_word_embeddings": False}
273
+ original(config, *args, **kwargs)
274
+ if tied:
275
+ config.tie_word_embeddings = True
276
+
277
+
278
+ def _load_weights(original, model, weights):
279
+ if not model.text_config.tie_word_embeddings or model.llm is None:
280
+ return original(model, weights)
281
+
282
+ def with_head():
283
+ embedding = None
284
+ for name, tensor in weights:
285
+ if name == "model.llm.unembed.weight":
286
+ raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
287
+ if name == "model.llm.embed.weight":
288
+ embedding = tensor
289
+ yield name, tensor
290
+ if embedding is None:
291
+ raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
292
+ yield "model.llm.unembed.weight", embedding
293
+
294
+ loaded = original(model, with_head())
295
+ embedding, head = model.llm.embed_tokens, model.llm.lm_head
296
+ if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
297
+ raise ValueError("Tied embedding/head tensor-parallel partitions must match")
298
+ head.weight = embedding.weight
299
+ assert head.weight is embedding.weight
300
+ logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
301
+ return loaded
302
+
303
+
304
+ class _PaddedMarlin(tuple):
305
+ def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
306
+ value = super().__new__(cls, packed)
307
+ value.input_width = input_width
308
+ value.output_width = output_width
309
+ value.padded_input = padded_input
310
+ value.padded_output = padded_output
311
+ return value
312
+
313
+
314
+ def _pad_matrix(tensor, rows, columns, fill=0):
315
+ output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
316
+ output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
317
+ return output
318
+
319
+
320
+ def _repack_marlin(original, weight, scales):
321
+ import torch
322
+
323
+ nvfp4 = weight.dtype == torch.uint8
324
+ packing = 2 if nvfp4 else 1
325
+ block = 16 if nvfp4 else 32
326
+ n, k = weight.shape[-2], weight.shape[-1] * packing
327
+ padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
328
+ if (padded_k, padded_n) == (k, n):
329
+ return original(weight, scales)
330
+ weight = _pad_matrix(weight, padded_n, padded_k // packing)
331
+ # Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
332
+ # Zero weight bytes make padded products zero in both precision families.
333
+ scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
334
+ packed = original(weight, scales)
335
+ return _PaddedMarlin(packed, k, n, padded_k, padded_n)
336
+
337
+
338
+ def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
339
+ if not isinstance(packed, _PaddedMarlin):
340
+ return original(x, packed, out_size, *args, **kwargs)
341
+ if x.shape[-1] != packed.input_width or out_size != packed.output_width:
342
+ raise ValueError("Marlin padding metadata does not match the expert projection")
343
+ padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
344
+ padded[..., :x.shape[-1]].copy_(x)
345
+ if original.__name__ == "gemv" and packed.padded_input == 128:
346
+ # Eight K tiles fit one split of eight warps. The large-model heuristic
347
+ # otherwise chooses four splits/four warps for short decode batches.
348
+ kwargs = {**kwargs, "splits": 1, "warps": 8}
349
+ result = original(padded, packed, packed.padded_output, *args, **kwargs)
350
+ return result[..., :out_size].contiguous()
351
+
352
+
353
+ def _relative_fa4(original, *args, **kwargs):
354
+ q = args[0] if args else kwargs.get("q")
355
+ if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
356
+ and kwargs.get("tile_mn") is None):
357
+ # The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
358
+ # relative-bias kernel explicitly requires M=N=128. Use its tile API.
359
+ kwargs = {**kwargs, "tile_mn": (128, 128)}
360
+ return original(*args, **kwargs)
361
+
362
+
363
+ def _attn_res_config(original, width, *args, **kwargs):
364
+ threads, stages, vector, cluster = original(width, *args, **kwargs)
365
+ if width == 32:
366
+ # One compute warp covers the original 32 residual coordinates.
367
+ # Keep the same CuTe kernel, RMS normalization, and source weighting.
368
+ return 32, stages, 1, 1
369
+ return threads, stages, vector, cluster
370
+
371
+
372
+ def _small_all_reduce(original, x, group, **kwargs):
373
+ if x.ndim == 2 and x.shape[-1] == 32:
374
+ return group.all_reduce(x.clone())
375
+ return original(x, group, **kwargs)
376
+
377
+
378
+ def _small_multicast_input(original, group, shape, dtype):
379
+ if len(shape) == 2 and shape[-1] == 32:
380
+ import torch
381
+
382
+ return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
383
+ return original(group, shape, dtype)
384
+
385
+
386
+ def _small_multicast_reduce(original, x, group):
387
+ if x.ndim == 2 and x.shape[-1] == 32:
388
+ x.copy_(group.all_reduce(x))
389
+ return x
390
+ return original(x, group)
391
+
392
+
393
+ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
394
+ if cols == 32:
395
+ # Use the existing FP32 combine path instead of the GB300-tuned
396
+ # symmetric-memory one-shot collective for these tiny projections.
397
+ return None
398
+ return original(group, rows, cols, dtype, tag=tag)
399
+
400
+
401
+ def register():
402
+ from sglang.srt.plugins.hook_registry import HookRegistry, HookType
403
+
404
+ HookRegistry.register(
405
+ "sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
406
+ )
407
+ HookRegistry.register(
408
+ "sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
409
+ )
410
+ prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
411
+ for name in ("repack", "repack_mxfp8"):
412
+ HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
413
+ for name in ("gemm", "gemv"):
414
+ HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
415
+ HookRegistry.register(
416
+ "sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
417
+ _relative_fa4, HookType.AROUND,
418
+ )
419
+ HookRegistry.register(
420
+ "sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
421
+ _attn_res_config, HookType.AROUND,
422
+ )
423
+ prefix = "sglang.kernels.ops.inkling2.comm"
424
+ for name, hook in (
425
+ ("all_reduce", _small_all_reduce),
426
+ ("multicast_all_reduce_input", _small_multicast_input),
427
+ ("multicast_all_reduce", _small_multicast_reduce),
428
+ ("oneshot_all_reduce_input", _small_oneshot_input),
429
+ ):
430
+ HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
431
+ '''
432
+ (folder / "sglang_inkling2_tied.py").write_text(plugin_code)
433
+ hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
434
+
435
+ from pathlib import Path
436
+
437
+ import torch
438
+ from huggingface_hub import hf_hub_download
439
+ from safetensors.torch import load_file
440
+ from transformers import Inkling2Config, Inkling2ForConditionalGeneration
441
+ from transformers.conversion_mapping import get_model_conversion_mapping
442
+ from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
443
+
444
+
445
+ def dequantize(weight, scale, amax=None, denominator=None):
446
+ if weight.dtype == torch.uint8:
447
+ if amax is None or denominator is None:
448
+ raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
449
+ codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
450
+ levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
451
+ values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
452
+ return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
453
+ if weight.dtype == torch.float8_e4m3fn:
454
+ scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
455
+ return weight.float() * scales
456
+ raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
457
+
458
+
459
+ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
460
+ @classmethod
461
+ def from_pretrained(
462
+ cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
463
+ revision=None, cache_dir=None, local_files_only=False, token=None,
464
+ attn_implementation="eager", **kwargs,
465
+ ):
466
+ if model_args:
467
+ raise ValueError("Inkling 2 does not take positional construction arguments")
468
+ options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
469
+ root = Path(pretrained_model_name_or_path)
470
+
471
+ def resolve(filename):
472
+ if root.is_dir():
473
+ return root / filename
474
+ return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
475
+
476
+ import json
477
+
478
+ raw = json.loads(resolve("config.json").read_text())
479
+ # The native config otherwise assumes the all-MXFP8 sibling. This
480
+ # adapter dequantizes each tensor according to its actual stored dtype.
481
+ raw.pop("quantization_config", None)
482
+ config = Inkling2Config(**raw)
483
+ config._attn_implementation = attn_implementation
484
+ dtype = kwargs.pop("torch_dtype", dtype)
485
+ if dtype in (None, "auto"):
486
+ dtype = torch.bfloat16
487
+ elif isinstance(dtype, str):
488
+ dtype = getattr(torch, dtype)
489
+ if dtype not in (torch.float32, torch.bfloat16, torch.float16):
490
+ raise ValueError("The Transformers adapter requires a floating compute dtype")
491
+ device_map = kwargs.pop("device_map", None)
492
+ if device_map == "auto":
493
+ device_map = "cuda" if torch.cuda.is_available() else "cpu"
494
+ if isinstance(device_map, dict):
495
+ if set(device_map) != {""}:
496
+ raise ValueError("Use a single device for this tiny Transformers adapter")
497
+ device_map = device_map[""]
498
+ allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
499
+ unexpected = set(kwargs) - allowed
500
+ if unexpected:
501
+ raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
502
+ model = Inkling2ForConditionalGeneration(config)
503
+ transforms = get_model_conversion_mapping(model)
504
+ renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
505
+ converters = [t for t in transforms if isinstance(t, WeightConverter)]
506
+ source = load_file(str(resolve("model.safetensors")))
507
+ state = {}
508
+ for name, weight in source.items():
509
+ if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
510
+ continue
511
+ if name + "_scale" in source:
512
+ weight = dequantize(
513
+ weight, source[name + "_scale"], source.get(name + "_amax"),
514
+ source.get(name + "_outer_scale_denominator"),
515
+ )
516
+ target, _ = rename_source_key(name, renamings, converters)
517
+ if name.endswith("w13_weight"):
518
+ # The source interleaves gate and up; Transformers chunks them.
519
+ e, rows, columns = weight.shape
520
+ weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
521
+ if target in state:
522
+ raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
523
+ keep_float32 = source[name].dtype == torch.float32 or any(
524
+ part in target for part in model._keep_in_fp32_modules_strict
525
+ )
526
+ target_dtype = torch.float32 if keep_float32 else dtype
527
+ state[target] = weight.to(target_dtype).contiguous()
528
+ state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
529
+ model.load_state_dict(state, strict=True, assign=True)
530
+ model.eval()
531
+ if device_map is not None:
532
+ model.to(device_map)
533
+ model.lm_head.weight = model.model.language_model.embed_tokens.weight
534
+ return model
535
+ '''
536
+ (folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
537
+ (folder / "pyproject.toml").write_text("""[build-system]
538
+ requires = ["setuptools>=68"]
539
+ build-backend = "setuptools.build_meta"
540
+
541
+ [project]
542
+ name = "sglang-inkling2-tied"
543
+ version = "0.1.0"
544
+ description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
545
+ requires-python = ">=3.10"
546
+
547
+ [project.entry-points."sglang.srt.plugins"]
548
+ inkling2_tied = "sglang_inkling2_tied:register"
549
+
550
+ [tool.setuptools]
551
+ py-modules = ["sglang_inkling2_tied"]
552
+ """)
553
+ return str(model)
554
+
555
+
556
+ if __name__ == "__main__":
557
+ create_model(
558
+ source_model_id="thinkingmachines/Inkling-2-Small",
559
+ source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
560
+ save_folder="/tmp/inkling-2-tiny-random",
561
+ )
.meta/env.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source_model": {
3
+ "name": "thinkingmachines/Inkling-2-Small",
4
+ "commit_id": "aac974122acd33ddb0d3e1f54433a9c5dd2249ce"
5
+ },
6
+ "environments": {
7
+ "creation": {
8
+ "python": "3.12.13",
9
+ "packages": {
10
+ "huggingface_hub": "2.3.0",
11
+ "safetensors": "0.8.0",
12
+ "tokenizers": "0.23.3",
13
+ "torch": "2.13.0+cu126",
14
+ "transformers": "5.23.0.dev0"
15
+ },
16
+ "cuda": "12.6"
17
+ },
18
+ "sglang": {
19
+ "python": "3.12.13",
20
+ "packages": {
21
+ "Pillow": "12.3.0",
22
+ "flashinfer-python": "0.7.4",
23
+ "huggingface_hub": "1.33.0",
24
+ "nvidia-cutlass-dsl": "4.8.0",
25
+ "safetensors": "0.8.0",
26
+ "sglang": "0.0.0.dev0",
27
+ "sglang-inkling2-tied": "0.1.0",
28
+ "sglang-kernel": "0.4.10",
29
+ "tiktoken": "0.14.0",
30
+ "tml-renderers": "0.4.1",
31
+ "tokenizers": "0.23.2",
32
+ "torch": "2.14.1+cu130",
33
+ "transformers": "5.19.0",
34
+ "triton": "3.8.0"
35
+ },
36
+ "cuda": "13.0",
37
+ "hardware": {
38
+ "gpu_model": "NVIDIA H20"
39
+ }
40
+ },
41
+ "transformers": {
42
+ "python": "3.12.13",
43
+ "packages": {
44
+ "Pillow": "12.3.0",
45
+ "huggingface_hub": "2.3.0",
46
+ "safetensors": "0.8.0",
47
+ "tokenizers": "0.23.3",
48
+ "torch": "2.13.0+cu126",
49
+ "transformers": "5.23.0.dev0"
50
+ },
51
+ "cuda": "12.6",
52
+ "hardware": {
53
+ "gpu_model": "NVIDIA H20"
54
+ }
55
+ }
56
+ }
57
+ }
.meta/model.log ADDED
@@ -0,0 +1,400 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Inkling2ForConditionalGeneration(
2
+ (model): Inkling2Model(
3
+ (language_model): Inkling2TextModel(
4
+ (embed_tokens): Inkling2NormedEmbedding(
5
+ 201024, 32
6
+ (embed_norm): Inkling2RMSNorm((32,), eps=1e-06)
7
+ )
8
+ (layers): ModuleList(
9
+ (0): Inkling2SlidingDecoderLayer(
10
+ (mlp_input): Inkling2BlockAttentionResidual(
11
+ (norm): Inkling2UnweightedRMSNorm()
12
+ )
13
+ (self_attn): Inkling2SlidingAttention(
14
+ (q_proj): Linear(in_features=32, out_features=768, bias=False)
15
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
16
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
17
+ (r_proj): Linear(in_features=32, out_features=384, bias=False)
18
+ (o_proj): Linear(in_features=768, out_features=32, bias=False)
19
+ (k_sconv): Inkling2ShortConvolution(
20
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
21
+ )
22
+ (v_sconv): Inkling2ShortConvolution(
23
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
24
+ )
25
+ (rel_logits_proj): Inkling2RelativeLogits()
26
+ (gate_proj): Linear(in_features=32, out_features=768, bias=False)
27
+ (sink_proj): Linear(in_features=32, out_features=24, bias=False)
28
+ (q_l2_clip): Inkling2ScaledL2Clip()
29
+ (k_l2_clip): Inkling2L2Clip()
30
+ (rel_l2_clip): Inkling2ScaledL2Clip()
31
+ (v_norm): Inkling2UnweightedRMSNorm()
32
+ )
33
+ (mlp): Inkling2SparseMoeBlock(
34
+ (experts): Inkling2Experts(
35
+ (act_fn): Inkling2SoftcappedSwiGLU(
36
+ (gate_softcap): Inkling2Softcap()
37
+ (up_softcap): Inkling2Softcap()
38
+ )
39
+ )
40
+ (gate): Inkling2TopKRouter(
41
+ (score_fn): Sigmoid()
42
+ )
43
+ )
44
+ (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
45
+ (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
46
+ (post_attention_convolution): Inkling2ShortConvolution(
47
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
48
+ )
49
+ (post_mlp_convolution): Inkling2ShortConvolution(
50
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
51
+ )
52
+ )
53
+ (1-4): 4 x Inkling2SlidingDecoderLayer(
54
+ (attention_input): Inkling2BlockAttentionResidual(
55
+ (norm): Inkling2UnweightedRMSNorm()
56
+ )
57
+ (mlp_input): Inkling2BlockAttentionResidual(
58
+ (norm): Inkling2UnweightedRMSNorm()
59
+ )
60
+ (self_attn): Inkling2SlidingAttention(
61
+ (q_proj): Linear(in_features=32, out_features=768, bias=False)
62
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
63
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
64
+ (r_proj): Linear(in_features=32, out_features=384, bias=False)
65
+ (o_proj): Linear(in_features=768, out_features=32, bias=False)
66
+ (k_sconv): Inkling2ShortConvolution(
67
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
68
+ )
69
+ (v_sconv): Inkling2ShortConvolution(
70
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
71
+ )
72
+ (rel_logits_proj): Inkling2RelativeLogits()
73
+ (gate_proj): Linear(in_features=32, out_features=768, bias=False)
74
+ (sink_proj): Linear(in_features=32, out_features=24, bias=False)
75
+ (q_l2_clip): Inkling2ScaledL2Clip()
76
+ (k_l2_clip): Inkling2L2Clip()
77
+ (rel_l2_clip): Inkling2ScaledL2Clip()
78
+ (v_norm): Inkling2UnweightedRMSNorm()
79
+ )
80
+ (mlp): Inkling2SparseMoeBlock(
81
+ (experts): Inkling2Experts(
82
+ (act_fn): Inkling2SoftcappedSwiGLU(
83
+ (gate_softcap): Inkling2Softcap()
84
+ (up_softcap): Inkling2Softcap()
85
+ )
86
+ )
87
+ (gate): Inkling2TopKRouter(
88
+ (score_fn): Sigmoid()
89
+ )
90
+ )
91
+ (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
92
+ (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
93
+ (post_attention_convolution): Inkling2ShortConvolution(
94
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
95
+ )
96
+ (post_mlp_convolution): Inkling2ShortConvolution(
97
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
98
+ )
99
+ )
100
+ (5): Inkling2FullDecoderLayer(
101
+ (attention_input): Inkling2BlockAttentionResidual(
102
+ (norm): Inkling2UnweightedRMSNorm()
103
+ )
104
+ (mlp_input): Inkling2BlockAttentionResidual(
105
+ (norm): Inkling2UnweightedRMSNorm()
106
+ )
107
+ (self_attn): Inkling2FullAttention(
108
+ (q_proj): Linear(in_features=32, out_features=768, bias=False)
109
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
110
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
111
+ (r_proj): Linear(in_features=32, out_features=384, bias=False)
112
+ (o_proj): Linear(in_features=768, out_features=32, bias=False)
113
+ (k_sconv): Inkling2DynamicShortConvolution(
114
+ (weight_proj): Linear(in_features=32, out_features=16, bias=False)
115
+ )
116
+ (v_sconv): Inkling2DynamicShortConvolution(
117
+ (weight_proj): Linear(in_features=32, out_features=16, bias=False)
118
+ )
119
+ (rel_logits_proj): Inkling2RelativeLogits()
120
+ (gate_proj): Linear(in_features=32, out_features=768, bias=False)
121
+ (sink_proj): Linear(in_features=32, out_features=24, bias=False)
122
+ (q_l2_clip): Inkling2ScaledL2Clip()
123
+ (k_l2_clip): Inkling2L2Clip()
124
+ (rel_l2_clip): Inkling2ScaledL2Clip()
125
+ (v_norm): Inkling2UnweightedRMSNorm()
126
+ )
127
+ (mlp): Inkling2SparseMoeBlock(
128
+ (experts): Inkling2Experts(
129
+ (act_fn): Inkling2SoftcappedSwiGLU(
130
+ (gate_softcap): Inkling2Softcap()
131
+ (up_softcap): Inkling2Softcap()
132
+ )
133
+ )
134
+ (gate): Inkling2TopKRouter(
135
+ (score_fn): Sigmoid()
136
+ )
137
+ )
138
+ (q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
139
+ (k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
140
+ (v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
141
+ (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
142
+ (post_attention_convolution): Inkling2ShortConvolution(
143
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
144
+ )
145
+ (post_mlp_convolution): Inkling2ShortConvolution(
146
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
147
+ )
148
+ )
149
+ (6-10): 5 x Inkling2SlidingDecoderLayer(
150
+ (attention_input): Inkling2BlockAttentionResidual(
151
+ (norm): Inkling2UnweightedRMSNorm()
152
+ )
153
+ (mlp_input): Inkling2BlockAttentionResidual(
154
+ (norm): Inkling2UnweightedRMSNorm()
155
+ )
156
+ (self_attn): Inkling2SlidingAttention(
157
+ (q_proj): Linear(in_features=32, out_features=768, bias=False)
158
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
159
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
160
+ (r_proj): Linear(in_features=32, out_features=384, bias=False)
161
+ (o_proj): Linear(in_features=768, out_features=32, bias=False)
162
+ (k_sconv): Inkling2ShortConvolution(
163
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
164
+ )
165
+ (v_sconv): Inkling2ShortConvolution(
166
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
167
+ )
168
+ (rel_logits_proj): Inkling2RelativeLogits()
169
+ (gate_proj): Linear(in_features=32, out_features=768, bias=False)
170
+ (sink_proj): Linear(in_features=32, out_features=24, bias=False)
171
+ (q_l2_clip): Inkling2ScaledL2Clip()
172
+ (k_l2_clip): Inkling2L2Clip()
173
+ (rel_l2_clip): Inkling2ScaledL2Clip()
174
+ (v_norm): Inkling2UnweightedRMSNorm()
175
+ )
176
+ (mlp): Inkling2SparseMoeBlock(
177
+ (experts): Inkling2Experts(
178
+ (act_fn): Inkling2SoftcappedSwiGLU(
179
+ (gate_softcap): Inkling2Softcap()
180
+ (up_softcap): Inkling2Softcap()
181
+ )
182
+ )
183
+ (gate): Inkling2TopKRouter(
184
+ (score_fn): Sigmoid()
185
+ )
186
+ )
187
+ (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
188
+ (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
189
+ (post_attention_convolution): Inkling2ShortConvolution(
190
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
191
+ )
192
+ (post_mlp_convolution): Inkling2ShortConvolution(
193
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
194
+ )
195
+ )
196
+ (11): Inkling2FullDecoderLayer(
197
+ (attention_input): Inkling2BlockAttentionResidual(
198
+ (norm): Inkling2UnweightedRMSNorm()
199
+ )
200
+ (mlp_input): Inkling2BlockAttentionResidual(
201
+ (norm): Inkling2UnweightedRMSNorm()
202
+ )
203
+ (self_attn): Inkling2FullAttention(
204
+ (q_proj): Linear(in_features=32, out_features=768, bias=False)
205
+ (k_proj): Linear(in_features=32, out_features=128, bias=False)
206
+ (v_proj): Linear(in_features=32, out_features=128, bias=False)
207
+ (r_proj): Linear(in_features=32, out_features=384, bias=False)
208
+ (o_proj): Linear(in_features=768, out_features=32, bias=False)
209
+ (k_sconv): Inkling2DynamicShortConvolution(
210
+ (weight_proj): Linear(in_features=32, out_features=16, bias=False)
211
+ )
212
+ (v_sconv): Inkling2DynamicShortConvolution(
213
+ (weight_proj): Linear(in_features=32, out_features=16, bias=False)
214
+ )
215
+ (rel_logits_proj): Inkling2RelativeLogits()
216
+ (gate_proj): Linear(in_features=32, out_features=768, bias=False)
217
+ (sink_proj): Linear(in_features=32, out_features=24, bias=False)
218
+ (q_l2_clip): Inkling2ScaledL2Clip()
219
+ (k_l2_clip): Inkling2L2Clip()
220
+ (rel_l2_clip): Inkling2ScaledL2Clip()
221
+ (v_norm): Inkling2UnweightedRMSNorm()
222
+ )
223
+ (mlp): Inkling2SparseMoeBlock(
224
+ (experts): Inkling2Experts(
225
+ (act_fn): Inkling2SoftcappedSwiGLU(
226
+ (gate_softcap): Inkling2Softcap()
227
+ (up_softcap): Inkling2Softcap()
228
+ )
229
+ )
230
+ (gate): Inkling2TopKRouter(
231
+ (score_fn): Sigmoid()
232
+ )
233
+ )
234
+ (q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
235
+ (k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
236
+ (v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
237
+ (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
238
+ (post_attention_convolution): Inkling2ShortConvolution(
239
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
240
+ )
241
+ (post_mlp_convolution): Inkling2ShortConvolution(
242
+ (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
243
+ )
244
+ )
245
+ )
246
+ (norm): Inkling2RMSNorm((32,), eps=1e-06)
247
+ (final_input): Inkling2BlockAttentionResidual(
248
+ (norm): Inkling2UnweightedRMSNorm()
249
+ )
250
+ )
251
+ (vision_tower): Inkling2VisionModel(
252
+ (embeddings): Inkling2VisionEmbeddings(
253
+ (patch_embedding): Linear(in_features=1200, out_features=32, bias=False)
254
+ (position_embedding): Embedding(1024, 32)
255
+ )
256
+ (pre_layernorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
257
+ (rotary_emb): Inkling2VisionRotaryEmbedding()
258
+ (layers): ModuleList(
259
+ (0-3): 4 x Inkling2VisionEncoderLayer(
260
+ (self_attn): Inkling2VisionAttention(
261
+ (k_proj): Linear(in_features=32, out_features=32, bias=True)
262
+ (v_proj): Linear(in_features=32, out_features=32, bias=True)
263
+ (q_proj): Linear(in_features=32, out_features=32, bias=True)
264
+ (out_proj): Linear(in_features=32, out_features=32, bias=True)
265
+ )
266
+ (mlp): Inkling2VisionMLP(
267
+ (activation_fn): GELUActivation()
268
+ (fc1): Linear(in_features=32, out_features=128, bias=True)
269
+ (fc2): Linear(in_features=128, out_features=32, bias=True)
270
+ )
271
+ (layernorm_before): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
272
+ (layernorm_after): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
273
+ )
274
+ )
275
+ (merger): Inkling2VisionMerger(
276
+ (linear_1): Linear(in_features=128, out_features=32, bias=False)
277
+ (act): GELUActivation()
278
+ (linear_2): Linear(in_features=32, out_features=32, bias=False)
279
+ )
280
+ )
281
+ )
282
+ (lm_head): Linear(in_features=32, out_features=201024, bias=False)
283
+ (logit_softcap): Inkling2Softcap()
284
+ (mtp): Module(
285
+ (layers): Module(
286
+ (0): Module(
287
+ (dflare_norm): Module()
288
+ (input_layernorm): Module()
289
+ (mlp): Module(
290
+ (down_proj): Module()
291
+ (gate_proj): Module()
292
+ (up_proj): Module()
293
+ )
294
+ (post_attention_layernorm): Module()
295
+ (self_attn): Module(
296
+ (k_norm): Module()
297
+ (k_norm_ctx): Module()
298
+ (k_proj): Module()
299
+ (k_proj_ctx): Module()
300
+ (o_proj): Module()
301
+ (q_norm): Module()
302
+ (q_proj): Module()
303
+ (v_proj): Module()
304
+ (v_proj_ctx): Module()
305
+ )
306
+ )
307
+ (1): Module(
308
+ (dflare_norm): Module()
309
+ (input_layernorm): Module()
310
+ (mlp): Module(
311
+ (down_proj): Module()
312
+ (gate_proj): Module()
313
+ (up_proj): Module()
314
+ )
315
+ (post_attention_layernorm): Module()
316
+ (self_attn): Module(
317
+ (k_norm): Module()
318
+ (k_norm_ctx): Module()
319
+ (k_proj): Module()
320
+ (k_proj_ctx): Module()
321
+ (o_proj): Module()
322
+ (q_norm): Module()
323
+ (q_proj): Module()
324
+ (v_proj): Module()
325
+ (v_proj_ctx): Module()
326
+ )
327
+ )
328
+ (2): Module(
329
+ (dflare_norm): Module()
330
+ (input_layernorm): Module()
331
+ (mlp): Module(
332
+ (down_proj): Module()
333
+ (gate_proj): Module()
334
+ (up_proj): Module()
335
+ )
336
+ (post_attention_layernorm): Module()
337
+ (self_attn): Module(
338
+ (k_norm): Module()
339
+ (k_norm_ctx): Module()
340
+ (k_proj): Module()
341
+ (k_proj_ctx): Module()
342
+ (o_proj): Module()
343
+ (q_norm): Module()
344
+ (q_proj): Module()
345
+ (v_proj): Module()
346
+ (v_proj_ctx): Module()
347
+ )
348
+ )
349
+ (3): Module(
350
+ (dflare_norm): Module()
351
+ (input_layernorm): Module()
352
+ (mlp): Module(
353
+ (down_proj): Module()
354
+ (gate_proj): Module()
355
+ (up_proj): Module()
356
+ )
357
+ (post_attention_layernorm): Module()
358
+ (self_attn): Module(
359
+ (k_norm): Module()
360
+ (k_norm_ctx): Module()
361
+ (k_proj): Module()
362
+ (k_proj_ctx): Module()
363
+ (o_proj): Module()
364
+ (q_norm): Module()
365
+ (q_proj): Module()
366
+ (v_proj): Module()
367
+ (v_proj_ctx): Module()
368
+ )
369
+ )
370
+ (4): Module(
371
+ (dflare_norm): Module()
372
+ (input_layernorm): Module()
373
+ (mlp): Module(
374
+ (down_proj): Module()
375
+ (gate_proj): Module()
376
+ (up_proj): Module()
377
+ )
378
+ (post_attention_layernorm): Module()
379
+ (self_attn): Module(
380
+ (k_norm): Module()
381
+ (k_norm_ctx): Module()
382
+ (k_proj): Module()
383
+ (k_proj_ctx): Module()
384
+ (o_proj): Module()
385
+ (q_norm): Module()
386
+ (q_proj): Module()
387
+ (v_proj): Module()
388
+ (v_proj_ctx): Module()
389
+ )
390
+ )
391
+ )
392
+ (markov_head): Module(
393
+ (dflare_norm): Module()
394
+ (hidden_out): Module()
395
+ (markov_w1): Module()
396
+ (rnn_proj): Module()
397
+ )
398
+ (norm): Module()
399
+ )
400
+ )
README.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: image-text-to-text
4
+ base_model:
5
+ - thinkingmachines/Inkling-2-Small
6
+ ---
7
+
8
+ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-Small](https://huggingface.co/thinkingmachines/Inkling-2-Small) for testing and debugging.
9
+
10
+ | Feature | Source | Tiny |
11
+ |---|---|---|
12
+ | Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 9.97M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
13
+ | Safetensors weight files (including MTP) | 393.48 GB (25 files) | 22.82 MB (2 files) |
14
+ | Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
15
+ | Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
16
+ | Routed experts / experts per token | 384 / 8 | 96 / 8 |
17
+ | Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
18
+ | Vocabulary / tied embeddings | 201,024 / false | 201,024 / true |
19
+ | Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
20
+ | Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
21
+ | Vision width / layers | 1,024 / 23 | 32 / 4 |
22
+ | MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
23
+ | MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
24
+
25
+ | File path | Size |
26
+ |------|------|
27
+ | model.safetensors | 18.8MB |
28
+ | mtp.safetensors | 4.0MB |
29
+
30
+ ### Example usage
31
+
32
+ #### Transformers (✅ verified)
33
+
34
+ Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
35
+
36
+ ```python
37
+ def run_transformers(model_id: str):
38
+ import torch
39
+ from PIL import Image
40
+ from transformers import AutoModelForImageTextToText, AutoProcessor
41
+
42
+ processor = AutoProcessor.from_pretrained(model_id)
43
+ model = AutoModelForImageTextToText.from_pretrained(
44
+ model_id, trust_remote_code=True, dtype=torch.bfloat16, device_map="cuda"
45
+ )
46
+ image = Image.new("RGB", (80, 80), (80, 120, 160))
47
+ inputs = processor(text=f"{processor.image_token}Describe this image.", images=image, return_tensors="pt").to(
48
+ model.device
49
+ )
50
+ with torch.inference_mode():
51
+ output = model.generate(**inputs, max_new_tokens=10, min_new_tokens=10, do_sample=False)
52
+ print(processor.batch_decode(output[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))
53
+
54
+
55
+ if __name__ == "__main__":
56
+ run_transformers(model_id="tiny-random/inkling-2")
57
+ ```
58
+
59
+ #### SGLang serving (✅ verified)
60
+
61
+ Use the Inkling 2 support branch of SGLang; the general release does not contain this architecture.
62
+
63
+ ```bash
64
+ SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
65
+ hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
66
+ pip install --no-deps ./inkling-2-tiny
67
+ sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
68
+ --dtype bfloat16 --attention-backend fa4 --enable-multimodal \
69
+ --context-length 2048 --mem-fraction-static 0.2 \
70
+ --disable-custom-all-reduce \
71
+ --cuda-graph-config '{"decode": {"bs": [1, 2, 4]}, "prefill": {"bs": [128, 2048]}}' \
72
+ --reasoning-parser inkling --tool-call-parser inkling2
73
+ ```
74
+
75
+ ### Creating this repo
76
+
77
+ - Code: [.meta/create_model.py](.meta/create_model.py)
78
+ - Model structure: [.meta/model.log](.meta/model.log)
79
+ - Environment: [.meta/env.json](.meta/env.json)
80
+
81
+ ### Change log
82
+
83
+ - 2026-11-13: Created tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, tied token embeddings, and BF16 MTP.
84
+
85
+ *This repo is generated by GPT-6.1-Sol*
chat_template.jinja ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "xhigh": 0.99, "max": 0.99} -%}
2
+ {%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
3
+
4
+ {%- macro emit_thinking_effort() -%}
5
+ {%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
6
+ {%- if eff is string -%}
7
+ {%- set key = eff | trim -%}
8
+ {%- if key not in effort_map -%}
9
+ {{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
10
+ {%- endif -%}
11
+ {%- set num = effort_map[key] -%}
12
+ {%- else -%}
13
+ {%- set num = eff | float -%}
14
+ {%- endif -%}
15
+ {%- if num < 0.0 or num > 0.99 -%}
16
+ {{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
17
+ {%- endif -%}
18
+ {{- "<|message_system|><|content_text|>Thinking effort level: " -}}
19
+ {%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
20
+ {{- "<|end_message|>" -}}
21
+ {%- endmacro -%}
22
+
23
+ {%- if tools -%}
24
+ {%- set tool_state = namespace(specs=[]) -%}
25
+ {%- for tool in tools -%}
26
+ {%- set fn = tool.function if tool.function is defined else tool -%}
27
+ {%- set spec = {
28
+ "description": (fn.description if fn.description is defined and fn.description else ""),
29
+ "name": fn.name,
30
+ "parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
31
+ "type": (tool.type if tool.type is defined and tool.type else "function"),
32
+ } -%}
33
+ {%- set tool_state.specs = tool_state.specs + [spec] -%}
34
+ {%- endfor -%}
35
+ {{- "<|message_system|>tool_declare<|content_xml|>" -}}
36
+ {{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
37
+ {{- "<|end_message|>" -}}
38
+ {%- endif -%}
39
+
40
+ {%- set state = namespace(effort_emitted=false) -%}
41
+ {%- for message in messages -%}
42
+ {%- if message.role not in role_token -%}
43
+ {{- raise_exception("Unknown message role: " ~ message.role) -}}
44
+ {%- endif -%}
45
+ {%- if not state.effort_emitted and message.role != "system" -%}
46
+ {{- emit_thinking_effort() -}}
47
+ {%- set state.effort_emitted = true -%}
48
+ {%- endif -%}
49
+
50
+ {%- set header = namespace(text=role_token[message.role]) -%}
51
+ {%- if message.role == "tool" -%}
52
+ {%- if message.name is defined and message.name -%}
53
+ {%- set header.text = header.text ~ message.name -%}
54
+ {%- elif message.tool_call_id is defined and message.tool_call_id -%}
55
+ {%- for prev in messages -%}
56
+ {%- if prev.role == "assistant" and prev.tool_calls -%}
57
+ {%- for tc in prev.tool_calls -%}
58
+ {%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
59
+ {%- set header.text = header.text ~ tc.function.name -%}
60
+ {%- endif -%}
61
+ {%- endfor -%}
62
+ {%- endif -%}
63
+ {%- endfor -%}
64
+ {%- endif -%}
65
+ {%- elif message.role == "assistant" and message.name is defined and message.name -%}
66
+ {%- set header.text = header.text ~ message.name -%}
67
+ {%- endif -%}
68
+ {%- set rtok = header.text -%}
69
+
70
+ {%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
71
+ {{- rtok ~ "<|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
72
+ {%- endif -%}
73
+
74
+ {%- if message.content is string and (message.content or not (message.role == "assistant" and (message.tool_calls or message.reasoning_content))) -%}
75
+ {{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
76
+ {%- elif message.content -%}
77
+ {%- for part in message.content -%}
78
+ {%- if part is string -%}
79
+ {{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
80
+ {%- elif part.type is not defined or part.type in ("text", "input_text") -%}
81
+ {%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
82
+ {{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
83
+ {%- elif part.type in ("image", "input_image", "image_url") -%}
84
+ {%- if message.role not in ("user", "tool") -%}
85
+ {{- raise_exception("Image content is only supported for user and tool messages") -}}
86
+ {%- endif -%}
87
+ {{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
88
+ {%- elif part.type in ("audio", "input_audio", "audio_url") -%}
89
+ {{- raise_exception("Inkling 2 does not accept audio input") -}}
90
+ {%- else -%}
91
+ {{- raise_exception("Unsupported content part type: " ~ part.type) -}}
92
+ {%- endif -%}
93
+ {%- endfor -%}
94
+ {%- endif -%}
95
+
96
+ {%- if message.role == "assistant" and message.tool_calls -%}
97
+ {%- for tc in message.tool_calls -%}
98
+ {%- set fn = tc.function -%}
99
+ {%- if fn.name is not defined or fn.name is not string -%}
100
+ {{- raise_exception("tool call function name must be a string") -}}
101
+ {%- endif -%}
102
+ {%- if message.name is defined and message.name and message.name != fn.name -%}
103
+ {{- raise_exception("assistant name " ~ message.name ~ " does not match its tool call " ~ fn.name) -}}
104
+ {%- endif -%}
105
+ {%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
106
+ {%- if args is string -%}
107
+ {{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
108
+ {%- endif -%}
109
+ {%- if args is not mapping -%}
110
+ {{- raise_exception("tool call arguments must be an object") -}}
111
+ {%- endif -%}
112
+ {{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
113
+ {{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
114
+ {{- "{" -}}
115
+ {%- for key, value in args | dictsort(case_sensitive=true) -%}
116
+ {{- (key | tojson) ~ ":" ~ (value | tojson(separators=(",", ":"))) -}}
117
+ {%- if not loop.last -%},{%- endif -%}
118
+ {%- endfor -%}
119
+ {{- "}}<|end_message|>" -}}
120
+ {%- endfor -%}
121
+ {%- endif -%}
122
+
123
+ {%- if message.role == "assistant" -%}
124
+ {{- "<|content_model_end_sampling|>" -}}
125
+ {%- endif -%}
126
+ {%- endfor -%}
127
+
128
+ {%- if not state.effort_emitted -%}
129
+ {{- emit_thinking_effort() -}}
130
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Inkling2ForConditionalGeneration"
4
+ ],
5
+ "model_type": "inkling2",
6
+ "eos_token_id": 200006,
7
+ "text_config": {
8
+ "model_max_length": 1048576,
9
+ "torch_dtype": "bfloat16",
10
+ "hidden_size": 32,
11
+ "num_hidden_layers": 12,
12
+ "vocab_size": 201024,
13
+ "num_attention_heads": 24,
14
+ "num_key_value_heads": 4,
15
+ "head_dim": 32,
16
+ "d_rel": 16,
17
+ "rel_extent": 1024,
18
+ "q_bias": false,
19
+ "o_bias": false,
20
+ "log_scaling_n_floor": 128000,
21
+ "log_scaling_alpha": 0.1,
22
+ "rms_norm_eps": 1e-06,
23
+ "pre_norm_pre_affine": false,
24
+ "final_norm_pre_affine": false,
25
+ "use_value_norm_no_gamma": true,
26
+ "qk_l2_clip_threshold": 22.627416997969522,
27
+ "rel_l2_clip_threshold": 8.0,
28
+ "local_layer_ids": [
29
+ 0,
30
+ 1,
31
+ 2,
32
+ 3,
33
+ 4,
34
+ 6,
35
+ 7,
36
+ 8,
37
+ 9,
38
+ 10
39
+ ],
40
+ "sconv_kernel_size": 4,
41
+ "unpadded_vocab_size": 200058,
42
+ "final_logit_softcapping": 30.0,
43
+ "swa_head_dim": 32,
44
+ "swa_num_attention_heads": 24,
45
+ "swa_num_key_value_heads": 4,
46
+ "sliding_window_size": 1024,
47
+ "n_layers_per_attn_res_block": 6,
48
+ "attn_res_normalize_values": true,
49
+ "n_routed_experts": 96,
50
+ "num_experts_per_tok": 8,
51
+ "n_shared_experts": 0,
52
+ "intermediate_size": 32,
53
+ "mlp_gate_softcap": 4.0,
54
+ "mlp_up_softcap": 25.0,
55
+ "route_scale": 8.0,
56
+ "use_gate_bias": true,
57
+ "tie_word_embeddings": true
58
+ },
59
+ "vision_config": {
60
+ "temporal_patch_size": 2,
61
+ "native_patch_size": 20,
62
+ "adapter_merge_size": 2,
63
+ "projection_hidden_dim": 32,
64
+ "projector_bias": false,
65
+ "hidden_size": 32,
66
+ "num_hidden_layers": 4,
67
+ "num_attention_heads": 4,
68
+ "mlp_ratio": 4.0,
69
+ "posemb_grid_size": 32,
70
+ "layer_norm_eps": 1e-05,
71
+ "text_hidden_size": 32
72
+ },
73
+ "mtp_config": {
74
+ "hidden_size": 32,
75
+ "num_hidden_layers": 5,
76
+ "num_attention_heads": 24,
77
+ "num_key_value_heads": 4,
78
+ "head_dim": 32,
79
+ "intermediate_size": 128,
80
+ "max_position_embeddings": 1048576,
81
+ "torch_dtype": "bfloat16",
82
+ "q_bias": false,
83
+ "o_bias": false,
84
+ "rms_norm_eps": 1e-06,
85
+ "rope_theta": 1000000.0,
86
+ "local_layer_ids": [
87
+ 0,
88
+ 1,
89
+ 3,
90
+ 4
91
+ ],
92
+ "sliding_window_size": 1024,
93
+ "block_size": 8,
94
+ "markov_rank": 8,
95
+ "markov_head_type": "rnn_hidden",
96
+ "target_layer_ids": [
97
+ 5,
98
+ 12
99
+ ]
100
+ },
101
+ "tie_word_embeddings": true,
102
+ "auto_map": {
103
+ "AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
104
+ }
105
+ }
hf_quant_config.json ADDED
@@ -0,0 +1,525 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "parameter_groups": [
4
+ {
5
+ "compute_dtype": "float32",
6
+ "output_dtype": "float32",
7
+ "required_activation_quantization": null,
8
+ "tensors": [
9
+ "model.llm.attn_res.attn_queries.0",
10
+ "model.llm.attn_res.attn_queries.1",
11
+ "model.llm.attn_res.attn_queries.10",
12
+ "model.llm.attn_res.attn_queries.2",
13
+ "model.llm.attn_res.attn_queries.3",
14
+ "model.llm.attn_res.attn_queries.4",
15
+ "model.llm.attn_res.attn_queries.5",
16
+ "model.llm.attn_res.attn_queries.6",
17
+ "model.llm.attn_res.attn_queries.7",
18
+ "model.llm.attn_res.attn_queries.8",
19
+ "model.llm.attn_res.attn_queries.9",
20
+ "model.llm.attn_res.mlp_queries.0",
21
+ "model.llm.attn_res.mlp_queries.1",
22
+ "model.llm.attn_res.mlp_queries.10",
23
+ "model.llm.attn_res.mlp_queries.11",
24
+ "model.llm.attn_res.mlp_queries.2",
25
+ "model.llm.attn_res.mlp_queries.3",
26
+ "model.llm.attn_res.mlp_queries.4",
27
+ "model.llm.attn_res.mlp_queries.5",
28
+ "model.llm.attn_res.mlp_queries.6",
29
+ "model.llm.attn_res.mlp_queries.7",
30
+ "model.llm.attn_res.mlp_queries.8",
31
+ "model.llm.attn_res.mlp_queries.9",
32
+ "model.llm.attn_res.output_query",
33
+ "model.llm.layers.0.mlp.gate.bias",
34
+ "model.llm.layers.1.mlp.gate.bias",
35
+ "model.llm.layers.10.mlp.gate.bias",
36
+ "model.llm.layers.11.mlp.gate.bias",
37
+ "model.llm.layers.2.mlp.gate.bias",
38
+ "model.llm.layers.3.mlp.gate.bias",
39
+ "model.llm.layers.4.mlp.gate.bias",
40
+ "model.llm.layers.5.mlp.gate.bias",
41
+ "model.llm.layers.6.mlp.gate.bias",
42
+ "model.llm.layers.7.mlp.gate.bias",
43
+ "model.llm.layers.8.mlp.gate.bias",
44
+ "model.llm.layers.9.mlp.gate.bias"
45
+ ],
46
+ "weight_block_shape": [],
47
+ "weight_transform": "cast"
48
+ },
49
+ {
50
+ "compute_dtype": "bfloat16",
51
+ "output_dtype": "bfloat16",
52
+ "required_activation_quantization": null,
53
+ "tensors": [
54
+ "model.llm.embed.weight",
55
+ "model.llm.embed_norm.weight",
56
+ "model.llm.layers.0.attn.attn_sink_proj.weight",
57
+ "model.llm.layers.0.attn.gate_attn.weight",
58
+ "model.llm.layers.0.attn.k_sconv.weight",
59
+ "model.llm.layers.0.attn.q_l2_clip.scale",
60
+ "model.llm.layers.0.attn.rel_l2_clip.scale",
61
+ "model.llm.layers.0.attn.rel_logits_proj.proj",
62
+ "model.llm.layers.0.attn.v_sconv.weight",
63
+ "model.llm.layers.0.attn.wk_dv.weight",
64
+ "model.llm.layers.0.attn.wo_ud.weight",
65
+ "model.llm.layers.0.attn.wq_du.weight",
66
+ "model.llm.layers.0.attn.wr_du.weight",
67
+ "model.llm.layers.0.attn.wv_dv.weight",
68
+ "model.llm.layers.0.attn_norm.weight",
69
+ "model.llm.layers.0.attn_sconv.weight",
70
+ "model.llm.layers.0.mlp.gate.weight",
71
+ "model.llm.layers.0.mlp_norm.weight",
72
+ "model.llm.layers.0.mlp_sconv.weight",
73
+ "model.llm.layers.1.attn.attn_sink_proj.weight",
74
+ "model.llm.layers.1.attn.gate_attn.weight",
75
+ "model.llm.layers.1.attn.k_sconv.weight",
76
+ "model.llm.layers.1.attn.q_l2_clip.scale",
77
+ "model.llm.layers.1.attn.rel_l2_clip.scale",
78
+ "model.llm.layers.1.attn.rel_logits_proj.proj",
79
+ "model.llm.layers.1.attn.v_sconv.weight",
80
+ "model.llm.layers.1.attn.wk_dv.weight",
81
+ "model.llm.layers.1.attn.wo_ud.weight",
82
+ "model.llm.layers.1.attn.wq_du.weight",
83
+ "model.llm.layers.1.attn.wr_du.weight",
84
+ "model.llm.layers.1.attn.wv_dv.weight",
85
+ "model.llm.layers.1.attn_norm.weight",
86
+ "model.llm.layers.1.attn_sconv.weight",
87
+ "model.llm.layers.1.mlp.gate.weight",
88
+ "model.llm.layers.1.mlp_norm.weight",
89
+ "model.llm.layers.1.mlp_sconv.weight",
90
+ "model.llm.layers.10.attn.attn_sink_proj.weight",
91
+ "model.llm.layers.10.attn.gate_attn.weight",
92
+ "model.llm.layers.10.attn.k_sconv.weight",
93
+ "model.llm.layers.10.attn.q_l2_clip.scale",
94
+ "model.llm.layers.10.attn.rel_l2_clip.scale",
95
+ "model.llm.layers.10.attn.rel_logits_proj.proj",
96
+ "model.llm.layers.10.attn.v_sconv.weight",
97
+ "model.llm.layers.10.attn.wk_dv.weight",
98
+ "model.llm.layers.10.attn.wo_ud.weight",
99
+ "model.llm.layers.10.attn.wq_du.weight",
100
+ "model.llm.layers.10.attn.wr_du.weight",
101
+ "model.llm.layers.10.attn.wv_dv.weight",
102
+ "model.llm.layers.10.attn_norm.weight",
103
+ "model.llm.layers.10.attn_sconv.weight",
104
+ "model.llm.layers.10.mlp.gate.weight",
105
+ "model.llm.layers.10.mlp_norm.weight",
106
+ "model.llm.layers.10.mlp_sconv.weight",
107
+ "model.llm.layers.11.attn.attn_sink_proj.weight",
108
+ "model.llm.layers.11.attn.gate_attn.weight",
109
+ "model.llm.layers.11.attn.k_sconv.static_w",
110
+ "model.llm.layers.11.attn.k_sconv.weight_proj",
111
+ "model.llm.layers.11.attn.q_l2_clip.scale",
112
+ "model.llm.layers.11.attn.rel_l2_clip.scale",
113
+ "model.llm.layers.11.attn.rel_logits_proj.proj",
114
+ "model.llm.layers.11.attn.v_sconv.static_w",
115
+ "model.llm.layers.11.attn.v_sconv.weight_proj",
116
+ "model.llm.layers.11.attn.wk_dv.weight",
117
+ "model.llm.layers.11.attn.wo_ud.weight",
118
+ "model.llm.layers.11.attn.wq_du.weight",
119
+ "model.llm.layers.11.attn.wr_du.weight",
120
+ "model.llm.layers.11.attn.wv_dv.weight",
121
+ "model.llm.layers.11.attn_sconv.weight",
122
+ "model.llm.layers.11.k_input_norm.weight",
123
+ "model.llm.layers.11.mlp.gate.weight",
124
+ "model.llm.layers.11.mlp_norm.weight",
125
+ "model.llm.layers.11.mlp_sconv.weight",
126
+ "model.llm.layers.11.q_input_norm.weight",
127
+ "model.llm.layers.11.v_input_norm.weight",
128
+ "model.llm.layers.2.attn.attn_sink_proj.weight",
129
+ "model.llm.layers.2.attn.gate_attn.weight",
130
+ "model.llm.layers.2.attn.k_sconv.weight",
131
+ "model.llm.layers.2.attn.q_l2_clip.scale",
132
+ "model.llm.layers.2.attn.rel_l2_clip.scale",
133
+ "model.llm.layers.2.attn.rel_logits_proj.proj",
134
+ "model.llm.layers.2.attn.v_sconv.weight",
135
+ "model.llm.layers.2.attn.wk_dv.weight",
136
+ "model.llm.layers.2.attn.wo_ud.weight",
137
+ "model.llm.layers.2.attn.wq_du.weight",
138
+ "model.llm.layers.2.attn.wr_du.weight",
139
+ "model.llm.layers.2.attn.wv_dv.weight",
140
+ "model.llm.layers.2.attn_norm.weight",
141
+ "model.llm.layers.2.attn_sconv.weight",
142
+ "model.llm.layers.2.mlp.gate.weight",
143
+ "model.llm.layers.2.mlp_norm.weight",
144
+ "model.llm.layers.2.mlp_sconv.weight",
145
+ "model.llm.layers.3.attn.attn_sink_proj.weight",
146
+ "model.llm.layers.3.attn.gate_attn.weight",
147
+ "model.llm.layers.3.attn.k_sconv.weight",
148
+ "model.llm.layers.3.attn.q_l2_clip.scale",
149
+ "model.llm.layers.3.attn.rel_l2_clip.scale",
150
+ "model.llm.layers.3.attn.rel_logits_proj.proj",
151
+ "model.llm.layers.3.attn.v_sconv.weight",
152
+ "model.llm.layers.3.attn.wk_dv.weight",
153
+ "model.llm.layers.3.attn.wo_ud.weight",
154
+ "model.llm.layers.3.attn.wq_du.weight",
155
+ "model.llm.layers.3.attn.wr_du.weight",
156
+ "model.llm.layers.3.attn.wv_dv.weight",
157
+ "model.llm.layers.3.attn_norm.weight",
158
+ "model.llm.layers.3.attn_sconv.weight",
159
+ "model.llm.layers.3.mlp.gate.weight",
160
+ "model.llm.layers.3.mlp_norm.weight",
161
+ "model.llm.layers.3.mlp_sconv.weight",
162
+ "model.llm.layers.4.attn.attn_sink_proj.weight",
163
+ "model.llm.layers.4.attn.gate_attn.weight",
164
+ "model.llm.layers.4.attn.k_sconv.weight",
165
+ "model.llm.layers.4.attn.q_l2_clip.scale",
166
+ "model.llm.layers.4.attn.rel_l2_clip.scale",
167
+ "model.llm.layers.4.attn.rel_logits_proj.proj",
168
+ "model.llm.layers.4.attn.v_sconv.weight",
169
+ "model.llm.layers.4.attn.wk_dv.weight",
170
+ "model.llm.layers.4.attn.wo_ud.weight",
171
+ "model.llm.layers.4.attn.wq_du.weight",
172
+ "model.llm.layers.4.attn.wr_du.weight",
173
+ "model.llm.layers.4.attn.wv_dv.weight",
174
+ "model.llm.layers.4.attn_norm.weight",
175
+ "model.llm.layers.4.attn_sconv.weight",
176
+ "model.llm.layers.4.mlp.gate.weight",
177
+ "model.llm.layers.4.mlp_norm.weight",
178
+ "model.llm.layers.4.mlp_sconv.weight",
179
+ "model.llm.layers.5.attn.attn_sink_proj.weight",
180
+ "model.llm.layers.5.attn.gate_attn.weight",
181
+ "model.llm.layers.5.attn.k_sconv.static_w",
182
+ "model.llm.layers.5.attn.k_sconv.weight_proj",
183
+ "model.llm.layers.5.attn.q_l2_clip.scale",
184
+ "model.llm.layers.5.attn.rel_l2_clip.scale",
185
+ "model.llm.layers.5.attn.rel_logits_proj.proj",
186
+ "model.llm.layers.5.attn.v_sconv.static_w",
187
+ "model.llm.layers.5.attn.v_sconv.weight_proj",
188
+ "model.llm.layers.5.attn.wk_dv.weight",
189
+ "model.llm.layers.5.attn.wo_ud.weight",
190
+ "model.llm.layers.5.attn.wq_du.weight",
191
+ "model.llm.layers.5.attn.wr_du.weight",
192
+ "model.llm.layers.5.attn.wv_dv.weight",
193
+ "model.llm.layers.5.attn_sconv.weight",
194
+ "model.llm.layers.5.k_input_norm.weight",
195
+ "model.llm.layers.5.mlp.gate.weight",
196
+ "model.llm.layers.5.mlp_norm.weight",
197
+ "model.llm.layers.5.mlp_sconv.weight",
198
+ "model.llm.layers.5.q_input_norm.weight",
199
+ "model.llm.layers.5.v_input_norm.weight",
200
+ "model.llm.layers.6.attn.attn_sink_proj.weight",
201
+ "model.llm.layers.6.attn.gate_attn.weight",
202
+ "model.llm.layers.6.attn.k_sconv.weight",
203
+ "model.llm.layers.6.attn.q_l2_clip.scale",
204
+ "model.llm.layers.6.attn.rel_l2_clip.scale",
205
+ "model.llm.layers.6.attn.rel_logits_proj.proj",
206
+ "model.llm.layers.6.attn.v_sconv.weight",
207
+ "model.llm.layers.6.attn.wk_dv.weight",
208
+ "model.llm.layers.6.attn.wo_ud.weight",
209
+ "model.llm.layers.6.attn.wq_du.weight",
210
+ "model.llm.layers.6.attn.wr_du.weight",
211
+ "model.llm.layers.6.attn.wv_dv.weight",
212
+ "model.llm.layers.6.attn_norm.weight",
213
+ "model.llm.layers.6.attn_sconv.weight",
214
+ "model.llm.layers.6.mlp.gate.weight",
215
+ "model.llm.layers.6.mlp_norm.weight",
216
+ "model.llm.layers.6.mlp_sconv.weight",
217
+ "model.llm.layers.7.attn.attn_sink_proj.weight",
218
+ "model.llm.layers.7.attn.gate_attn.weight",
219
+ "model.llm.layers.7.attn.k_sconv.weight",
220
+ "model.llm.layers.7.attn.q_l2_clip.scale",
221
+ "model.llm.layers.7.attn.rel_l2_clip.scale",
222
+ "model.llm.layers.7.attn.rel_logits_proj.proj",
223
+ "model.llm.layers.7.attn.v_sconv.weight",
224
+ "model.llm.layers.7.attn.wk_dv.weight",
225
+ "model.llm.layers.7.attn.wo_ud.weight",
226
+ "model.llm.layers.7.attn.wq_du.weight",
227
+ "model.llm.layers.7.attn.wr_du.weight",
228
+ "model.llm.layers.7.attn.wv_dv.weight",
229
+ "model.llm.layers.7.attn_norm.weight",
230
+ "model.llm.layers.7.attn_sconv.weight",
231
+ "model.llm.layers.7.mlp.gate.weight",
232
+ "model.llm.layers.7.mlp_norm.weight",
233
+ "model.llm.layers.7.mlp_sconv.weight",
234
+ "model.llm.layers.8.attn.attn_sink_proj.weight",
235
+ "model.llm.layers.8.attn.gate_attn.weight",
236
+ "model.llm.layers.8.attn.k_sconv.weight",
237
+ "model.llm.layers.8.attn.q_l2_clip.scale",
238
+ "model.llm.layers.8.attn.rel_l2_clip.scale",
239
+ "model.llm.layers.8.attn.rel_logits_proj.proj",
240
+ "model.llm.layers.8.attn.v_sconv.weight",
241
+ "model.llm.layers.8.attn.wk_dv.weight",
242
+ "model.llm.layers.8.attn.wo_ud.weight",
243
+ "model.llm.layers.8.attn.wq_du.weight",
244
+ "model.llm.layers.8.attn.wr_du.weight",
245
+ "model.llm.layers.8.attn.wv_dv.weight",
246
+ "model.llm.layers.8.attn_norm.weight",
247
+ "model.llm.layers.8.attn_sconv.weight",
248
+ "model.llm.layers.8.mlp.gate.weight",
249
+ "model.llm.layers.8.mlp_norm.weight",
250
+ "model.llm.layers.8.mlp_sconv.weight",
251
+ "model.llm.layers.9.attn.attn_sink_proj.weight",
252
+ "model.llm.layers.9.attn.gate_attn.weight",
253
+ "model.llm.layers.9.attn.k_sconv.weight",
254
+ "model.llm.layers.9.attn.q_l2_clip.scale",
255
+ "model.llm.layers.9.attn.rel_l2_clip.scale",
256
+ "model.llm.layers.9.attn.rel_logits_proj.proj",
257
+ "model.llm.layers.9.attn.v_sconv.weight",
258
+ "model.llm.layers.9.attn.wk_dv.weight",
259
+ "model.llm.layers.9.attn.wo_ud.weight",
260
+ "model.llm.layers.9.attn.wq_du.weight",
261
+ "model.llm.layers.9.attn.wr_du.weight",
262
+ "model.llm.layers.9.attn.wv_dv.weight",
263
+ "model.llm.layers.9.attn_norm.weight",
264
+ "model.llm.layers.9.attn_sconv.weight",
265
+ "model.llm.layers.9.mlp.gate.weight",
266
+ "model.llm.layers.9.mlp_norm.weight",
267
+ "model.llm.layers.9.mlp_sconv.weight",
268
+ "model.llm.norm.weight",
269
+ "model.visual.backbone.embeddings.class_embedding",
270
+ "model.visual.backbone.embeddings.cls_position_embedding",
271
+ "model.visual.backbone.embeddings.patch_embedding.weight",
272
+ "model.visual.backbone.embeddings.position_embedding",
273
+ "model.visual.backbone.layers.0.ln_1.bias",
274
+ "model.visual.backbone.layers.0.ln_1.weight",
275
+ "model.visual.backbone.layers.0.ln_2.bias",
276
+ "model.visual.backbone.layers.0.ln_2.weight",
277
+ "model.visual.backbone.layers.0.ls_1.gamma",
278
+ "model.visual.backbone.layers.0.ls_2.gamma",
279
+ "model.visual.backbone.layers.0.mlp.c_fc.bias",
280
+ "model.visual.backbone.layers.0.mlp.c_fc.weight",
281
+ "model.visual.backbone.layers.0.mlp.c_proj.bias",
282
+ "model.visual.backbone.layers.0.mlp.c_proj.weight",
283
+ "model.visual.backbone.layers.0.self_attn.k_proj.bias",
284
+ "model.visual.backbone.layers.0.self_attn.k_proj.weight",
285
+ "model.visual.backbone.layers.0.self_attn.out_proj.bias",
286
+ "model.visual.backbone.layers.0.self_attn.out_proj.weight",
287
+ "model.visual.backbone.layers.0.self_attn.q_proj.bias",
288
+ "model.visual.backbone.layers.0.self_attn.q_proj.weight",
289
+ "model.visual.backbone.layers.0.self_attn.v_proj.bias",
290
+ "model.visual.backbone.layers.0.self_attn.v_proj.weight",
291
+ "model.visual.backbone.layers.1.ln_1.bias",
292
+ "model.visual.backbone.layers.1.ln_1.weight",
293
+ "model.visual.backbone.layers.1.ln_2.bias",
294
+ "model.visual.backbone.layers.1.ln_2.weight",
295
+ "model.visual.backbone.layers.1.ls_1.gamma",
296
+ "model.visual.backbone.layers.1.ls_2.gamma",
297
+ "model.visual.backbone.layers.1.mlp.c_fc.bias",
298
+ "model.visual.backbone.layers.1.mlp.c_fc.weight",
299
+ "model.visual.backbone.layers.1.mlp.c_proj.bias",
300
+ "model.visual.backbone.layers.1.mlp.c_proj.weight",
301
+ "model.visual.backbone.layers.1.self_attn.k_proj.bias",
302
+ "model.visual.backbone.layers.1.self_attn.k_proj.weight",
303
+ "model.visual.backbone.layers.1.self_attn.out_proj.bias",
304
+ "model.visual.backbone.layers.1.self_attn.out_proj.weight",
305
+ "model.visual.backbone.layers.1.self_attn.q_proj.bias",
306
+ "model.visual.backbone.layers.1.self_attn.q_proj.weight",
307
+ "model.visual.backbone.layers.1.self_attn.v_proj.bias",
308
+ "model.visual.backbone.layers.1.self_attn.v_proj.weight",
309
+ "model.visual.backbone.layers.2.ln_1.bias",
310
+ "model.visual.backbone.layers.2.ln_1.weight",
311
+ "model.visual.backbone.layers.2.ln_2.bias",
312
+ "model.visual.backbone.layers.2.ln_2.weight",
313
+ "model.visual.backbone.layers.2.ls_1.gamma",
314
+ "model.visual.backbone.layers.2.ls_2.gamma",
315
+ "model.visual.backbone.layers.2.mlp.c_fc.bias",
316
+ "model.visual.backbone.layers.2.mlp.c_fc.weight",
317
+ "model.visual.backbone.layers.2.mlp.c_proj.bias",
318
+ "model.visual.backbone.layers.2.mlp.c_proj.weight",
319
+ "model.visual.backbone.layers.2.self_attn.k_proj.bias",
320
+ "model.visual.backbone.layers.2.self_attn.k_proj.weight",
321
+ "model.visual.backbone.layers.2.self_attn.out_proj.bias",
322
+ "model.visual.backbone.layers.2.self_attn.out_proj.weight",
323
+ "model.visual.backbone.layers.2.self_attn.q_proj.bias",
324
+ "model.visual.backbone.layers.2.self_attn.q_proj.weight",
325
+ "model.visual.backbone.layers.2.self_attn.v_proj.bias",
326
+ "model.visual.backbone.layers.2.self_attn.v_proj.weight",
327
+ "model.visual.backbone.layers.3.ln_1.bias",
328
+ "model.visual.backbone.layers.3.ln_1.weight",
329
+ "model.visual.backbone.layers.3.ln_2.bias",
330
+ "model.visual.backbone.layers.3.ln_2.weight",
331
+ "model.visual.backbone.layers.3.ls_1.gamma",
332
+ "model.visual.backbone.layers.3.ls_2.gamma",
333
+ "model.visual.backbone.layers.3.mlp.c_fc.bias",
334
+ "model.visual.backbone.layers.3.mlp.c_fc.weight",
335
+ "model.visual.backbone.layers.3.mlp.c_proj.bias",
336
+ "model.visual.backbone.layers.3.mlp.c_proj.weight",
337
+ "model.visual.backbone.layers.3.self_attn.k_proj.bias",
338
+ "model.visual.backbone.layers.3.self_attn.k_proj.weight",
339
+ "model.visual.backbone.layers.3.self_attn.out_proj.bias",
340
+ "model.visual.backbone.layers.3.self_attn.out_proj.weight",
341
+ "model.visual.backbone.layers.3.self_attn.q_proj.bias",
342
+ "model.visual.backbone.layers.3.self_attn.q_proj.weight",
343
+ "model.visual.backbone.layers.3.self_attn.v_proj.bias",
344
+ "model.visual.backbone.layers.3.self_attn.v_proj.weight",
345
+ "model.visual.backbone.ln_pre.bias",
346
+ "model.visual.backbone.ln_pre.weight",
347
+ "model.visual.proj_in.weight",
348
+ "model.visual.proj_out.weight"
349
+ ],
350
+ "weight_block_shape": [],
351
+ "weight_transform": "cast"
352
+ },
353
+ {
354
+ "compute_dtype": "nvfp4",
355
+ "output_dtype": "uint8",
356
+ "required_activation_quantization": {
357
+ "amax_type": "per_token",
358
+ "block_size": 16,
359
+ "format": "nvfp4_e2m1",
360
+ "rounding": "nearest",
361
+ "scale_dtype": "float8_e4m3fn",
362
+ "scale_selection_rule": "fouroversix_mae"
363
+ },
364
+ "tensors": [
365
+ "model.llm.layers.0.mlp.experts.w13_weight",
366
+ "model.llm.layers.0.mlp.experts.w2_weight",
367
+ "model.llm.layers.1.mlp.experts.w13_weight",
368
+ "model.llm.layers.1.mlp.experts.w2_weight",
369
+ "model.llm.layers.2.mlp.experts.w13_weight",
370
+ "model.llm.layers.2.mlp.experts.w2_weight",
371
+ "model.llm.layers.3.mlp.experts.w13_weight",
372
+ "model.llm.layers.3.mlp.experts.w2_weight",
373
+ "model.llm.layers.4.mlp.experts.w13_weight",
374
+ "model.llm.layers.4.mlp.experts.w2_weight",
375
+ "model.llm.layers.5.mlp.experts.w13_weight",
376
+ "model.llm.layers.5.mlp.experts.w2_weight"
377
+ ],
378
+ "weight_block_shape": [
379
+ 1,
380
+ 16
381
+ ],
382
+ "weight_scale_format": "amax_denominator",
383
+ "weight_transform": "nvfp4_nearest"
384
+ },
385
+ {
386
+ "compute_dtype": "mxfp8",
387
+ "output_dtype": "float8_e4m3fn",
388
+ "required_activation_quantization": {
389
+ "block_size": 32,
390
+ "format": "mxfp8_e4m3",
391
+ "rounding": "nearest",
392
+ "scale_dtype": "ue8m0"
393
+ },
394
+ "tensors": [
395
+ "model.llm.layers.10.mlp.experts.w13_weight",
396
+ "model.llm.layers.10.mlp.experts.w2_weight",
397
+ "model.llm.layers.11.mlp.experts.w13_weight",
398
+ "model.llm.layers.11.mlp.experts.w2_weight",
399
+ "model.llm.layers.6.mlp.experts.w13_weight",
400
+ "model.llm.layers.6.mlp.experts.w2_weight",
401
+ "model.llm.layers.7.mlp.experts.w13_weight",
402
+ "model.llm.layers.7.mlp.experts.w2_weight",
403
+ "model.llm.layers.8.mlp.experts.w13_weight",
404
+ "model.llm.layers.8.mlp.experts.w2_weight",
405
+ "model.llm.layers.9.mlp.experts.w13_weight",
406
+ "model.llm.layers.9.mlp.experts.w2_weight"
407
+ ],
408
+ "weight_block_shape": [
409
+ 32,
410
+ 32
411
+ ],
412
+ "weight_transform": "mxfp8_nearest"
413
+ },
414
+ {
415
+ "compute_dtype": "bfloat16",
416
+ "fwd_output_dtype": "float32",
417
+ "output_dtype": "bfloat16",
418
+ "required_activation_quantization": null,
419
+ "tensors": [
420
+ "model.llm.unembed.weight"
421
+ ],
422
+ "weight_block_shape": [],
423
+ "weight_transform": "cast"
424
+ }
425
+ ]
426
+ },
427
+ "mtp": {
428
+ "parameter_groups": [
429
+ {
430
+ "compute_dtype": "bfloat16",
431
+ "output_dtype": "bfloat16",
432
+ "required_activation_quantization": null,
433
+ "tensors": [
434
+ "fuse_weight",
435
+ "layers.0.dflare_norm.weight",
436
+ "layers.0.input_layernorm.weight",
437
+ "layers.0.mlp.down_proj.weight",
438
+ "layers.0.mlp.gate_proj.weight",
439
+ "layers.0.mlp.up_proj.weight",
440
+ "layers.0.post_attention_layernorm.weight",
441
+ "layers.0.self_attn.k_norm.weight",
442
+ "layers.0.self_attn.k_norm_ctx.weight",
443
+ "layers.0.self_attn.k_proj.weight",
444
+ "layers.0.self_attn.k_proj_ctx.weight",
445
+ "layers.0.self_attn.o_proj.weight",
446
+ "layers.0.self_attn.q_norm.weight",
447
+ "layers.0.self_attn.q_proj.weight",
448
+ "layers.0.self_attn.v_proj.weight",
449
+ "layers.0.self_attn.v_proj_ctx.weight",
450
+ "layers.1.dflare_norm.weight",
451
+ "layers.1.input_layernorm.weight",
452
+ "layers.1.mlp.down_proj.weight",
453
+ "layers.1.mlp.gate_proj.weight",
454
+ "layers.1.mlp.up_proj.weight",
455
+ "layers.1.post_attention_layernorm.weight",
456
+ "layers.1.self_attn.k_norm.weight",
457
+ "layers.1.self_attn.k_norm_ctx.weight",
458
+ "layers.1.self_attn.k_proj.weight",
459
+ "layers.1.self_attn.k_proj_ctx.weight",
460
+ "layers.1.self_attn.o_proj.weight",
461
+ "layers.1.self_attn.q_norm.weight",
462
+ "layers.1.self_attn.q_proj.weight",
463
+ "layers.1.self_attn.v_proj.weight",
464
+ "layers.1.self_attn.v_proj_ctx.weight",
465
+ "layers.2.dflare_norm.weight",
466
+ "layers.2.input_layernorm.weight",
467
+ "layers.2.mlp.down_proj.weight",
468
+ "layers.2.mlp.gate_proj.weight",
469
+ "layers.2.mlp.up_proj.weight",
470
+ "layers.2.post_attention_layernorm.weight",
471
+ "layers.2.self_attn.k_norm.weight",
472
+ "layers.2.self_attn.k_norm_ctx.weight",
473
+ "layers.2.self_attn.k_proj.weight",
474
+ "layers.2.self_attn.k_proj_ctx.weight",
475
+ "layers.2.self_attn.o_proj.weight",
476
+ "layers.2.self_attn.q_norm.weight",
477
+ "layers.2.self_attn.q_proj.weight",
478
+ "layers.2.self_attn.v_proj.weight",
479
+ "layers.2.self_attn.v_proj_ctx.weight",
480
+ "layers.3.dflare_norm.weight",
481
+ "layers.3.input_layernorm.weight",
482
+ "layers.3.mlp.down_proj.weight",
483
+ "layers.3.mlp.gate_proj.weight",
484
+ "layers.3.mlp.up_proj.weight",
485
+ "layers.3.post_attention_layernorm.weight",
486
+ "layers.3.self_attn.k_norm.weight",
487
+ "layers.3.self_attn.k_norm_ctx.weight",
488
+ "layers.3.self_attn.k_proj.weight",
489
+ "layers.3.self_attn.k_proj_ctx.weight",
490
+ "layers.3.self_attn.o_proj.weight",
491
+ "layers.3.self_attn.q_norm.weight",
492
+ "layers.3.self_attn.q_proj.weight",
493
+ "layers.3.self_attn.v_proj.weight",
494
+ "layers.3.self_attn.v_proj_ctx.weight",
495
+ "layers.4.dflare_norm.weight",
496
+ "layers.4.input_layernorm.weight",
497
+ "layers.4.mlp.down_proj.weight",
498
+ "layers.4.mlp.gate_proj.weight",
499
+ "layers.4.mlp.up_proj.weight",
500
+ "layers.4.post_attention_layernorm.weight",
501
+ "layers.4.self_attn.k_norm.weight",
502
+ "layers.4.self_attn.k_norm_ctx.weight",
503
+ "layers.4.self_attn.k_proj.weight",
504
+ "layers.4.self_attn.k_proj_ctx.weight",
505
+ "layers.4.self_attn.o_proj.weight",
506
+ "layers.4.self_attn.q_norm.weight",
507
+ "layers.4.self_attn.q_proj.weight",
508
+ "layers.4.self_attn.v_proj.weight",
509
+ "layers.4.self_attn.v_proj_ctx.weight",
510
+ "markov_head.dflare_norm.weight",
511
+ "markov_head.hidden_out.bias",
512
+ "markov_head.hidden_out.weight",
513
+ "markov_head.markov_w1.weight",
514
+ "markov_head.rnn_proj.bias",
515
+ "markov_head.rnn_proj.weight",
516
+ "mask_embed",
517
+ "norm.weight"
518
+ ],
519
+ "weight_block_shape": [],
520
+ "weight_transform": "cast"
521
+ }
522
+ ]
523
+ },
524
+ "quant_method": "inkling2"
525
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c24250280ea240d1957cb6b92fb597d1a6c2beca4c926ef0d10d8342d669657
3
+ size 18808680
modeling_inkling2_tiny.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
2
+
3
+ from pathlib import Path
4
+
5
+ import torch
6
+ from huggingface_hub import hf_hub_download
7
+ from safetensors.torch import load_file
8
+ from transformers import Inkling2Config, Inkling2ForConditionalGeneration
9
+ from transformers.conversion_mapping import get_model_conversion_mapping
10
+ from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
11
+
12
+
13
+ def dequantize(weight, scale, amax=None, denominator=None):
14
+ if weight.dtype == torch.uint8:
15
+ if amax is None or denominator is None:
16
+ raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
17
+ codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
18
+ levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
19
+ values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
20
+ return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
21
+ if weight.dtype == torch.float8_e4m3fn:
22
+ scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
23
+ return weight.float() * scales
24
+ raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
25
+
26
+
27
+ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
28
+ @classmethod
29
+ def from_pretrained(
30
+ cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
31
+ revision=None, cache_dir=None, local_files_only=False, token=None,
32
+ attn_implementation="eager", **kwargs,
33
+ ):
34
+ if model_args:
35
+ raise ValueError("Inkling 2 does not take positional construction arguments")
36
+ options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
37
+ root = Path(pretrained_model_name_or_path)
38
+
39
+ def resolve(filename):
40
+ if root.is_dir():
41
+ return root / filename
42
+ return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
43
+
44
+ import json
45
+
46
+ raw = json.loads(resolve("config.json").read_text())
47
+ # The native config otherwise assumes the all-MXFP8 sibling. This
48
+ # adapter dequantizes each tensor according to its actual stored dtype.
49
+ raw.pop("quantization_config", None)
50
+ config = Inkling2Config(**raw)
51
+ config._attn_implementation = attn_implementation
52
+ dtype = kwargs.pop("torch_dtype", dtype)
53
+ if dtype in (None, "auto"):
54
+ dtype = torch.bfloat16
55
+ elif isinstance(dtype, str):
56
+ dtype = getattr(torch, dtype)
57
+ if dtype not in (torch.float32, torch.bfloat16, torch.float16):
58
+ raise ValueError("The Transformers adapter requires a floating compute dtype")
59
+ device_map = kwargs.pop("device_map", None)
60
+ if device_map == "auto":
61
+ device_map = "cuda" if torch.cuda.is_available() else "cpu"
62
+ if isinstance(device_map, dict):
63
+ if set(device_map) != {""}:
64
+ raise ValueError("Use a single device for this tiny Transformers adapter")
65
+ device_map = device_map[""]
66
+ allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
67
+ unexpected = set(kwargs) - allowed
68
+ if unexpected:
69
+ raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
70
+ model = Inkling2ForConditionalGeneration(config)
71
+ transforms = get_model_conversion_mapping(model)
72
+ renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
73
+ converters = [t for t in transforms if isinstance(t, WeightConverter)]
74
+ source = load_file(str(resolve("model.safetensors")))
75
+ state = {}
76
+ for name, weight in source.items():
77
+ if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
78
+ continue
79
+ if name + "_scale" in source:
80
+ weight = dequantize(
81
+ weight, source[name + "_scale"], source.get(name + "_amax"),
82
+ source.get(name + "_outer_scale_denominator"),
83
+ )
84
+ target, _ = rename_source_key(name, renamings, converters)
85
+ if name.endswith("w13_weight"):
86
+ # The source interleaves gate and up; Transformers chunks them.
87
+ e, rows, columns = weight.shape
88
+ weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
89
+ if target in state:
90
+ raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
91
+ keep_float32 = source[name].dtype == torch.float32 or any(
92
+ part in target for part in model._keep_in_fp32_modules_strict
93
+ )
94
+ target_dtype = torch.float32 if keep_float32 else dtype
95
+ state[target] = weight.to(target_dtype).contiguous()
96
+ state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
97
+ model.load_state_dict(state, strict=True, assign=True)
98
+ model.eval()
99
+ if device_map is not None:
100
+ model.to(device_map)
101
+ model.lm_head.weight = model.model.language_model.embed_tokens.weight
102
+ return model
mtp.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:194ed2549514367c14ba8e3df0a1f1e0a7397bb9ceaf1971b647535f52531663
3
+ size 4009408
processor_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "do_convert_rgb": true,
4
+ "do_normalize": true,
5
+ "do_rescale": true,
6
+ "image_mean": [0.5, 0.5, 0.5],
7
+ "image_processor_type": "Inkling2ImageProcessor",
8
+ "image_std": [0.5, 0.5, 0.5],
9
+ "rescale_factor": 0.00392156862745098,
10
+ "size": {"height": 40, "width": 40}
11
+ },
12
+ "image_token": "<|unused_200054|>",
13
+ "processor_class": "Inkling2Processor"
14
+ }
pyproject.toml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "sglang-inkling2-tied"
7
+ version = "0.1.0"
8
+ description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
9
+ requires-python = ">=3.10"
10
+
11
+ [project.entry-points."sglang.srt.plugins"]
12
+ inkling2_tied = "sglang_inkling2_tied:register"
13
+
14
+ [tool.setuptools]
15
+ py-modules = ["sglang_inkling2_tied"]
sglang_inkling2_tied.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
2
+
3
+ import logging
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+
8
+ def _config_init(original, config, *args, **kwargs):
9
+ tied = kwargs.get("tie_word_embeddings", False)
10
+ if tied:
11
+ kwargs = {**kwargs, "tie_word_embeddings": False}
12
+ original(config, *args, **kwargs)
13
+ if tied:
14
+ config.tie_word_embeddings = True
15
+
16
+
17
+ def _load_weights(original, model, weights):
18
+ if not model.text_config.tie_word_embeddings or model.llm is None:
19
+ return original(model, weights)
20
+
21
+ def with_head():
22
+ embedding = None
23
+ for name, tensor in weights:
24
+ if name == "model.llm.unembed.weight":
25
+ raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
26
+ if name == "model.llm.embed.weight":
27
+ embedding = tensor
28
+ yield name, tensor
29
+ if embedding is None:
30
+ raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
31
+ yield "model.llm.unembed.weight", embedding
32
+
33
+ loaded = original(model, with_head())
34
+ embedding, head = model.llm.embed_tokens, model.llm.lm_head
35
+ if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
36
+ raise ValueError("Tied embedding/head tensor-parallel partitions must match")
37
+ head.weight = embedding.weight
38
+ assert head.weight is embedding.weight
39
+ logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
40
+ return loaded
41
+
42
+
43
+ class _PaddedMarlin(tuple):
44
+ def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
45
+ value = super().__new__(cls, packed)
46
+ value.input_width = input_width
47
+ value.output_width = output_width
48
+ value.padded_input = padded_input
49
+ value.padded_output = padded_output
50
+ return value
51
+
52
+
53
+ def _pad_matrix(tensor, rows, columns, fill=0):
54
+ output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
55
+ output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
56
+ return output
57
+
58
+
59
+ def _repack_marlin(original, weight, scales):
60
+ import torch
61
+
62
+ nvfp4 = weight.dtype == torch.uint8
63
+ packing = 2 if nvfp4 else 1
64
+ block = 16 if nvfp4 else 32
65
+ n, k = weight.shape[-2], weight.shape[-1] * packing
66
+ padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
67
+ if (padded_k, padded_n) == (k, n):
68
+ return original(weight, scales)
69
+ weight = _pad_matrix(weight, padded_n, padded_k // packing)
70
+ # Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
71
+ # Zero weight bytes make padded products zero in both precision families.
72
+ scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
73
+ packed = original(weight, scales)
74
+ return _PaddedMarlin(packed, k, n, padded_k, padded_n)
75
+
76
+
77
+ def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
78
+ if not isinstance(packed, _PaddedMarlin):
79
+ return original(x, packed, out_size, *args, **kwargs)
80
+ if x.shape[-1] != packed.input_width or out_size != packed.output_width:
81
+ raise ValueError("Marlin padding metadata does not match the expert projection")
82
+ padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
83
+ padded[..., :x.shape[-1]].copy_(x)
84
+ if original.__name__ == "gemv" and packed.padded_input == 128:
85
+ # Eight K tiles fit one split of eight warps. The large-model heuristic
86
+ # otherwise chooses four splits/four warps for short decode batches.
87
+ kwargs = {**kwargs, "splits": 1, "warps": 8}
88
+ result = original(padded, packed, packed.padded_output, *args, **kwargs)
89
+ return result[..., :out_size].contiguous()
90
+
91
+
92
+ def _relative_fa4(original, *args, **kwargs):
93
+ q = args[0] if args else kwargs.get("q")
94
+ if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
95
+ and kwargs.get("tile_mn") is None):
96
+ # The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
97
+ # relative-bias kernel explicitly requires M=N=128. Use its tile API.
98
+ kwargs = {**kwargs, "tile_mn": (128, 128)}
99
+ return original(*args, **kwargs)
100
+
101
+
102
+ def _attn_res_config(original, width, *args, **kwargs):
103
+ threads, stages, vector, cluster = original(width, *args, **kwargs)
104
+ if width == 32:
105
+ # One compute warp covers the original 32 residual coordinates.
106
+ # Keep the same CuTe kernel, RMS normalization, and source weighting.
107
+ return 32, stages, 1, 1
108
+ return threads, stages, vector, cluster
109
+
110
+
111
+ def _small_all_reduce(original, x, group, **kwargs):
112
+ if x.ndim == 2 and x.shape[-1] == 32:
113
+ return group.all_reduce(x.clone())
114
+ return original(x, group, **kwargs)
115
+
116
+
117
+ def _small_multicast_input(original, group, shape, dtype):
118
+ if len(shape) == 2 and shape[-1] == 32:
119
+ import torch
120
+
121
+ return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
122
+ return original(group, shape, dtype)
123
+
124
+
125
+ def _small_multicast_reduce(original, x, group):
126
+ if x.ndim == 2 and x.shape[-1] == 32:
127
+ x.copy_(group.all_reduce(x))
128
+ return x
129
+ return original(x, group)
130
+
131
+
132
+ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
133
+ if cols == 32:
134
+ # Use the existing FP32 combine path instead of the GB300-tuned
135
+ # symmetric-memory one-shot collective for these tiny projections.
136
+ return None
137
+ return original(group, rows, cols, dtype, tag=tag)
138
+
139
+
140
+ def register():
141
+ from sglang.srt.plugins.hook_registry import HookRegistry, HookType
142
+
143
+ HookRegistry.register(
144
+ "sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
145
+ )
146
+ HookRegistry.register(
147
+ "sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
148
+ )
149
+ prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
150
+ for name in ("repack", "repack_mxfp8"):
151
+ HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
152
+ for name in ("gemm", "gemv"):
153
+ HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
154
+ HookRegistry.register(
155
+ "sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
156
+ _relative_fa4, HookType.AROUND,
157
+ )
158
+ HookRegistry.register(
159
+ "sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
160
+ _attn_res_config, HookType.AROUND,
161
+ )
162
+ prefix = "sglang.kernels.ops.inkling2.comm"
163
+ for name, hook in (
164
+ ("all_reduce", _small_all_reduce),
165
+ ("multicast_all_reduce_input", _small_multicast_input),
166
+ ("multicast_all_reduce", _small_multicast_reduce),
167
+ ("oneshot_all_reduce_input", _small_oneshot_input),
168
+ ):
169
+ HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
special_tokens_map.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<|message_user|>",
5
+ "<|message_model|>",
6
+ "<|message_system|>",
7
+ "<|message_tool|>",
8
+ "<|content_text|>",
9
+ "<|content_image|>",
10
+ "<|content_model_end_sampling|>",
11
+ "<|content_thinking|>",
12
+ "<|end_message|>",
13
+ "<|content_tool_error|>",
14
+ "<|content_xml|>",
15
+ "<|begin_of_text|>",
16
+ "<|content_invoke_tool_json|>",
17
+ "<|content_invoke_tool_text|>"
18
+ ]
19
+ }
tiktoken/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f170372dd655728909228220d77cf6b8104a59f38a91262a7d2c4027906ec0d8
3
+ size 3386494
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f332727f9a3f093dd452a39cc978e6757627e5dad1b0673aa4e75174f236dd6
3
+ size 26178678
tokenizer_config.json ADDED
@@ -0,0 +1,546 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "model_max_length": 1048576,
4
+ "clean_up_tokenization_spaces": false,
5
+ "fix_mistral_regex": false,
6
+ "extra_special_tokens": {
7
+ "endoftext": "<|endoftext|>",
8
+ "message_user": "<|message_user|>",
9
+ "message_model": "<|message_model|>",
10
+ "message_system": "<|message_system|>",
11
+ "message_tool": "<|message_tool|>",
12
+ "content_text": "<|content_text|>",
13
+ "content_image": "<|content_image|>",
14
+ "content_model_end_sampling": "<|content_model_end_sampling|>",
15
+ "content_thinking": "<|content_thinking|>",
16
+ "end_message": "<|end_message|>",
17
+ "content_tool_error": "<|content_tool_error|>",
18
+ "content_xml": "<|content_xml|>",
19
+ "begin_of_text": "<|begin_of_text|>",
20
+ "content_invoke_tool_json": "<|content_invoke_tool_json|>",
21
+ "content_invoke_tool_text": "<|content_invoke_tool_text|>"
22
+ },
23
+ "added_tokens_decoder": {
24
+ "199998": {
25
+ "content": "<|unused|>",
26
+ "single_word": false,
27
+ "lstrip": false,
28
+ "rstrip": false,
29
+ "normalized": false,
30
+ "special": true
31
+ },
32
+ "199999": {
33
+ "content": "<|endoftext|>",
34
+ "single_word": false,
35
+ "lstrip": false,
36
+ "rstrip": false,
37
+ "normalized": false,
38
+ "special": true
39
+ },
40
+ "200000": {
41
+ "content": "<|message_user|>",
42
+ "single_word": false,
43
+ "lstrip": false,
44
+ "rstrip": false,
45
+ "normalized": false,
46
+ "special": true
47
+ },
48
+ "200001": {
49
+ "content": "<|message_model|>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": false,
54
+ "special": true
55
+ },
56
+ "200002": {
57
+ "content": "<|message_system|>",
58
+ "single_word": false,
59
+ "lstrip": false,
60
+ "rstrip": false,
61
+ "normalized": false,
62
+ "special": true
63
+ },
64
+ "200003": {
65
+ "content": "<|message_tool|>",
66
+ "single_word": false,
67
+ "lstrip": false,
68
+ "rstrip": false,
69
+ "normalized": false,
70
+ "special": true
71
+ },
72
+ "200004": {
73
+ "content": "<|content_text|>",
74
+ "single_word": false,
75
+ "lstrip": false,
76
+ "rstrip": false,
77
+ "normalized": false,
78
+ "special": true
79
+ },
80
+ "200005": {
81
+ "content": "<|content_image|>",
82
+ "single_word": false,
83
+ "lstrip": false,
84
+ "rstrip": false,
85
+ "normalized": false,
86
+ "special": true
87
+ },
88
+ "200006": {
89
+ "content": "<|content_model_end_sampling|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ "200007": {
97
+ "content": "<|unused_200007|>",
98
+ "single_word": false,
99
+ "lstrip": false,
100
+ "rstrip": false,
101
+ "normalized": false,
102
+ "special": true
103
+ },
104
+ "200008": {
105
+ "content": "<|content_thinking|>",
106
+ "single_word": false,
107
+ "lstrip": false,
108
+ "rstrip": false,
109
+ "normalized": false,
110
+ "special": true
111
+ },
112
+ "200009": {
113
+ "content": "<|unused_200009|>",
114
+ "single_word": false,
115
+ "lstrip": false,
116
+ "rstrip": false,
117
+ "normalized": false,
118
+ "special": true
119
+ },
120
+ "200010": {
121
+ "content": "<|end_message|>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": false,
126
+ "special": true
127
+ },
128
+ "200011": {
129
+ "content": "<|unused_200011|>",
130
+ "single_word": false,
131
+ "lstrip": false,
132
+ "rstrip": false,
133
+ "normalized": false,
134
+ "special": true
135
+ },
136
+ "200012": {
137
+ "content": "<|unused_200012|>",
138
+ "single_word": false,
139
+ "lstrip": false,
140
+ "rstrip": false,
141
+ "normalized": false,
142
+ "special": true
143
+ },
144
+ "200013": {
145
+ "content": "<|unused_200013|>",
146
+ "single_word": false,
147
+ "lstrip": false,
148
+ "rstrip": false,
149
+ "normalized": false,
150
+ "special": true
151
+ },
152
+ "200014": {
153
+ "content": "<|unused_200014|>",
154
+ "single_word": false,
155
+ "lstrip": false,
156
+ "rstrip": false,
157
+ "normalized": false,
158
+ "special": true
159
+ },
160
+ "200015": {
161
+ "content": "<|unused_200015|>",
162
+ "single_word": false,
163
+ "lstrip": false,
164
+ "rstrip": false,
165
+ "normalized": false,
166
+ "special": true
167
+ },
168
+ "200016": {
169
+ "content": "<|unused_200016|>",
170
+ "single_word": false,
171
+ "lstrip": false,
172
+ "rstrip": false,
173
+ "normalized": false,
174
+ "special": true
175
+ },
176
+ "200017": {
177
+ "content": "<|unused_200017|>",
178
+ "single_word": false,
179
+ "lstrip": false,
180
+ "rstrip": false,
181
+ "normalized": false,
182
+ "special": true
183
+ },
184
+ "200018": {
185
+ "content": "<|unused_200018|>",
186
+ "single_word": false,
187
+ "lstrip": false,
188
+ "rstrip": false,
189
+ "normalized": false,
190
+ "special": true
191
+ },
192
+ "200019": {
193
+ "content": "<|unused_200019|>",
194
+ "single_word": false,
195
+ "lstrip": false,
196
+ "rstrip": false,
197
+ "normalized": false,
198
+ "special": true
199
+ },
200
+ "200020": {
201
+ "content": "<|unused_200020|>",
202
+ "single_word": false,
203
+ "lstrip": false,
204
+ "rstrip": false,
205
+ "normalized": false,
206
+ "special": true
207
+ },
208
+ "200021": {
209
+ "content": "<|unused_200021|>",
210
+ "single_word": false,
211
+ "lstrip": false,
212
+ "rstrip": false,
213
+ "normalized": false,
214
+ "special": true
215
+ },
216
+ "200022": {
217
+ "content": "<|content_tool_error|>",
218
+ "single_word": false,
219
+ "lstrip": false,
220
+ "rstrip": false,
221
+ "normalized": false,
222
+ "special": true
223
+ },
224
+ "200023": {
225
+ "content": "<|unused_200023|>",
226
+ "single_word": false,
227
+ "lstrip": false,
228
+ "rstrip": false,
229
+ "normalized": false,
230
+ "special": true
231
+ },
232
+ "200024": {
233
+ "content": "<|content_xml|>",
234
+ "single_word": false,
235
+ "lstrip": false,
236
+ "rstrip": false,
237
+ "normalized": false,
238
+ "special": true
239
+ },
240
+ "200025": {
241
+ "content": "<|unused_200025|>",
242
+ "single_word": false,
243
+ "lstrip": false,
244
+ "rstrip": false,
245
+ "normalized": false,
246
+ "special": true
247
+ },
248
+ "200026": {
249
+ "content": "<|unused_200026|>",
250
+ "single_word": false,
251
+ "lstrip": false,
252
+ "rstrip": false,
253
+ "normalized": false,
254
+ "special": true
255
+ },
256
+ "200027": {
257
+ "content": "<|unused_200027|>",
258
+ "single_word": false,
259
+ "lstrip": false,
260
+ "rstrip": false,
261
+ "normalized": false,
262
+ "special": true
263
+ },
264
+ "200028": {
265
+ "content": "<|begin_of_text|>",
266
+ "single_word": false,
267
+ "lstrip": false,
268
+ "rstrip": false,
269
+ "normalized": false,
270
+ "special": true
271
+ },
272
+ "200029": {
273
+ "content": "<|unused_200029|>",
274
+ "single_word": false,
275
+ "lstrip": false,
276
+ "rstrip": false,
277
+ "normalized": false,
278
+ "special": true
279
+ },
280
+ "200030": {
281
+ "content": "<|unused_200030|>",
282
+ "single_word": false,
283
+ "lstrip": false,
284
+ "rstrip": false,
285
+ "normalized": false,
286
+ "special": true
287
+ },
288
+ "200031": {
289
+ "content": "<|unused_200031|>",
290
+ "single_word": false,
291
+ "lstrip": false,
292
+ "rstrip": false,
293
+ "normalized": false,
294
+ "special": true
295
+ },
296
+ "200032": {
297
+ "content": "<|unused_200032|>",
298
+ "single_word": false,
299
+ "lstrip": false,
300
+ "rstrip": false,
301
+ "normalized": false,
302
+ "special": true
303
+ },
304
+ "200033": {
305
+ "content": "<|unused_200033|>",
306
+ "single_word": false,
307
+ "lstrip": false,
308
+ "rstrip": false,
309
+ "normalized": false,
310
+ "special": true
311
+ },
312
+ "200034": {
313
+ "content": "<|unused_200034|>",
314
+ "single_word": false,
315
+ "lstrip": false,
316
+ "rstrip": false,
317
+ "normalized": false,
318
+ "special": true
319
+ },
320
+ "200035": {
321
+ "content": "<|unused_200035|>",
322
+ "single_word": false,
323
+ "lstrip": false,
324
+ "rstrip": false,
325
+ "normalized": false,
326
+ "special": true
327
+ },
328
+ "200036": {
329
+ "content": "<|unused_200036|>",
330
+ "single_word": false,
331
+ "lstrip": false,
332
+ "rstrip": false,
333
+ "normalized": false,
334
+ "special": true
335
+ },
336
+ "200037": {
337
+ "content": "<|unused_200037|>",
338
+ "single_word": false,
339
+ "lstrip": false,
340
+ "rstrip": false,
341
+ "normalized": false,
342
+ "special": true
343
+ },
344
+ "200038": {
345
+ "content": "<|unused_200038|>",
346
+ "single_word": false,
347
+ "lstrip": false,
348
+ "rstrip": false,
349
+ "normalized": false,
350
+ "special": true
351
+ },
352
+ "200039": {
353
+ "content": "<|unused_200039|>",
354
+ "single_word": false,
355
+ "lstrip": false,
356
+ "rstrip": false,
357
+ "normalized": false,
358
+ "special": true
359
+ },
360
+ "200040": {
361
+ "content": "<|unused_200040|>",
362
+ "single_word": false,
363
+ "lstrip": false,
364
+ "rstrip": false,
365
+ "normalized": false,
366
+ "special": true
367
+ },
368
+ "200041": {
369
+ "content": "<|unused_200041|>",
370
+ "single_word": false,
371
+ "lstrip": false,
372
+ "rstrip": false,
373
+ "normalized": false,
374
+ "special": true
375
+ },
376
+ "200042": {
377
+ "content": "<|unused_200042|>",
378
+ "single_word": false,
379
+ "lstrip": false,
380
+ "rstrip": false,
381
+ "normalized": false,
382
+ "special": true
383
+ },
384
+ "200043": {
385
+ "content": "<|unused_200043|>",
386
+ "single_word": false,
387
+ "lstrip": false,
388
+ "rstrip": false,
389
+ "normalized": false,
390
+ "special": true
391
+ },
392
+ "200044": {
393
+ "content": "<|unused_200044|>",
394
+ "single_word": false,
395
+ "lstrip": false,
396
+ "rstrip": false,
397
+ "normalized": false,
398
+ "special": true
399
+ },
400
+ "200045": {
401
+ "content": "<|unused_200045|>",
402
+ "single_word": false,
403
+ "lstrip": false,
404
+ "rstrip": false,
405
+ "normalized": false,
406
+ "special": true
407
+ },
408
+ "200046": {
409
+ "content": "<|unused_200046|>",
410
+ "single_word": false,
411
+ "lstrip": false,
412
+ "rstrip": false,
413
+ "normalized": false,
414
+ "special": true
415
+ },
416
+ "200047": {
417
+ "content": "<|unused_200047|>",
418
+ "single_word": false,
419
+ "lstrip": false,
420
+ "rstrip": false,
421
+ "normalized": false,
422
+ "special": true
423
+ },
424
+ "200048": {
425
+ "content": "<|unused_200048|>",
426
+ "single_word": false,
427
+ "lstrip": false,
428
+ "rstrip": false,
429
+ "normalized": false,
430
+ "special": true
431
+ },
432
+ "200049": {
433
+ "content": "<|content_invoke_tool_json|>",
434
+ "single_word": false,
435
+ "lstrip": false,
436
+ "rstrip": false,
437
+ "normalized": false,
438
+ "special": true
439
+ },
440
+ "200050": {
441
+ "content": "<|unused_200050|>",
442
+ "single_word": false,
443
+ "lstrip": false,
444
+ "rstrip": false,
445
+ "normalized": false,
446
+ "special": true
447
+ },
448
+ "200051": {
449
+ "content": "<|unused_200051|>",
450
+ "single_word": false,
451
+ "lstrip": false,
452
+ "rstrip": false,
453
+ "normalized": false,
454
+ "special": true
455
+ },
456
+ "200052": {
457
+ "content": "<|unused_200052|>",
458
+ "single_word": false,
459
+ "lstrip": false,
460
+ "rstrip": false,
461
+ "normalized": false,
462
+ "special": true
463
+ },
464
+ "200053": {
465
+ "content": "<|unused_200053|>",
466
+ "single_word": false,
467
+ "lstrip": false,
468
+ "rstrip": false,
469
+ "normalized": false,
470
+ "special": true
471
+ },
472
+ "200054": {
473
+ "content": "<|unused_200054|>",
474
+ "single_word": false,
475
+ "lstrip": false,
476
+ "rstrip": false,
477
+ "normalized": false,
478
+ "special": true
479
+ },
480
+ "200055": {
481
+ "content": "<|unused_200055|>",
482
+ "single_word": false,
483
+ "lstrip": false,
484
+ "rstrip": false,
485
+ "normalized": false,
486
+ "special": true
487
+ },
488
+ "200056": {
489
+ "content": "<|unused_200056|>",
490
+ "single_word": false,
491
+ "lstrip": false,
492
+ "rstrip": false,
493
+ "normalized": false,
494
+ "special": true
495
+ },
496
+ "200057": {
497
+ "content": "<|content_invoke_tool_text|>",
498
+ "single_word": false,
499
+ "lstrip": false,
500
+ "rstrip": false,
501
+ "normalized": false,
502
+ "special": true
503
+ }
504
+ },
505
+ "response_template": {
506
+ "defaults": {
507
+ "role": "assistant"
508
+ },
509
+ "start_anchor_pattern": "<\\|end_message\\|>|<\\|content_model_end_sampling\\|>",
510
+ "fields": {
511
+ "reasoning_content": {
512
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_thinking\\|>",
513
+ "close": "<|end_message|>",
514
+ "repeats": true,
515
+ "join": "",
516
+ "content": "text",
517
+ "content_args": {
518
+ "strip": false
519
+ }
520
+ },
521
+ "content": {
522
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_(?:text|tool_error|invoke_tool_text)\\|>",
523
+ "close": "<|end_message|>",
524
+ "repeats": true,
525
+ "join": "",
526
+ "content": "text",
527
+ "content_args": {
528
+ "strip": false
529
+ }
530
+ },
531
+ "tool_calls": {
532
+ "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_invoke_tool_json\\|>",
533
+ "close": "<|end_message|>",
534
+ "repeats": true,
535
+ "content": "json",
536
+ "transform": {
537
+ "type": "function",
538
+ "function": {
539
+ "name": "{content.name}",
540
+ "arguments": "{content.args}"
541
+ }
542
+ }
543
+ }
544
+ }
545
+ }
546
+ }