yujiepan commited on
Commit
6c0013a
·
verified ·
1 Parent(s): ed0d0c5

Preserve native mixed NVFP4/MXFP8 Inkling 2 without model-specific plugins

Browse files
.meta/create_model.py CHANGED
@@ -14,27 +14,32 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
14
  precision = json.loads(
15
  Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
16
  )
17
- # Two full 5-SWA/1-global cycles, sampling both source expert precision regions.
18
  source_layers = [*range(6), *range(42, 48)]
19
- hidden_size = 32
20
- intermediate_size = 32
21
  text = raw["text_config"]
22
  text.update(
23
  hidden_size=hidden_size,
24
  intermediate_size=intermediate_size,
25
  num_hidden_layers=12,
26
- num_attention_heads=24,
27
  num_key_value_heads=4,
28
- head_dim=32,
29
- swa_num_attention_heads=24,
30
  swa_num_key_value_heads=4,
31
- swa_head_dim=32,
32
  n_routed_experts=96,
33
  tie_word_embeddings=False,
34
  local_layer_ids=[i for i in range(12) if i % 6 != 5],
35
  )
36
  raw["tie_word_embeddings"] = False
37
- raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
 
 
 
 
 
38
  raw["vision_config"].update(
39
  hidden_size=32,
40
  num_hidden_layers=4,
@@ -46,9 +51,9 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
46
  mtp.update(
47
  hidden_size=hidden_size,
48
  intermediate_size=128,
49
- num_attention_heads=24,
50
  num_key_value_heads=4,
51
- head_dim=32,
52
  markov_rank=8,
53
  target_layer_ids=[5, 12],
54
  )
@@ -185,7 +190,7 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
185
  draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
186
  draft = {}
187
  draft_module = torch.nn.Module()
188
- replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 24 * 32, 1024: 4 * 32, 128: 32}
189
  for name, tensor in sorted(draft_metadata.tensors.items()):
190
  if name == "fuse_weight":
191
  shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
@@ -250,255 +255,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
250
  path = folder / name
251
  path.parent.mkdir(parents=True, exist_ok=True)
252
  path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
253
- plugin_code = '''"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
254
-
255
-
256
-
257
-
258
-
259
-
260
-
261
- class _PaddedMarlin(tuple):
262
- def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
263
- value = super().__new__(cls, packed)
264
- value.input_width = input_width
265
- value.output_width = output_width
266
- value.padded_input = padded_input
267
- value.padded_output = padded_output
268
- return value
269
-
270
-
271
- def _pad_matrix(tensor, rows, columns, fill=0):
272
- output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
273
- output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
274
- return output
275
-
276
-
277
- def _repack_marlin(original, weight, scales):
278
- import torch
279
-
280
- nvfp4 = weight.dtype == torch.uint8
281
- packing = 2 if nvfp4 else 1
282
- block = 16 if nvfp4 else 32
283
- n, k = weight.shape[-2], weight.shape[-1] * packing
284
- padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
285
- if (padded_k, padded_n) == (k, n):
286
- return original(weight, scales)
287
- weight = _pad_matrix(weight, padded_n, padded_k // packing)
288
- # Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
289
- # Zero weight bytes make padded products zero in both precision families.
290
- scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
291
- packed = original(weight, scales)
292
- return _PaddedMarlin(packed, k, n, padded_k, padded_n)
293
-
294
-
295
- def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
296
- if not isinstance(packed, _PaddedMarlin):
297
- return original(x, packed, out_size, *args, **kwargs)
298
- if x.shape[-1] != packed.input_width or out_size != packed.output_width:
299
- raise ValueError("Marlin padding metadata does not match the expert projection")
300
- padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
301
- padded[..., :x.shape[-1]].copy_(x)
302
- if original.__name__ == "gemv" and packed.padded_input == 128:
303
- # Eight K tiles fit one split of eight warps. The large-model heuristic
304
- # otherwise chooses four splits/four warps for short decode batches.
305
- kwargs = {**kwargs, "splits": 1, "warps": 8}
306
- result = original(padded, packed, packed.padded_output, *args, **kwargs)
307
- return result[..., :out_size].contiguous()
308
-
309
-
310
- def _relative_fa4(original, *args, **kwargs):
311
- q = args[0] if args else kwargs.get("q")
312
- if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
313
- and kwargs.get("tile_mn") is None):
314
- # The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
315
- # relative-bias kernel explicitly requires M=N=128. Use its tile API.
316
- kwargs = {**kwargs, "tile_mn": (128, 128)}
317
- return original(*args, **kwargs)
318
-
319
-
320
- def _attn_res_config(original, width, *args, **kwargs):
321
- threads, stages, vector, cluster = original(width, *args, **kwargs)
322
- if width == 32:
323
- # One compute warp covers the original 32 residual coordinates.
324
- # Keep the same CuTe kernel, RMS normalization, and source weighting.
325
- return 32, stages, 1, 1
326
- return threads, stages, vector, cluster
327
-
328
-
329
- def _small_all_reduce(original, x, group, **kwargs):
330
- if x.ndim == 2 and x.shape[-1] == 32:
331
- return group.all_reduce(x.clone())
332
- return original(x, group, **kwargs)
333
-
334
-
335
- def _small_multicast_input(original, group, shape, dtype):
336
- if len(shape) == 2 and shape[-1] == 32:
337
- import torch
338
-
339
- return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
340
- return original(group, shape, dtype)
341
-
342
-
343
- def _small_multicast_reduce(original, x, group):
344
- if x.ndim == 2 and x.shape[-1] == 32:
345
- x.copy_(group.all_reduce(x))
346
- return x
347
- return original(x, group)
348
-
349
-
350
- def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
351
- if cols == 32:
352
- # Use the existing FP32 combine path instead of the GB300-tuned
353
- # symmetric-memory one-shot collective for these tiny projections.
354
- return None
355
- return original(group, rows, cols, dtype, tag=tag)
356
-
357
-
358
- def register():
359
- from sglang.srt.plugins.hook_registry import HookRegistry, HookType
360
-
361
- prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
362
- for name in ("repack", "repack_mxfp8"):
363
- HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
364
- for name in ("gemm", "gemv"):
365
- HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
366
- HookRegistry.register(
367
- "sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
368
- _relative_fa4, HookType.AROUND,
369
- )
370
- HookRegistry.register(
371
- "sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
372
- _attn_res_config, HookType.AROUND,
373
- )
374
- prefix = "sglang.kernels.ops.inkling2.comm"
375
- for name, hook in (
376
- ("all_reduce", _small_all_reduce),
377
- ("multicast_all_reduce_input", _small_multicast_input),
378
- ("multicast_all_reduce", _small_multicast_reduce),
379
- ("oneshot_all_reduce_input", _small_oneshot_input),
380
- ):
381
- HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
382
- '''
383
- (folder / "sglang_inkling2_tiny_kernels.py").write_text(plugin_code)
384
- hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
385
-
386
- from pathlib import Path
387
-
388
- import torch
389
- from huggingface_hub import hf_hub_download
390
- from safetensors.torch import load_file
391
- from transformers import Inkling2Config, Inkling2ForConditionalGeneration
392
- from transformers.conversion_mapping import get_model_conversion_mapping
393
- from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
394
-
395
-
396
- def dequantize(weight, scale, amax=None, denominator=None):
397
- if weight.dtype == torch.uint8:
398
- if amax is None or denominator is None:
399
- raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
400
- codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
401
- levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
402
- values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
403
- return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
404
- if weight.dtype == torch.float8_e4m3fn:
405
- scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
406
- return weight.float() * scales
407
- raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
408
-
409
-
410
- class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
411
- @classmethod
412
- def from_pretrained(
413
- cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
414
- revision=None, cache_dir=None, local_files_only=False, token=None,
415
- attn_implementation="eager", **kwargs,
416
- ):
417
- if model_args:
418
- raise ValueError("Inkling 2 does not take positional construction arguments")
419
- options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
420
- root = Path(pretrained_model_name_or_path)
421
-
422
- def resolve(filename):
423
- if root.is_dir():
424
- return root / filename
425
- return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
426
-
427
- import json
428
-
429
- raw = json.loads(resolve("config.json").read_text())
430
- # The native config otherwise assumes the all-MXFP8 sibling. This
431
- # adapter dequantizes each tensor according to its actual stored dtype.
432
- raw.pop("quantization_config", None)
433
- config = Inkling2Config(**raw)
434
- config._attn_implementation = attn_implementation
435
- dtype = kwargs.pop("torch_dtype", dtype)
436
- if dtype in (None, "auto"):
437
- dtype = torch.bfloat16
438
- elif isinstance(dtype, str):
439
- dtype = getattr(torch, dtype)
440
- if dtype not in (torch.float32, torch.bfloat16, torch.float16):
441
- raise ValueError("The Transformers adapter requires a floating compute dtype")
442
- device_map = kwargs.pop("device_map", None)
443
- if device_map == "auto":
444
- device_map = "cuda" if torch.cuda.is_available() else "cpu"
445
- if isinstance(device_map, dict):
446
- if set(device_map) != {""}:
447
- raise ValueError("Use a single device for this tiny Transformers adapter")
448
- device_map = device_map[""]
449
- allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
450
- unexpected = set(kwargs) - allowed
451
- if unexpected:
452
- raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
453
- model = Inkling2ForConditionalGeneration(config)
454
- transforms = get_model_conversion_mapping(model)
455
- renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
456
- converters = [t for t in transforms if isinstance(t, WeightConverter)]
457
- source = load_file(str(resolve("model.safetensors")))
458
- state = {}
459
- for name, weight in source.items():
460
- if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
461
- continue
462
- if name + "_scale" in source:
463
- weight = dequantize(
464
- weight, source[name + "_scale"], source.get(name + "_amax"),
465
- source.get(name + "_outer_scale_denominator"),
466
- )
467
- target, _ = rename_source_key(name, renamings, converters)
468
- if name.endswith("w13_weight"):
469
- # The source interleaves gate and up; Transformers chunks them.
470
- e, rows, columns = weight.shape
471
- weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
472
- if target in state:
473
- raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
474
- keep_float32 = source[name].dtype == torch.float32 or any(
475
- part in target for part in model._keep_in_fp32_modules_strict
476
- )
477
- target_dtype = torch.float32 if keep_float32 else dtype
478
- state[target] = weight.to(target_dtype).contiguous()
479
- model.load_state_dict(state, strict=True, assign=True)
480
- model.eval()
481
- if device_map is not None:
482
- model.to(device_map)
483
- return model
484
- '''
485
- (folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
486
- (folder / "pyproject.toml").write_text("""[build-system]
487
- requires = ["setuptools>=68"]
488
- build-backend = "setuptools.build_meta"
489
-
490
- [project]
491
- name = "sglang-inkling2-tiny-kernels"
492
- version = "0.2.0"
493
- description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
494
- requires-python = ">=3.10"
495
-
496
- [project.entry-points."sglang.srt.plugins"]
497
- inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
498
-
499
- [tool.setuptools]
500
- py-modules = ["sglang_inkling2_tiny_kernels"]
501
- """)
502
  return str(model)
503
 
504
 
 
14
  precision = json.loads(
15
  Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
16
  )
17
+ # Two full 5-SWA/1-global cycles, sampling early and late source layers.
18
  source_layers = [*range(6), *range(42, 48)]
19
+ hidden_size = 128
20
+ intermediate_size = 128
21
  text = raw["text_config"]
22
  text.update(
23
  hidden_size=hidden_size,
24
  intermediate_size=intermediate_size,
25
  num_hidden_layers=12,
26
+ num_attention_heads=8,
27
  num_key_value_heads=4,
28
+ head_dim=128,
29
+ swa_num_attention_heads=8,
30
  swa_num_key_value_heads=4,
31
+ swa_head_dim=128,
32
  n_routed_experts=96,
33
  tie_word_embeddings=False,
34
  local_layer_ids=[i for i in range(12) if i % 6 != 5],
35
  )
36
  raw["tie_word_embeddings"] = False
37
+ import transformers
38
+
39
+ raw.pop("auto_map", None)
40
+ raw.pop("quantization_config", None)
41
+ raw["transformers_version"] = transformers.__version__
42
+ raw["dtype"] = "bfloat16"
43
  raw["vision_config"].update(
44
  hidden_size=32,
45
  num_hidden_layers=4,
 
51
  mtp.update(
52
  hidden_size=hidden_size,
53
  intermediate_size=128,
54
+ num_attention_heads=8,
55
  num_key_value_heads=4,
56
+ head_dim=128,
57
  markov_rank=8,
58
  target_layer_ids=[5, 12],
59
  )
 
190
  draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
191
  draft = {}
192
  draft_module = torch.nn.Module()
193
+ replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128}
194
  for name, tensor in sorted(draft_metadata.tensors.items()):
195
  if name == "fuse_weight":
196
  shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
 
255
  path = folder / name
256
  path.parent.mkdir(parents=True, exist_ok=True)
257
  path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
  return str(model)
259
 
260
 
.meta/env.json CHANGED
@@ -15,16 +15,27 @@
15
  },
16
  "cuda": "12.6"
17
  },
18
- "transformers-reference": {
19
  "python": "3.12.13",
20
  "packages": {
21
- "huggingface_hub": "2.3.0",
 
 
 
22
  "safetensors": "0.8.0",
23
- "tokenizers": "0.23.3",
24
- "torch": "2.13.0+cu126",
25
- "transformers": "5.23.0.dev0"
 
 
 
 
 
26
  },
27
- "cuda": "12.6"
 
 
 
28
  }
29
  }
30
  }
 
15
  },
16
  "cuda": "12.6"
17
  },
18
+ "sglang": {
19
  "python": "3.12.13",
20
  "packages": {
21
+ "Pillow": "12.3.0",
22
+ "flashinfer-python": "0.7.4",
23
+ "huggingface_hub": "1.33.0",
24
+ "nvidia-cutlass-dsl": "4.8.0",
25
  "safetensors": "0.8.0",
26
+ "sglang": "0.0.0.dev0",
27
+ "sglang-kernel": "0.4.10",
28
+ "tiktoken": "0.14.0",
29
+ "tml-renderers": "0.4.1",
30
+ "tokenizers": "0.23.2",
31
+ "torch": "2.14.1+cu130",
32
+ "transformers": "5.19.0",
33
+ "triton": "3.8.0"
34
  },
35
+ "cuda": "13.0",
36
+ "hardware": {
37
+ "gpu_model": "NVIDIA H20"
38
+ }
39
  }
40
  }
41
  }
.meta/model.log CHANGED
@@ -2,8 +2,8 @@ Inkling2ForConditionalGeneration(
2
  (model): Inkling2Model(
3
  (language_model): Inkling2TextModel(
4
  (embed_tokens): Inkling2NormedEmbedding(
5
- 201024, 32
6
- (embed_norm): Inkling2RMSNorm((32,), eps=1e-06)
7
  )
8
  (layers): ModuleList(
9
  (0): Inkling2SlidingDecoderLayer(
@@ -11,20 +11,20 @@ Inkling2ForConditionalGeneration(
11
  (norm): Inkling2UnweightedRMSNorm()
12
  )
13
  (self_attn): Inkling2SlidingAttention(
14
- (q_proj): Linear(in_features=32, out_features=768, bias=False)
15
- (k_proj): Linear(in_features=32, out_features=128, bias=False)
16
- (v_proj): Linear(in_features=32, out_features=128, bias=False)
17
- (r_proj): Linear(in_features=32, out_features=384, bias=False)
18
- (o_proj): Linear(in_features=768, out_features=32, bias=False)
19
  (k_sconv): Inkling2ShortConvolution(
20
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
21
  )
22
  (v_sconv): Inkling2ShortConvolution(
23
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
24
  )
25
  (rel_logits_proj): Inkling2RelativeLogits()
26
- (gate_proj): Linear(in_features=32, out_features=768, bias=False)
27
- (sink_proj): Linear(in_features=32, out_features=24, bias=False)
28
  (q_l2_clip): Inkling2ScaledL2Clip()
29
  (k_l2_clip): Inkling2L2Clip()
30
  (rel_l2_clip): Inkling2ScaledL2Clip()
@@ -41,13 +41,13 @@ Inkling2ForConditionalGeneration(
41
  (score_fn): Sigmoid()
42
  )
43
  )
44
- (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
45
- (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
46
  (post_attention_convolution): Inkling2ShortConvolution(
47
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
48
  )
49
  (post_mlp_convolution): Inkling2ShortConvolution(
50
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
51
  )
52
  )
53
  (1-4): 4 x Inkling2SlidingDecoderLayer(
@@ -58,20 +58,20 @@ Inkling2ForConditionalGeneration(
58
  (norm): Inkling2UnweightedRMSNorm()
59
  )
60
  (self_attn): Inkling2SlidingAttention(
61
- (q_proj): Linear(in_features=32, out_features=768, bias=False)
62
- (k_proj): Linear(in_features=32, out_features=128, bias=False)
63
- (v_proj): Linear(in_features=32, out_features=128, bias=False)
64
- (r_proj): Linear(in_features=32, out_features=384, bias=False)
65
- (o_proj): Linear(in_features=768, out_features=32, bias=False)
66
  (k_sconv): Inkling2ShortConvolution(
67
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
68
  )
69
  (v_sconv): Inkling2ShortConvolution(
70
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
71
  )
72
  (rel_logits_proj): Inkling2RelativeLogits()
73
- (gate_proj): Linear(in_features=32, out_features=768, bias=False)
74
- (sink_proj): Linear(in_features=32, out_features=24, bias=False)
75
  (q_l2_clip): Inkling2ScaledL2Clip()
76
  (k_l2_clip): Inkling2L2Clip()
77
  (rel_l2_clip): Inkling2ScaledL2Clip()
@@ -88,13 +88,13 @@ Inkling2ForConditionalGeneration(
88
  (score_fn): Sigmoid()
89
  )
90
  )
91
- (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
92
- (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
93
  (post_attention_convolution): Inkling2ShortConvolution(
94
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
95
  )
96
  (post_mlp_convolution): Inkling2ShortConvolution(
97
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
98
  )
99
  )
100
  (5): Inkling2FullDecoderLayer(
@@ -105,20 +105,20 @@ Inkling2ForConditionalGeneration(
105
  (norm): Inkling2UnweightedRMSNorm()
106
  )
107
  (self_attn): Inkling2FullAttention(
108
- (q_proj): Linear(in_features=32, out_features=768, bias=False)
109
- (k_proj): Linear(in_features=32, out_features=128, bias=False)
110
- (v_proj): Linear(in_features=32, out_features=128, bias=False)
111
- (r_proj): Linear(in_features=32, out_features=384, bias=False)
112
- (o_proj): Linear(in_features=768, out_features=32, bias=False)
113
  (k_sconv): Inkling2DynamicShortConvolution(
114
- (weight_proj): Linear(in_features=32, out_features=16, bias=False)
115
  )
116
  (v_sconv): Inkling2DynamicShortConvolution(
117
- (weight_proj): Linear(in_features=32, out_features=16, bias=False)
118
  )
119
  (rel_logits_proj): Inkling2RelativeLogits()
120
- (gate_proj): Linear(in_features=32, out_features=768, bias=False)
121
- (sink_proj): Linear(in_features=32, out_features=24, bias=False)
122
  (q_l2_clip): Inkling2ScaledL2Clip()
123
  (k_l2_clip): Inkling2L2Clip()
124
  (rel_l2_clip): Inkling2ScaledL2Clip()
@@ -135,15 +135,15 @@ Inkling2ForConditionalGeneration(
135
  (score_fn): Sigmoid()
136
  )
137
  )
138
- (q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
139
- (k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
140
- (v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
141
- (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
142
  (post_attention_convolution): Inkling2ShortConvolution(
143
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
144
  )
145
  (post_mlp_convolution): Inkling2ShortConvolution(
146
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
147
  )
148
  )
149
  (6-10): 5 x Inkling2SlidingDecoderLayer(
@@ -154,20 +154,20 @@ Inkling2ForConditionalGeneration(
154
  (norm): Inkling2UnweightedRMSNorm()
155
  )
156
  (self_attn): Inkling2SlidingAttention(
157
- (q_proj): Linear(in_features=32, out_features=768, bias=False)
158
- (k_proj): Linear(in_features=32, out_features=128, bias=False)
159
- (v_proj): Linear(in_features=32, out_features=128, bias=False)
160
- (r_proj): Linear(in_features=32, out_features=384, bias=False)
161
- (o_proj): Linear(in_features=768, out_features=32, bias=False)
162
  (k_sconv): Inkling2ShortConvolution(
163
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
164
  )
165
  (v_sconv): Inkling2ShortConvolution(
166
- (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
167
  )
168
  (rel_logits_proj): Inkling2RelativeLogits()
169
- (gate_proj): Linear(in_features=32, out_features=768, bias=False)
170
- (sink_proj): Linear(in_features=32, out_features=24, bias=False)
171
  (q_l2_clip): Inkling2ScaledL2Clip()
172
  (k_l2_clip): Inkling2L2Clip()
173
  (rel_l2_clip): Inkling2ScaledL2Clip()
@@ -184,13 +184,13 @@ Inkling2ForConditionalGeneration(
184
  (score_fn): Sigmoid()
185
  )
186
  )
187
- (input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
188
- (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
189
  (post_attention_convolution): Inkling2ShortConvolution(
190
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
191
  )
192
  (post_mlp_convolution): Inkling2ShortConvolution(
193
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
194
  )
195
  )
196
  (11): Inkling2FullDecoderLayer(
@@ -201,20 +201,20 @@ Inkling2ForConditionalGeneration(
201
  (norm): Inkling2UnweightedRMSNorm()
202
  )
203
  (self_attn): Inkling2FullAttention(
204
- (q_proj): Linear(in_features=32, out_features=768, bias=False)
205
- (k_proj): Linear(in_features=32, out_features=128, bias=False)
206
- (v_proj): Linear(in_features=32, out_features=128, bias=False)
207
- (r_proj): Linear(in_features=32, out_features=384, bias=False)
208
- (o_proj): Linear(in_features=768, out_features=32, bias=False)
209
  (k_sconv): Inkling2DynamicShortConvolution(
210
- (weight_proj): Linear(in_features=32, out_features=16, bias=False)
211
  )
212
  (v_sconv): Inkling2DynamicShortConvolution(
213
- (weight_proj): Linear(in_features=32, out_features=16, bias=False)
214
  )
215
  (rel_logits_proj): Inkling2RelativeLogits()
216
- (gate_proj): Linear(in_features=32, out_features=768, bias=False)
217
- (sink_proj): Linear(in_features=32, out_features=24, bias=False)
218
  (q_l2_clip): Inkling2ScaledL2Clip()
219
  (k_l2_clip): Inkling2L2Clip()
220
  (rel_l2_clip): Inkling2ScaledL2Clip()
@@ -231,19 +231,19 @@ Inkling2ForConditionalGeneration(
231
  (score_fn): Sigmoid()
232
  )
233
  )
234
- (q_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
235
- (k_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
236
- (v_input_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
237
- (post_attention_layernorm): Inkling2RMSNorm((32,), eps=1e-06)
238
  (post_attention_convolution): Inkling2ShortConvolution(
239
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
240
  )
241
  (post_mlp_convolution): Inkling2ShortConvolution(
242
- (conv1d): Conv1d(32, 32, kernel_size=(4,), stride=(1,), padding=(3,), groups=32, bias=False)
243
  )
244
  )
245
  )
246
- (norm): Inkling2RMSNorm((32,), eps=1e-06)
247
  (final_input): Inkling2BlockAttentionResidual(
248
  (norm): Inkling2UnweightedRMSNorm()
249
  )
@@ -275,11 +275,11 @@ Inkling2ForConditionalGeneration(
275
  (merger): Inkling2VisionMerger(
276
  (linear_1): Linear(in_features=128, out_features=32, bias=False)
277
  (act): GELUActivation()
278
- (linear_2): Linear(in_features=32, out_features=32, bias=False)
279
  )
280
  )
281
  )
282
- (lm_head): Linear(in_features=32, out_features=201024, bias=False)
283
  (logit_softcap): Inkling2Softcap()
284
  (mtp): Module(
285
  (layers): Module(
 
2
  (model): Inkling2Model(
3
  (language_model): Inkling2TextModel(
4
  (embed_tokens): Inkling2NormedEmbedding(
5
+ 201024, 128
6
+ (embed_norm): Inkling2RMSNorm((128,), eps=1e-06)
7
  )
8
  (layers): ModuleList(
9
  (0): Inkling2SlidingDecoderLayer(
 
11
  (norm): Inkling2UnweightedRMSNorm()
12
  )
13
  (self_attn): Inkling2SlidingAttention(
14
+ (q_proj): Linear(in_features=128, out_features=1024, bias=False)
15
+ (k_proj): Linear(in_features=128, out_features=512, bias=False)
16
+ (v_proj): Linear(in_features=128, out_features=512, bias=False)
17
+ (r_proj): Linear(in_features=128, out_features=128, bias=False)
18
+ (o_proj): Linear(in_features=1024, out_features=128, bias=False)
19
  (k_sconv): Inkling2ShortConvolution(
20
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
21
  )
22
  (v_sconv): Inkling2ShortConvolution(
23
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
24
  )
25
  (rel_logits_proj): Inkling2RelativeLogits()
26
+ (gate_proj): Linear(in_features=128, out_features=1024, bias=False)
27
+ (sink_proj): Linear(in_features=128, out_features=8, bias=False)
28
  (q_l2_clip): Inkling2ScaledL2Clip()
29
  (k_l2_clip): Inkling2L2Clip()
30
  (rel_l2_clip): Inkling2ScaledL2Clip()
 
41
  (score_fn): Sigmoid()
42
  )
43
  )
44
+ (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
45
+ (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
46
  (post_attention_convolution): Inkling2ShortConvolution(
47
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
48
  )
49
  (post_mlp_convolution): Inkling2ShortConvolution(
50
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
51
  )
52
  )
53
  (1-4): 4 x Inkling2SlidingDecoderLayer(
 
58
  (norm): Inkling2UnweightedRMSNorm()
59
  )
60
  (self_attn): Inkling2SlidingAttention(
61
+ (q_proj): Linear(in_features=128, out_features=1024, bias=False)
62
+ (k_proj): Linear(in_features=128, out_features=512, bias=False)
63
+ (v_proj): Linear(in_features=128, out_features=512, bias=False)
64
+ (r_proj): Linear(in_features=128, out_features=128, bias=False)
65
+ (o_proj): Linear(in_features=1024, out_features=128, bias=False)
66
  (k_sconv): Inkling2ShortConvolution(
67
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
68
  )
69
  (v_sconv): Inkling2ShortConvolution(
70
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
71
  )
72
  (rel_logits_proj): Inkling2RelativeLogits()
73
+ (gate_proj): Linear(in_features=128, out_features=1024, bias=False)
74
+ (sink_proj): Linear(in_features=128, out_features=8, bias=False)
75
  (q_l2_clip): Inkling2ScaledL2Clip()
76
  (k_l2_clip): Inkling2L2Clip()
77
  (rel_l2_clip): Inkling2ScaledL2Clip()
 
88
  (score_fn): Sigmoid()
89
  )
90
  )
91
+ (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
92
+ (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
93
  (post_attention_convolution): Inkling2ShortConvolution(
94
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
95
  )
96
  (post_mlp_convolution): Inkling2ShortConvolution(
97
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
98
  )
99
  )
100
  (5): Inkling2FullDecoderLayer(
 
105
  (norm): Inkling2UnweightedRMSNorm()
106
  )
107
  (self_attn): Inkling2FullAttention(
108
+ (q_proj): Linear(in_features=128, out_features=1024, bias=False)
109
+ (k_proj): Linear(in_features=128, out_features=512, bias=False)
110
+ (v_proj): Linear(in_features=128, out_features=512, bias=False)
111
+ (r_proj): Linear(in_features=128, out_features=128, bias=False)
112
+ (o_proj): Linear(in_features=1024, out_features=128, bias=False)
113
  (k_sconv): Inkling2DynamicShortConvolution(
114
+ (weight_proj): Linear(in_features=128, out_features=16, bias=False)
115
  )
116
  (v_sconv): Inkling2DynamicShortConvolution(
117
+ (weight_proj): Linear(in_features=128, out_features=16, bias=False)
118
  )
119
  (rel_logits_proj): Inkling2RelativeLogits()
120
+ (gate_proj): Linear(in_features=128, out_features=1024, bias=False)
121
+ (sink_proj): Linear(in_features=128, out_features=8, bias=False)
122
  (q_l2_clip): Inkling2ScaledL2Clip()
123
  (k_l2_clip): Inkling2L2Clip()
124
  (rel_l2_clip): Inkling2ScaledL2Clip()
 
135
  (score_fn): Sigmoid()
136
  )
137
  )
138
+ (q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
139
+ (k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
140
+ (v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
141
+ (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
142
  (post_attention_convolution): Inkling2ShortConvolution(
143
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
144
  )
145
  (post_mlp_convolution): Inkling2ShortConvolution(
146
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
147
  )
148
  )
149
  (6-10): 5 x Inkling2SlidingDecoderLayer(
 
154
  (norm): Inkling2UnweightedRMSNorm()
155
  )
156
  (self_attn): Inkling2SlidingAttention(
157
+ (q_proj): Linear(in_features=128, out_features=1024, bias=False)
158
+ (k_proj): Linear(in_features=128, out_features=512, bias=False)
159
+ (v_proj): Linear(in_features=128, out_features=512, bias=False)
160
+ (r_proj): Linear(in_features=128, out_features=128, bias=False)
161
+ (o_proj): Linear(in_features=1024, out_features=128, bias=False)
162
  (k_sconv): Inkling2ShortConvolution(
163
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
164
  )
165
  (v_sconv): Inkling2ShortConvolution(
166
+ (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
167
  )
168
  (rel_logits_proj): Inkling2RelativeLogits()
169
+ (gate_proj): Linear(in_features=128, out_features=1024, bias=False)
170
+ (sink_proj): Linear(in_features=128, out_features=8, bias=False)
171
  (q_l2_clip): Inkling2ScaledL2Clip()
172
  (k_l2_clip): Inkling2L2Clip()
173
  (rel_l2_clip): Inkling2ScaledL2Clip()
 
184
  (score_fn): Sigmoid()
185
  )
186
  )
187
+ (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
188
+ (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
189
  (post_attention_convolution): Inkling2ShortConvolution(
190
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
191
  )
192
  (post_mlp_convolution): Inkling2ShortConvolution(
193
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
194
  )
195
  )
196
  (11): Inkling2FullDecoderLayer(
 
201
  (norm): Inkling2UnweightedRMSNorm()
202
  )
203
  (self_attn): Inkling2FullAttention(
204
+ (q_proj): Linear(in_features=128, out_features=1024, bias=False)
205
+ (k_proj): Linear(in_features=128, out_features=512, bias=False)
206
+ (v_proj): Linear(in_features=128, out_features=512, bias=False)
207
+ (r_proj): Linear(in_features=128, out_features=128, bias=False)
208
+ (o_proj): Linear(in_features=1024, out_features=128, bias=False)
209
  (k_sconv): Inkling2DynamicShortConvolution(
210
+ (weight_proj): Linear(in_features=128, out_features=16, bias=False)
211
  )
212
  (v_sconv): Inkling2DynamicShortConvolution(
213
+ (weight_proj): Linear(in_features=128, out_features=16, bias=False)
214
  )
215
  (rel_logits_proj): Inkling2RelativeLogits()
216
+ (gate_proj): Linear(in_features=128, out_features=1024, bias=False)
217
+ (sink_proj): Linear(in_features=128, out_features=8, bias=False)
218
  (q_l2_clip): Inkling2ScaledL2Clip()
219
  (k_l2_clip): Inkling2L2Clip()
220
  (rel_l2_clip): Inkling2ScaledL2Clip()
 
231
  (score_fn): Sigmoid()
232
  )
233
  )
234
+ (q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
235
+ (k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
236
+ (v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
237
+ (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
238
  (post_attention_convolution): Inkling2ShortConvolution(
239
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
240
  )
241
  (post_mlp_convolution): Inkling2ShortConvolution(
242
+ (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
243
  )
244
  )
245
  )
246
+ (norm): Inkling2RMSNorm((128,), eps=1e-06)
247
  (final_input): Inkling2BlockAttentionResidual(
248
  (norm): Inkling2UnweightedRMSNorm()
249
  )
 
275
  (merger): Inkling2VisionMerger(
276
  (linear_1): Linear(in_features=128, out_features=32, bias=False)
277
  (act): GELUActivation()
278
+ (linear_2): Linear(in_features=32, out_features=128, bias=False)
279
  )
280
  )
281
  )
282
+ (lm_head): Linear(in_features=128, out_features=201024, bias=False)
283
  (logit_softcap): Inkling2Softcap()
284
  (mtp): Module(
285
  (layers): Module(
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- library_name: transformers
3
  pipeline_tag: image-text-to-text
4
  base_model:
5
  - thinkingmachines/Inkling-2-Small
@@ -9,66 +9,39 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
9
 
10
  | Feature | Source | Tiny |
11
  |---|---|---|
12
- | Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 16.40M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
13
- | Safetensors weight files (including MTP) | 393.48 GB (25 files) | 35.68 MB (2 files) |
14
  | Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
15
  | Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
16
  | Routed experts / experts per token | 384 / 8 | 96 / 8 |
17
  | Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
18
  | Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
19
- | Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
20
- | Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
21
  | Vision width / layers | 1,024 / 23 | 32 / 4 |
22
  | MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
23
  | MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
24
 
25
  | File path | Size |
26
  |------|------|
27
- | model.safetensors | 31.7MB |
28
- | mtp.safetensors | 4.0MB |
29
 
30
  ### Example usage
31
 
32
- #### Transformers (unverified)
33
 
34
- Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
35
-
36
- ```python
37
- def run_transformers(model_id: str):
38
- import torch
39
- from PIL import Image
40
- from transformers import AutoModelForImageTextToText, AutoProcessor
41
-
42
- processor = AutoProcessor.from_pretrained(model_id)
43
- model = AutoModelForImageTextToText.from_pretrained(
44
- model_id, trust_remote_code=True, dtype=torch.bfloat16, device_map="cuda"
45
- )
46
- image = Image.new("RGB", (80, 80), (80, 120, 160))
47
- inputs = processor(text=f"{processor.image_token}Describe this image.", images=image, return_tensors="pt").to(
48
- model.device
49
- )
50
- with torch.inference_mode():
51
- output = model.generate(**inputs, max_new_tokens=10, min_new_tokens=10, do_sample=False)
52
- print(processor.batch_decode(output[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))
53
-
54
-
55
- if __name__ == "__main__":
56
- run_transformers(model_id="tiny-random/inkling-2")
57
- ```
58
-
59
- #### SGLang serving (unverified)
60
-
61
- Use the pinned SGLang version and the bundled small-matrix kernel plugin.
62
 
63
  ```bash
64
  SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
65
  hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
66
- pip install --no-deps ./inkling-2-tiny
67
- sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
68
  --dtype bfloat16 --attention-backend fa4 --enable-multimodal \
69
  --context-length 2048 --mem-fraction-static 0.2 \
70
  --disable-custom-all-reduce \
71
- --cuda-graph-config '{"decode": {"bs": [1, 2, 4]}, "prefill": {"bs": [128, 2048]}}' \
 
72
  --reasoning-parser inkling --tool-call-parser inkling2
73
  ```
74
 
@@ -80,6 +53,6 @@ sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
80
 
81
  ### Change log
82
 
83
- - 2026-11-14: Updated tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, independent token embeddings/output head, and BF16 MTP.
84
 
85
  *This repo is generated by GPT-6.1-Sol*
 
1
  ---
2
+ library_name: sglang
3
  pipeline_tag: image-text-to-text
4
  base_model:
5
  - thinkingmachines/Inkling-2-Small
 
9
 
10
  | Feature | Source | Tiny |
11
  |---|---|---|
12
+ | Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 63.01M BF16, 5.90K F32, 30.08M F8_E4M3, 15.04M U8 |
13
+ | Safetensors weight files (including MTP) | 393.48 GB (25 files) | 171.22 MB (2 files) |
14
  | Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
15
  | Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
16
  | Routed experts / experts per token | 384 / 8 | 96 / 8 |
17
  | Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
18
  | Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
19
+ | Hidden / expert width | 4,608 / 2,304 | 128 / 128 |
20
+ | Q heads / KV heads / head dimension | 48 / 8 / 128 | 8 / 4 / 128 |
21
  | Vision width / layers | 1,024 / 23 | 32 / 4 |
22
  | MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
23
  | MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
24
 
25
  | File path | Size |
26
  |------|------|
27
+ | model.safetensors | 162.2MB |
28
+ | mtp.safetensors | 9.0MB |
29
 
30
  ### Example usage
31
 
32
+ #### SGLang serving (✅ verified)
33
 
34
+ Use SGLang with Inkling 2 support. No model-specific package is required.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
 
36
  ```bash
37
  SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
38
  hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
39
+ sglang serve --model-path ./inkling-2-tiny --tp 2 --ep 2 \
 
40
  --dtype bfloat16 --attention-backend fa4 --enable-multimodal \
41
  --context-length 2048 --mem-fraction-static 0.2 \
42
  --disable-custom-all-reduce \
43
+ --skip-server-warmup \
44
+ --cuda-graph-config '{"decode": {"bs": [16]}, "prefill": {"bs": [128, 2048]}}' \
45
  --reasoning-parser inkling --tool-call-parser inkling2
46
  ```
47
 
 
53
 
54
  ### Change log
55
 
56
+ - 2026-11-14: Preserve source mixed NVFP4/MXFP8 expert precision, with 96 experts and independent token tables.
57
 
58
  *This repo is generated by GPT-6.1-Sol*
config.json CHANGED
@@ -7,12 +7,12 @@
7
  "text_config": {
8
  "model_max_length": 1048576,
9
  "torch_dtype": "bfloat16",
10
- "hidden_size": 32,
11
  "num_hidden_layers": 12,
12
  "vocab_size": 201024,
13
- "num_attention_heads": 24,
14
  "num_key_value_heads": 4,
15
- "head_dim": 32,
16
  "d_rel": 16,
17
  "rel_extent": 1024,
18
  "q_bias": false,
@@ -40,8 +40,8 @@
40
  "sconv_kernel_size": 4,
41
  "unpadded_vocab_size": 200058,
42
  "final_logit_softcapping": 30.0,
43
- "swa_head_dim": 32,
44
- "swa_num_attention_heads": 24,
45
  "swa_num_key_value_heads": 4,
46
  "sliding_window_size": 1024,
47
  "n_layers_per_attn_res_block": 6,
@@ -49,7 +49,7 @@
49
  "n_routed_experts": 96,
50
  "num_experts_per_tok": 8,
51
  "n_shared_experts": 0,
52
- "intermediate_size": 32,
53
  "mlp_gate_softcap": 4.0,
54
  "mlp_up_softcap": 25.0,
55
  "route_scale": 8.0,
@@ -68,14 +68,14 @@
68
  "mlp_ratio": 4.0,
69
  "posemb_grid_size": 32,
70
  "layer_norm_eps": 1e-05,
71
- "text_hidden_size": 32
72
  },
73
  "mtp_config": {
74
- "hidden_size": 32,
75
  "num_hidden_layers": 5,
76
- "num_attention_heads": 24,
77
  "num_key_value_heads": 4,
78
- "head_dim": 32,
79
  "intermediate_size": 128,
80
  "max_position_embeddings": 1048576,
81
  "torch_dtype": "bfloat16",
@@ -99,7 +99,6 @@
99
  ]
100
  },
101
  "tie_word_embeddings": false,
102
- "auto_map": {
103
- "AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
104
- }
105
  }
 
7
  "text_config": {
8
  "model_max_length": 1048576,
9
  "torch_dtype": "bfloat16",
10
+ "hidden_size": 128,
11
  "num_hidden_layers": 12,
12
  "vocab_size": 201024,
13
+ "num_attention_heads": 8,
14
  "num_key_value_heads": 4,
15
+ "head_dim": 128,
16
  "d_rel": 16,
17
  "rel_extent": 1024,
18
  "q_bias": false,
 
40
  "sconv_kernel_size": 4,
41
  "unpadded_vocab_size": 200058,
42
  "final_logit_softcapping": 30.0,
43
+ "swa_head_dim": 128,
44
+ "swa_num_attention_heads": 8,
45
  "swa_num_key_value_heads": 4,
46
  "sliding_window_size": 1024,
47
  "n_layers_per_attn_res_block": 6,
 
49
  "n_routed_experts": 96,
50
  "num_experts_per_tok": 8,
51
  "n_shared_experts": 0,
52
+ "intermediate_size": 128,
53
  "mlp_gate_softcap": 4.0,
54
  "mlp_up_softcap": 25.0,
55
  "route_scale": 8.0,
 
68
  "mlp_ratio": 4.0,
69
  "posemb_grid_size": 32,
70
  "layer_norm_eps": 1e-05,
71
+ "text_hidden_size": 128
72
  },
73
  "mtp_config": {
74
+ "hidden_size": 128,
75
  "num_hidden_layers": 5,
76
+ "num_attention_heads": 8,
77
  "num_key_value_heads": 4,
78
+ "head_dim": 128,
79
  "intermediate_size": 128,
80
  "max_position_embeddings": 1048576,
81
  "torch_dtype": "bfloat16",
 
99
  ]
100
  },
101
  "tie_word_embeddings": false,
102
+ "transformers_version": "5.23.0.dev0",
103
+ "dtype": "bfloat16"
 
104
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6a3c70ce16c57bdfd587f0e6720c7e1e33384f4651cf4b14204ce71732a318ae
3
- size 31674312
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac7f0eaa84e50b53c4afce3ea7eecdc49ccca21bd467edb16f3d052d9099a3de
3
+ size 162239128
modeling_inkling2_tiny.py DELETED
@@ -1,100 +0,0 @@
1
- """Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
2
-
3
- from pathlib import Path
4
-
5
- import torch
6
- from huggingface_hub import hf_hub_download
7
- from safetensors.torch import load_file
8
- from transformers import Inkling2Config, Inkling2ForConditionalGeneration
9
- from transformers.conversion_mapping import get_model_conversion_mapping
10
- from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
11
-
12
-
13
- def dequantize(weight, scale, amax=None, denominator=None):
14
- if weight.dtype == torch.uint8:
15
- if amax is None or denominator is None:
16
- raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
17
- codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
18
- levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
19
- values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
20
- return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
21
- if weight.dtype == torch.float8_e4m3fn:
22
- scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
23
- return weight.float() * scales
24
- raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
25
-
26
-
27
- class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
28
- @classmethod
29
- def from_pretrained(
30
- cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
31
- revision=None, cache_dir=None, local_files_only=False, token=None,
32
- attn_implementation="eager", **kwargs,
33
- ):
34
- if model_args:
35
- raise ValueError("Inkling 2 does not take positional construction arguments")
36
- options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
37
- root = Path(pretrained_model_name_or_path)
38
-
39
- def resolve(filename):
40
- if root.is_dir():
41
- return root / filename
42
- return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
43
-
44
- import json
45
-
46
- raw = json.loads(resolve("config.json").read_text())
47
- # The native config otherwise assumes the all-MXFP8 sibling. This
48
- # adapter dequantizes each tensor according to its actual stored dtype.
49
- raw.pop("quantization_config", None)
50
- config = Inkling2Config(**raw)
51
- config._attn_implementation = attn_implementation
52
- dtype = kwargs.pop("torch_dtype", dtype)
53
- if dtype in (None, "auto"):
54
- dtype = torch.bfloat16
55
- elif isinstance(dtype, str):
56
- dtype = getattr(torch, dtype)
57
- if dtype not in (torch.float32, torch.bfloat16, torch.float16):
58
- raise ValueError("The Transformers adapter requires a floating compute dtype")
59
- device_map = kwargs.pop("device_map", None)
60
- if device_map == "auto":
61
- device_map = "cuda" if torch.cuda.is_available() else "cpu"
62
- if isinstance(device_map, dict):
63
- if set(device_map) != {""}:
64
- raise ValueError("Use a single device for this tiny Transformers adapter")
65
- device_map = device_map[""]
66
- allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
67
- unexpected = set(kwargs) - allowed
68
- if unexpected:
69
- raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
70
- model = Inkling2ForConditionalGeneration(config)
71
- transforms = get_model_conversion_mapping(model)
72
- renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
73
- converters = [t for t in transforms if isinstance(t, WeightConverter)]
74
- source = load_file(str(resolve("model.safetensors")))
75
- state = {}
76
- for name, weight in source.items():
77
- if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
78
- continue
79
- if name + "_scale" in source:
80
- weight = dequantize(
81
- weight, source[name + "_scale"], source.get(name + "_amax"),
82
- source.get(name + "_outer_scale_denominator"),
83
- )
84
- target, _ = rename_source_key(name, renamings, converters)
85
- if name.endswith("w13_weight"):
86
- # The source interleaves gate and up; Transformers chunks them.
87
- e, rows, columns = weight.shape
88
- weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
89
- if target in state:
90
- raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
91
- keep_float32 = source[name].dtype == torch.float32 or any(
92
- part in target for part in model._keep_in_fp32_modules_strict
93
- )
94
- target_dtype = torch.float32 if keep_float32 else dtype
95
- state[target] = weight.to(target_dtype).contiguous()
96
- model.load_state_dict(state, strict=True, assign=True)
97
- model.eval()
98
- if device_map is not None:
99
- model.to(device_map)
100
- return model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mtp.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:194ed2549514367c14ba8e3df0a1f1e0a7397bb9ceaf1971b647535f52531663
3
- size 4009408
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b2d1d9b26b66f9d222696d5103486d30849f34734ac59ef172bd1d2622b0285
3
+ size 8983080
pyproject.toml DELETED
@@ -1,15 +0,0 @@
1
- [build-system]
2
- requires = ["setuptools>=68"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "sglang-inkling2-tiny-kernels"
7
- version = "0.2.0"
8
- description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
9
- requires-python = ">=3.10"
10
-
11
- [project.entry-points."sglang.srt.plugins"]
12
- inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
13
-
14
- [tool.setuptools]
15
- py-modules = ["sglang_inkling2_tiny_kernels"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sglang_inkling2_tiny_kernels.py DELETED
@@ -1,129 +0,0 @@
1
- """Small-matrix kernels for the tiny Inkling 2 checkpoint."""
2
-
3
-
4
-
5
-
6
-
7
-
8
-
9
- class _PaddedMarlin(tuple):
10
- def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
11
- value = super().__new__(cls, packed)
12
- value.input_width = input_width
13
- value.output_width = output_width
14
- value.padded_input = padded_input
15
- value.padded_output = padded_output
16
- return value
17
-
18
-
19
- def _pad_matrix(tensor, rows, columns, fill=0):
20
- output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
21
- output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
22
- return output
23
-
24
-
25
- def _repack_marlin(original, weight, scales):
26
- import torch
27
-
28
- nvfp4 = weight.dtype == torch.uint8
29
- packing = 2 if nvfp4 else 1
30
- block = 16 if nvfp4 else 32
31
- n, k = weight.shape[-2], weight.shape[-1] * packing
32
- padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
33
- if (padded_k, padded_n) == (k, n):
34
- return original(weight, scales)
35
- weight = _pad_matrix(weight, padded_n, padded_k // packing)
36
- # Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
37
- # Zero weight bytes make padded products zero in both precision families.
38
- scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
39
- packed = original(weight, scales)
40
- return _PaddedMarlin(packed, k, n, padded_k, padded_n)
41
-
42
-
43
- def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
44
- if not isinstance(packed, _PaddedMarlin):
45
- return original(x, packed, out_size, *args, **kwargs)
46
- if x.shape[-1] != packed.input_width or out_size != packed.output_width:
47
- raise ValueError("Marlin padding metadata does not match the expert projection")
48
- padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
49
- padded[..., :x.shape[-1]].copy_(x)
50
- if original.__name__ == "gemv" and packed.padded_input == 128:
51
- # Eight K tiles fit one split of eight warps. The large-model heuristic
52
- # otherwise chooses four splits/four warps for short decode batches.
53
- kwargs = {**kwargs, "splits": 1, "warps": 8}
54
- result = original(padded, packed, packed.padded_output, *args, **kwargs)
55
- return result[..., :out_size].contiguous()
56
-
57
-
58
- def _relative_fa4(original, *args, **kwargs):
59
- q = args[0] if args else kwargs.get("q")
60
- if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
61
- and kwargs.get("tile_mn") is None):
62
- # The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
63
- # relative-bias kernel explicitly requires M=N=128. Use its tile API.
64
- kwargs = {**kwargs, "tile_mn": (128, 128)}
65
- return original(*args, **kwargs)
66
-
67
-
68
- def _attn_res_config(original, width, *args, **kwargs):
69
- threads, stages, vector, cluster = original(width, *args, **kwargs)
70
- if width == 32:
71
- # One compute warp covers the original 32 residual coordinates.
72
- # Keep the same CuTe kernel, RMS normalization, and source weighting.
73
- return 32, stages, 1, 1
74
- return threads, stages, vector, cluster
75
-
76
-
77
- def _small_all_reduce(original, x, group, **kwargs):
78
- if x.ndim == 2 and x.shape[-1] == 32:
79
- return group.all_reduce(x.clone())
80
- return original(x, group, **kwargs)
81
-
82
-
83
- def _small_multicast_input(original, group, shape, dtype):
84
- if len(shape) == 2 and shape[-1] == 32:
85
- import torch
86
-
87
- return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
88
- return original(group, shape, dtype)
89
-
90
-
91
- def _small_multicast_reduce(original, x, group):
92
- if x.ndim == 2 and x.shape[-1] == 32:
93
- x.copy_(group.all_reduce(x))
94
- return x
95
- return original(x, group)
96
-
97
-
98
- def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
99
- if cols == 32:
100
- # Use the existing FP32 combine path instead of the GB300-tuned
101
- # symmetric-memory one-shot collective for these tiny projections.
102
- return None
103
- return original(group, rows, cols, dtype, tag=tag)
104
-
105
-
106
- def register():
107
- from sglang.srt.plugins.hook_registry import HookRegistry, HookType
108
-
109
- prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
110
- for name in ("repack", "repack_mxfp8"):
111
- HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
112
- for name in ("gemm", "gemv"):
113
- HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
114
- HookRegistry.register(
115
- "sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
116
- _relative_fa4, HookType.AROUND,
117
- )
118
- HookRegistry.register(
119
- "sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
120
- _attn_res_config, HookType.AROUND,
121
- )
122
- prefix = "sglang.kernels.ops.inkling2.comm"
123
- for name, hook in (
124
- ("all_reduce", _small_all_reduce),
125
- ("multicast_all_reduce_input", _small_multicast_input),
126
- ("multicast_all_reduce", _small_multicast_reduce),
127
- ("oneshot_all_reduce_input", _small_oneshot_input),
128
- ):
129
- HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)