yujiepan commited on
Commit
4c88f31
·
verified ·
1 Parent(s): 08c77f0

Restore independent Inkling 2 embedding and output head

Browse files
.meta/create_model.py CHANGED
@@ -30,10 +30,10 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
30
  swa_num_key_value_heads=4,
31
  swa_head_dim=32,
32
  n_routed_experts=96,
33
- tie_word_embeddings=True,
34
  local_layer_ids=[i for i in range(12) if i % 6 != 5],
35
  )
36
- raw["tie_word_embeddings"] = True
37
  raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
38
  raw["vision_config"].update(
39
  hidden_size=32,
@@ -74,9 +74,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
74
  parameter.fill_(1)
75
  else:
76
  parameter.normal_(0, 0.02)
77
- # Inkling 2's concrete Transformers class deliberately has no tied-weight
78
- # mapping. Tie explicitly, then serialize the embedding only.
79
- model.lm_head.weight = model.model.language_model.embed_tokens.weight
80
 
81
  # Reverse Transformers' documented source-to-HF name conversion. Retain the
82
  # source's interleaved gate/up rows and original serialized precision.
@@ -145,8 +142,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
145
  }
146
  state = {}
147
  for hf_name, value in sorted(model.state_dict().items()):
148
- if hf_name == "lm_head.weight":
149
- continue
150
  name = original_name(hf_name)
151
  source = source_name(name)
152
  if source not in headers:
@@ -239,10 +234,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
239
  remapped_groups = []
240
  for group in groups:
241
  names = [name for name in state if source_name(name) in group["tensors"]]
242
- if "model.llm.unembed.weight" in group["tensors"]:
243
- # The adapter supplies this virtual tensor to the native loader;
244
- # keep its FP32-logit computation descriptor even without a file copy.
245
- names.append("model.llm.unembed.weight")
246
  if names:
247
  remapped_groups.append({**group, "tensors": sorted(names)})
248
  precision["model"]["parameter_groups"] = remapped_groups
@@ -259,47 +250,13 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
259
  path = folder / name
260
  path.parent.mkdir(parents=True, exist_ok=True)
261
  path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
262
- plugin_code = '''"""Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
263
 
264
- import logging
265
 
266
- logger = logging.getLogger(__name__)
267
 
268
 
269
- def _config_init(original, config, *args, **kwargs):
270
- tied = kwargs.get("tie_word_embeddings", False)
271
- if tied:
272
- kwargs = {**kwargs, "tie_word_embeddings": False}
273
- original(config, *args, **kwargs)
274
- if tied:
275
- config.tie_word_embeddings = True
276
 
277
 
278
- def _load_weights(original, model, weights):
279
- if not model.text_config.tie_word_embeddings or model.llm is None:
280
- return original(model, weights)
281
-
282
- def with_head():
283
- embedding = None
284
- for name, tensor in weights:
285
- if name == "model.llm.unembed.weight":
286
- raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
287
- if name == "model.llm.embed.weight":
288
- embedding = tensor
289
- yield name, tensor
290
- if embedding is None:
291
- raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
292
- yield "model.llm.unembed.weight", embedding
293
-
294
- loaded = original(model, with_head())
295
- embedding, head = model.llm.embed_tokens, model.llm.lm_head
296
- if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
297
- raise ValueError("Tied embedding/head tensor-parallel partitions must match")
298
- head.weight = embedding.weight
299
- assert head.weight is embedding.weight
300
- logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
301
- return loaded
302
-
303
 
304
  class _PaddedMarlin(tuple):
305
  def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
@@ -401,12 +358,6 @@ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
401
  def register():
402
  from sglang.srt.plugins.hook_registry import HookRegistry, HookType
403
 
404
- HookRegistry.register(
405
- "sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
406
- )
407
- HookRegistry.register(
408
- "sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
409
- )
410
  prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
411
  for name in ("repack", "repack_mxfp8"):
412
  HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
@@ -429,7 +380,7 @@ def register():
429
  ):
430
  HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
431
  '''
432
- (folder / "sglang_inkling2_tied.py").write_text(plugin_code)
433
  hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
434
 
435
  from pathlib import Path
@@ -525,12 +476,10 @@ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
525
  )
526
  target_dtype = torch.float32 if keep_float32 else dtype
527
  state[target] = weight.to(target_dtype).contiguous()
528
- state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
529
  model.load_state_dict(state, strict=True, assign=True)
530
  model.eval()
531
  if device_map is not None:
532
  model.to(device_map)
533
- model.lm_head.weight = model.model.language_model.embed_tokens.weight
534
  return model
535
  '''
536
  (folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
@@ -539,16 +488,16 @@ requires = ["setuptools>=68"]
539
  build-backend = "setuptools.build_meta"
540
 
541
  [project]
542
- name = "sglang-inkling2-tied"
543
- version = "0.1.0"
544
- description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
545
  requires-python = ">=3.10"
546
 
547
  [project.entry-points."sglang.srt.plugins"]
548
- inkling2_tied = "sglang_inkling2_tied:register"
549
 
550
  [tool.setuptools]
551
- py-modules = ["sglang_inkling2_tied"]
552
  """)
553
  return str(model)
554
 
@@ -557,5 +506,5 @@ if __name__ == "__main__":
557
  create_model(
558
  source_model_id="thinkingmachines/Inkling-2-Small",
559
  source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
560
- save_folder="/tmp/inkling-2-tiny-random",
561
  )
 
30
  swa_num_key_value_heads=4,
31
  swa_head_dim=32,
32
  n_routed_experts=96,
33
+ tie_word_embeddings=False,
34
  local_layer_ids=[i for i in range(12) if i % 6 != 5],
35
  )
36
+ raw["tie_word_embeddings"] = False
37
  raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
38
  raw["vision_config"].update(
39
  hidden_size=32,
 
74
  parameter.fill_(1)
75
  else:
76
  parameter.normal_(0, 0.02)
 
 
 
77
 
78
  # Reverse Transformers' documented source-to-HF name conversion. Retain the
79
  # source's interleaved gate/up rows and original serialized precision.
 
142
  }
143
  state = {}
144
  for hf_name, value in sorted(model.state_dict().items()):
 
 
145
  name = original_name(hf_name)
146
  source = source_name(name)
147
  if source not in headers:
 
234
  remapped_groups = []
235
  for group in groups:
236
  names = [name for name in state if source_name(name) in group["tensors"]]
 
 
 
 
237
  if names:
238
  remapped_groups.append({**group, "tensors": sorted(names)})
239
  precision["model"]["parameter_groups"] = remapped_groups
 
250
  path = folder / name
251
  path.parent.mkdir(parents=True, exist_ok=True)
252
  path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
253
+ plugin_code = '''"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
254
 
 
255
 
 
256
 
257
 
 
 
 
 
 
 
 
258
 
259
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
260
 
261
  class _PaddedMarlin(tuple):
262
  def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
 
358
  def register():
359
  from sglang.srt.plugins.hook_registry import HookRegistry, HookType
360
 
 
 
 
 
 
 
361
  prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
362
  for name in ("repack", "repack_mxfp8"):
363
  HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
 
380
  ):
381
  HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
382
  '''
383
+ (folder / "sglang_inkling2_tiny_kernels.py").write_text(plugin_code)
384
  hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
385
 
386
  from pathlib import Path
 
476
  )
477
  target_dtype = torch.float32 if keep_float32 else dtype
478
  state[target] = weight.to(target_dtype).contiguous()
 
479
  model.load_state_dict(state, strict=True, assign=True)
480
  model.eval()
481
  if device_map is not None:
482
  model.to(device_map)
 
483
  return model
484
  '''
485
  (folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
 
488
  build-backend = "setuptools.build_meta"
489
 
490
  [project]
491
+ name = "sglang-inkling2-tiny-kernels"
492
+ version = "0.2.0"
493
+ description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
494
  requires-python = ">=3.10"
495
 
496
  [project.entry-points."sglang.srt.plugins"]
497
+ inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
498
 
499
  [tool.setuptools]
500
+ py-modules = ["sglang_inkling2_tiny_kernels"]
501
  """)
502
  return str(model)
503
 
 
506
  create_model(
507
  source_model_id="thinkingmachines/Inkling-2-Small",
508
  source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
509
+ save_folder="/tmp/inkling-2",
510
  )
.meta/env.json CHANGED
@@ -15,43 +15,16 @@
15
  },
16
  "cuda": "12.6"
17
  },
18
- "sglang": {
19
  "python": "3.12.13",
20
  "packages": {
21
- "Pillow": "12.3.0",
22
- "flashinfer-python": "0.7.4",
23
- "huggingface_hub": "1.33.0",
24
- "nvidia-cutlass-dsl": "4.8.0",
25
- "safetensors": "0.8.0",
26
- "sglang": "0.0.0.dev0",
27
- "sglang-inkling2-tied": "0.1.0",
28
- "sglang-kernel": "0.4.10",
29
- "tiktoken": "0.14.0",
30
- "tml-renderers": "0.4.1",
31
- "tokenizers": "0.23.2",
32
- "torch": "2.14.1+cu130",
33
- "transformers": "5.19.0",
34
- "triton": "3.8.0"
35
- },
36
- "cuda": "13.0",
37
- "hardware": {
38
- "gpu_model": "NVIDIA H20"
39
- }
40
- },
41
- "transformers": {
42
- "python": "3.12.13",
43
- "packages": {
44
- "Pillow": "12.3.0",
45
  "huggingface_hub": "2.3.0",
46
  "safetensors": "0.8.0",
47
  "tokenizers": "0.23.3",
48
  "torch": "2.13.0+cu126",
49
  "transformers": "5.23.0.dev0"
50
  },
51
- "cuda": "12.6",
52
- "hardware": {
53
- "gpu_model": "NVIDIA H20"
54
- }
55
  }
56
  }
57
  }
 
15
  },
16
  "cuda": "12.6"
17
  },
18
+ "transformers-reference": {
19
  "python": "3.12.13",
20
  "packages": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  "huggingface_hub": "2.3.0",
22
  "safetensors": "0.8.0",
23
  "tokenizers": "0.23.3",
24
  "torch": "2.13.0+cu126",
25
  "transformers": "5.23.0.dev0"
26
  },
27
+ "cuda": "12.6"
 
 
 
28
  }
29
  }
30
  }
README.md CHANGED
@@ -9,13 +9,13 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
9
 
10
  | Feature | Source | Tiny |
11
  |---|---|---|
12
- | Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 9.97M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
13
- | Safetensors weight files (including MTP) | 393.48 GB (25 files) | 22.82 MB (2 files) |
14
  | Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
15
  | Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
16
  | Routed experts / experts per token | 384 / 8 | 96 / 8 |
17
  | Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
18
- | Vocabulary / tied embeddings | 201,024 / false | 201,024 / true |
19
  | Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
20
  | Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
21
  | Vision width / layers | 1,024 / 23 | 32 / 4 |
@@ -24,12 +24,12 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
24
 
25
  | File path | Size |
26
  |------|------|
27
- | model.safetensors | 18.8MB |
28
  | mtp.safetensors | 4.0MB |
29
 
30
  ### Example usage
31
 
32
- #### Transformers (✅ verified)
33
 
34
  Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
35
 
@@ -56,9 +56,9 @@ if __name__ == "__main__":
56
  run_transformers(model_id="tiny-random/inkling-2")
57
  ```
58
 
59
- #### SGLang serving (✅ verified)
60
 
61
- Use the Inkling 2 support branch of SGLang; the general release does not contain this architecture.
62
 
63
  ```bash
64
  SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
@@ -80,6 +80,6 @@ sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
80
 
81
  ### Change log
82
 
83
- - 2026-11-13: Created tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, tied token embeddings, and BF16 MTP.
84
 
85
  *This repo is generated by GPT-6.1-Sol*
 
9
 
10
  | Feature | Source | Tiny |
11
  |---|---|---|
12
+ | Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 16.40M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
13
+ | Safetensors weight files (including MTP) | 393.48 GB (25 files) | 35.68 MB (2 files) |
14
  | Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
15
  | Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
16
  | Routed experts / experts per token | 384 / 8 | 96 / 8 |
17
  | Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
18
+ | Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
19
  | Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
20
  | Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
21
  | Vision width / layers | 1,024 / 23 | 32 / 4 |
 
24
 
25
  | File path | Size |
26
  |------|------|
27
+ | model.safetensors | 31.7MB |
28
  | mtp.safetensors | 4.0MB |
29
 
30
  ### Example usage
31
 
32
+ #### Transformers (unverified)
33
 
34
  Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
35
 
 
56
  run_transformers(model_id="tiny-random/inkling-2")
57
  ```
58
 
59
+ #### SGLang serving (unverified)
60
 
61
+ Use the pinned SGLang version and the bundled small-matrix kernel plugin.
62
 
63
  ```bash
64
  SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
 
80
 
81
  ### Change log
82
 
83
+ - 2026-11-14: Updated tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, independent token embeddings/output head, and BF16 MTP.
84
 
85
  *This repo is generated by GPT-6.1-Sol*
config.json CHANGED
@@ -54,7 +54,7 @@
54
  "mlp_up_softcap": 25.0,
55
  "route_scale": 8.0,
56
  "use_gate_bias": true,
57
- "tie_word_embeddings": true
58
  },
59
  "vision_config": {
60
  "temporal_patch_size": 2,
@@ -98,7 +98,7 @@
98
  12
99
  ]
100
  },
101
- "tie_word_embeddings": true,
102
  "auto_map": {
103
  "AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
104
  }
 
54
  "mlp_up_softcap": 25.0,
55
  "route_scale": 8.0,
56
  "use_gate_bias": true,
57
+ "tie_word_embeddings": false
58
  },
59
  "vision_config": {
60
  "temporal_patch_size": 2,
 
98
  12
99
  ]
100
  },
101
+ "tie_word_embeddings": false,
102
  "auto_map": {
103
  "AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
104
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9c24250280ea240d1957cb6b92fb597d1a6c2beca4c926ef0d10d8342d669657
3
- size 18808680
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a3c70ce16c57bdfd587f0e6720c7e1e33384f4651cf4b14204ce71732a318ae
3
+ size 31674312
modeling_inkling2_tiny.py CHANGED
@@ -93,10 +93,8 @@ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
93
  )
94
  target_dtype = torch.float32 if keep_float32 else dtype
95
  state[target] = weight.to(target_dtype).contiguous()
96
- state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
97
  model.load_state_dict(state, strict=True, assign=True)
98
  model.eval()
99
  if device_map is not None:
100
  model.to(device_map)
101
- model.lm_head.weight = model.model.language_model.embed_tokens.weight
102
  return model
 
93
  )
94
  target_dtype = torch.float32 if keep_float32 else dtype
95
  state[target] = weight.to(target_dtype).contiguous()
 
96
  model.load_state_dict(state, strict=True, assign=True)
97
  model.eval()
98
  if device_map is not None:
99
  model.to(device_map)
 
100
  return model
pyproject.toml CHANGED
@@ -3,13 +3,13 @@ requires = ["setuptools>=68"]
3
  build-backend = "setuptools.build_meta"
4
 
5
  [project]
6
- name = "sglang-inkling2-tied"
7
- version = "0.1.0"
8
- description = "Tied tiny Inkling 2 checkpoint loading through SGLang hooks"
9
  requires-python = ">=3.10"
10
 
11
  [project.entry-points."sglang.srt.plugins"]
12
- inkling2_tied = "sglang_inkling2_tied:register"
13
 
14
  [tool.setuptools]
15
- py-modules = ["sglang_inkling2_tied"]
 
3
  build-backend = "setuptools.build_meta"
4
 
5
  [project]
6
+ name = "sglang-inkling2-tiny-kernels"
7
+ version = "0.2.0"
8
+ description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
9
  requires-python = ">=3.10"
10
 
11
  [project.entry-points."sglang.srt.plugins"]
12
+ inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
13
 
14
  [tool.setuptools]
15
+ py-modules = ["sglang_inkling2_tiny_kernels"]
sglang_inkling2_tiny_kernels.egg-info/PKG-INFO ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: sglang-inkling2-tiny-kernels
3
+ Version: 0.2.0
4
+ Summary: Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint
5
+ Requires-Python: >=3.10
sglang_inkling2_tiny_kernels.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ README.md
2
+ pyproject.toml
3
+ sglang_inkling2_tiny_kernels.py
4
+ sglang_inkling2_tiny_kernels.egg-info/PKG-INFO
5
+ sglang_inkling2_tiny_kernels.egg-info/SOURCES.txt
6
+ sglang_inkling2_tiny_kernels.egg-info/dependency_links.txt
7
+ sglang_inkling2_tiny_kernels.egg-info/entry_points.txt
8
+ sglang_inkling2_tiny_kernels.egg-info/top_level.txt
sglang_inkling2_tiny_kernels.egg-info/dependency_links.txt ADDED
@@ -0,0 +1 @@
 
 
1
+
sglang_inkling2_tiny_kernels.egg-info/entry_points.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ [sglang.srt.plugins]
2
+ inkling2_tiny_kernels = sglang_inkling2_tiny_kernels:register
sglang_inkling2_tiny_kernels.egg-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ sglang_inkling2_tiny_kernels
sglang_inkling2_tied.py → sglang_inkling2_tiny_kernels.py RENAMED
@@ -1,44 +1,10 @@
1
- """Tied Inkling 2 token tables through SGLang's supported plugin hooks."""
2
 
3
- import logging
4
 
5
- logger = logging.getLogger(__name__)
6
 
7
 
8
- def _config_init(original, config, *args, **kwargs):
9
- tied = kwargs.get("tie_word_embeddings", False)
10
- if tied:
11
- kwargs = {**kwargs, "tie_word_embeddings": False}
12
- original(config, *args, **kwargs)
13
- if tied:
14
- config.tie_word_embeddings = True
15
 
16
 
17
- def _load_weights(original, model, weights):
18
- if not model.text_config.tie_word_embeddings or model.llm is None:
19
- return original(model, weights)
20
-
21
- def with_head():
22
- embedding = None
23
- for name, tensor in weights:
24
- if name == "model.llm.unembed.weight":
25
- raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
26
- if name == "model.llm.embed.weight":
27
- embedding = tensor
28
- yield name, tensor
29
- if embedding is None:
30
- raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
31
- yield "model.llm.unembed.weight", embedding
32
-
33
- loaded = original(model, with_head())
34
- embedding, head = model.llm.embed_tokens, model.llm.lm_head
35
- if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
36
- raise ValueError("Tied embedding/head tensor-parallel partitions must match")
37
- head.weight = embedding.weight
38
- assert head.weight is embedding.weight
39
- logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
40
- return loaded
41
-
42
 
43
  class _PaddedMarlin(tuple):
44
  def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
@@ -140,12 +106,6 @@ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
140
  def register():
141
  from sglang.srt.plugins.hook_registry import HookRegistry, HookType
142
 
143
- HookRegistry.register(
144
- "sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
145
- )
146
- HookRegistry.register(
147
- "sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
148
- )
149
  prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
150
  for name in ("repack", "repack_mxfp8"):
151
  HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
 
1
+ """Small-matrix kernels for the tiny Inkling 2 checkpoint."""
2
 
 
3
 
 
4
 
5
 
 
 
 
 
 
 
 
6
 
7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  class _PaddedMarlin(tuple):
10
  def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
 
106
  def register():
107
  from sglang.srt.plugins.hook_registry import HookRegistry, HookType
108
 
 
 
 
 
 
 
109
  prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
110
  for name in ("repack", "repack_mxfp8"):
111
  HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)