Restore independent Inkling 2 embedding and output head
Browse files- .meta/create_model.py +10 -61
- .meta/env.json +2 -29
- README.md +8 -8
- config.json +2 -2
- model.safetensors +2 -2
- modeling_inkling2_tiny.py +0 -2
- pyproject.toml +5 -5
- sglang_inkling2_tiny_kernels.egg-info/PKG-INFO +5 -0
- sglang_inkling2_tiny_kernels.egg-info/SOURCES.txt +8 -0
- sglang_inkling2_tiny_kernels.egg-info/dependency_links.txt +1 -0
- sglang_inkling2_tiny_kernels.egg-info/entry_points.txt +2 -0
- sglang_inkling2_tiny_kernels.egg-info/top_level.txt +1 -0
- sglang_inkling2_tied.py → sglang_inkling2_tiny_kernels.py +1 -41
.meta/create_model.py
CHANGED
|
@@ -30,10 +30,10 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 30 |
swa_num_key_value_heads=4,
|
| 31 |
swa_head_dim=32,
|
| 32 |
n_routed_experts=96,
|
| 33 |
-
tie_word_embeddings=
|
| 34 |
local_layer_ids=[i for i in range(12) if i % 6 != 5],
|
| 35 |
)
|
| 36 |
-
raw["tie_word_embeddings"] =
|
| 37 |
raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
|
| 38 |
raw["vision_config"].update(
|
| 39 |
hidden_size=32,
|
|
@@ -74,9 +74,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 74 |
parameter.fill_(1)
|
| 75 |
else:
|
| 76 |
parameter.normal_(0, 0.02)
|
| 77 |
-
# Inkling 2's concrete Transformers class deliberately has no tied-weight
|
| 78 |
-
# mapping. Tie explicitly, then serialize the embedding only.
|
| 79 |
-
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 80 |
|
| 81 |
# Reverse Transformers' documented source-to-HF name conversion. Retain the
|
| 82 |
# source's interleaved gate/up rows and original serialized precision.
|
|
@@ -145,8 +142,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 145 |
}
|
| 146 |
state = {}
|
| 147 |
for hf_name, value in sorted(model.state_dict().items()):
|
| 148 |
-
if hf_name == "lm_head.weight":
|
| 149 |
-
continue
|
| 150 |
name = original_name(hf_name)
|
| 151 |
source = source_name(name)
|
| 152 |
if source not in headers:
|
|
@@ -239,10 +234,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 239 |
remapped_groups = []
|
| 240 |
for group in groups:
|
| 241 |
names = [name for name in state if source_name(name) in group["tensors"]]
|
| 242 |
-
if "model.llm.unembed.weight" in group["tensors"]:
|
| 243 |
-
# The adapter supplies this virtual tensor to the native loader;
|
| 244 |
-
# keep its FP32-logit computation descriptor even without a file copy.
|
| 245 |
-
names.append("model.llm.unembed.weight")
|
| 246 |
if names:
|
| 247 |
remapped_groups.append({**group, "tensors": sorted(names)})
|
| 248 |
precision["model"]["parameter_groups"] = remapped_groups
|
|
@@ -259,47 +250,13 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 259 |
path = folder / name
|
| 260 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 261 |
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
| 262 |
-
plugin_code = '''"""
|
| 263 |
|
| 264 |
-
import logging
|
| 265 |
|
| 266 |
-
logger = logging.getLogger(__name__)
|
| 267 |
|
| 268 |
|
| 269 |
-
def _config_init(original, config, *args, **kwargs):
|
| 270 |
-
tied = kwargs.get("tie_word_embeddings", False)
|
| 271 |
-
if tied:
|
| 272 |
-
kwargs = {**kwargs, "tie_word_embeddings": False}
|
| 273 |
-
original(config, *args, **kwargs)
|
| 274 |
-
if tied:
|
| 275 |
-
config.tie_word_embeddings = True
|
| 276 |
|
| 277 |
|
| 278 |
-
def _load_weights(original, model, weights):
|
| 279 |
-
if not model.text_config.tie_word_embeddings or model.llm is None:
|
| 280 |
-
return original(model, weights)
|
| 281 |
-
|
| 282 |
-
def with_head():
|
| 283 |
-
embedding = None
|
| 284 |
-
for name, tensor in weights:
|
| 285 |
-
if name == "model.llm.unembed.weight":
|
| 286 |
-
raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
|
| 287 |
-
if name == "model.llm.embed.weight":
|
| 288 |
-
embedding = tensor
|
| 289 |
-
yield name, tensor
|
| 290 |
-
if embedding is None:
|
| 291 |
-
raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
|
| 292 |
-
yield "model.llm.unembed.weight", embedding
|
| 293 |
-
|
| 294 |
-
loaded = original(model, with_head())
|
| 295 |
-
embedding, head = model.llm.embed_tokens, model.llm.lm_head
|
| 296 |
-
if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
|
| 297 |
-
raise ValueError("Tied embedding/head tensor-parallel partitions must match")
|
| 298 |
-
head.weight = embedding.weight
|
| 299 |
-
assert head.weight is embedding.weight
|
| 300 |
-
logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
|
| 301 |
-
return loaded
|
| 302 |
-
|
| 303 |
|
| 304 |
class _PaddedMarlin(tuple):
|
| 305 |
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
|
@@ -401,12 +358,6 @@ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
|
| 401 |
def register():
|
| 402 |
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 403 |
|
| 404 |
-
HookRegistry.register(
|
| 405 |
-
"sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
|
| 406 |
-
)
|
| 407 |
-
HookRegistry.register(
|
| 408 |
-
"sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
|
| 409 |
-
)
|
| 410 |
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 411 |
for name in ("repack", "repack_mxfp8"):
|
| 412 |
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
|
@@ -429,7 +380,7 @@ def register():
|
|
| 429 |
):
|
| 430 |
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
| 431 |
'''
|
| 432 |
-
(folder / "
|
| 433 |
hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 434 |
|
| 435 |
from pathlib import Path
|
|
@@ -525,12 +476,10 @@ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
|
| 525 |
)
|
| 526 |
target_dtype = torch.float32 if keep_float32 else dtype
|
| 527 |
state[target] = weight.to(target_dtype).contiguous()
|
| 528 |
-
state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
|
| 529 |
model.load_state_dict(state, strict=True, assign=True)
|
| 530 |
model.eval()
|
| 531 |
if device_map is not None:
|
| 532 |
model.to(device_map)
|
| 533 |
-
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 534 |
return model
|
| 535 |
'''
|
| 536 |
(folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
|
|
@@ -539,16 +488,16 @@ requires = ["setuptools>=68"]
|
|
| 539 |
build-backend = "setuptools.build_meta"
|
| 540 |
|
| 541 |
[project]
|
| 542 |
-
name = "sglang-inkling2-
|
| 543 |
-
version = "0.
|
| 544 |
-
description = "
|
| 545 |
requires-python = ">=3.10"
|
| 546 |
|
| 547 |
[project.entry-points."sglang.srt.plugins"]
|
| 548 |
-
|
| 549 |
|
| 550 |
[tool.setuptools]
|
| 551 |
-
py-modules = ["
|
| 552 |
""")
|
| 553 |
return str(model)
|
| 554 |
|
|
@@ -557,5 +506,5 @@ if __name__ == "__main__":
|
|
| 557 |
create_model(
|
| 558 |
source_model_id="thinkingmachines/Inkling-2-Small",
|
| 559 |
source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
|
| 560 |
-
save_folder="/tmp/inkling-2
|
| 561 |
)
|
|
|
|
| 30 |
swa_num_key_value_heads=4,
|
| 31 |
swa_head_dim=32,
|
| 32 |
n_routed_experts=96,
|
| 33 |
+
tie_word_embeddings=False,
|
| 34 |
local_layer_ids=[i for i in range(12) if i % 6 != 5],
|
| 35 |
)
|
| 36 |
+
raw["tie_word_embeddings"] = False
|
| 37 |
raw["auto_map"] = {"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"}
|
| 38 |
raw["vision_config"].update(
|
| 39 |
hidden_size=32,
|
|
|
|
| 74 |
parameter.fill_(1)
|
| 75 |
else:
|
| 76 |
parameter.normal_(0, 0.02)
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
# Reverse Transformers' documented source-to-HF name conversion. Retain the
|
| 79 |
# source's interleaved gate/up rows and original serialized precision.
|
|
|
|
| 142 |
}
|
| 143 |
state = {}
|
| 144 |
for hf_name, value in sorted(model.state_dict().items()):
|
|
|
|
|
|
|
| 145 |
name = original_name(hf_name)
|
| 146 |
source = source_name(name)
|
| 147 |
if source not in headers:
|
|
|
|
| 234 |
remapped_groups = []
|
| 235 |
for group in groups:
|
| 236 |
names = [name for name in state if source_name(name) in group["tensors"]]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
if names:
|
| 238 |
remapped_groups.append({**group, "tensors": sorted(names)})
|
| 239 |
precision["model"]["parameter_groups"] = remapped_groups
|
|
|
|
| 250 |
path = folder / name
|
| 251 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 252 |
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
| 253 |
+
plugin_code = '''"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
|
| 254 |
|
|
|
|
| 255 |
|
|
|
|
| 256 |
|
| 257 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
|
| 261 |
class _PaddedMarlin(tuple):
|
| 262 |
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
|
|
|
| 358 |
def register():
|
| 359 |
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 360 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 362 |
for name in ("repack", "repack_mxfp8"):
|
| 363 |
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
|
|
|
| 380 |
):
|
| 381 |
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
| 382 |
'''
|
| 383 |
+
(folder / "sglang_inkling2_tiny_kernels.py").write_text(plugin_code)
|
| 384 |
hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 385 |
|
| 386 |
from pathlib import Path
|
|
|
|
| 476 |
)
|
| 477 |
target_dtype = torch.float32 if keep_float32 else dtype
|
| 478 |
state[target] = weight.to(target_dtype).contiguous()
|
|
|
|
| 479 |
model.load_state_dict(state, strict=True, assign=True)
|
| 480 |
model.eval()
|
| 481 |
if device_map is not None:
|
| 482 |
model.to(device_map)
|
|
|
|
| 483 |
return model
|
| 484 |
'''
|
| 485 |
(folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
|
|
|
|
| 488 |
build-backend = "setuptools.build_meta"
|
| 489 |
|
| 490 |
[project]
|
| 491 |
+
name = "sglang-inkling2-tiny-kernels"
|
| 492 |
+
version = "0.2.0"
|
| 493 |
+
description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
|
| 494 |
requires-python = ">=3.10"
|
| 495 |
|
| 496 |
[project.entry-points."sglang.srt.plugins"]
|
| 497 |
+
inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
|
| 498 |
|
| 499 |
[tool.setuptools]
|
| 500 |
+
py-modules = ["sglang_inkling2_tiny_kernels"]
|
| 501 |
""")
|
| 502 |
return str(model)
|
| 503 |
|
|
|
|
| 506 |
create_model(
|
| 507 |
source_model_id="thinkingmachines/Inkling-2-Small",
|
| 508 |
source_revision="aac974122acd33ddb0d3e1f54433a9c5dd2249ce",
|
| 509 |
+
save_folder="/tmp/inkling-2",
|
| 510 |
)
|
.meta/env.json
CHANGED
|
@@ -15,43 +15,16 @@
|
|
| 15 |
},
|
| 16 |
"cuda": "12.6"
|
| 17 |
},
|
| 18 |
-
"
|
| 19 |
"python": "3.12.13",
|
| 20 |
"packages": {
|
| 21 |
-
"Pillow": "12.3.0",
|
| 22 |
-
"flashinfer-python": "0.7.4",
|
| 23 |
-
"huggingface_hub": "1.33.0",
|
| 24 |
-
"nvidia-cutlass-dsl": "4.8.0",
|
| 25 |
-
"safetensors": "0.8.0",
|
| 26 |
-
"sglang": "0.0.0.dev0",
|
| 27 |
-
"sglang-inkling2-tied": "0.1.0",
|
| 28 |
-
"sglang-kernel": "0.4.10",
|
| 29 |
-
"tiktoken": "0.14.0",
|
| 30 |
-
"tml-renderers": "0.4.1",
|
| 31 |
-
"tokenizers": "0.23.2",
|
| 32 |
-
"torch": "2.14.1+cu130",
|
| 33 |
-
"transformers": "5.19.0",
|
| 34 |
-
"triton": "3.8.0"
|
| 35 |
-
},
|
| 36 |
-
"cuda": "13.0",
|
| 37 |
-
"hardware": {
|
| 38 |
-
"gpu_model": "NVIDIA H20"
|
| 39 |
-
}
|
| 40 |
-
},
|
| 41 |
-
"transformers": {
|
| 42 |
-
"python": "3.12.13",
|
| 43 |
-
"packages": {
|
| 44 |
-
"Pillow": "12.3.0",
|
| 45 |
"huggingface_hub": "2.3.0",
|
| 46 |
"safetensors": "0.8.0",
|
| 47 |
"tokenizers": "0.23.3",
|
| 48 |
"torch": "2.13.0+cu126",
|
| 49 |
"transformers": "5.23.0.dev0"
|
| 50 |
},
|
| 51 |
-
"cuda": "12.6"
|
| 52 |
-
"hardware": {
|
| 53 |
-
"gpu_model": "NVIDIA H20"
|
| 54 |
-
}
|
| 55 |
}
|
| 56 |
}
|
| 57 |
}
|
|
|
|
| 15 |
},
|
| 16 |
"cuda": "12.6"
|
| 17 |
},
|
| 18 |
+
"transformers-reference": {
|
| 19 |
"python": "3.12.13",
|
| 20 |
"packages": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
"huggingface_hub": "2.3.0",
|
| 22 |
"safetensors": "0.8.0",
|
| 23 |
"tokenizers": "0.23.3",
|
| 24 |
"torch": "2.13.0+cu126",
|
| 25 |
"transformers": "5.23.0.dev0"
|
| 26 |
},
|
| 27 |
+
"cuda": "12.6"
|
|
|
|
|
|
|
|
|
|
| 28 |
}
|
| 29 |
}
|
| 30 |
}
|
README.md
CHANGED
|
@@ -9,13 +9,13 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
|
|
| 9 |
|
| 10 |
| Feature | Source | Tiny |
|
| 11 |
|---|---|---|
|
| 12 |
-
| Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 |
|
| 13 |
-
| Safetensors weight files (including MTP) | 393.48 GB (25 files) |
|
| 14 |
| Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
|
| 15 |
| Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
|
| 16 |
| Routed experts / experts per token | 384 / 8 | 96 / 8 |
|
| 17 |
| Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
|
| 18 |
-
| Vocabulary / tied embeddings | 201,024 / false | 201,024 /
|
| 19 |
| Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
|
| 20 |
| Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
|
| 21 |
| Vision width / layers | 1,024 / 23 | 32 / 4 |
|
|
@@ -24,12 +24,12 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
|
|
| 24 |
|
| 25 |
| File path | Size |
|
| 26 |
|------|------|
|
| 27 |
-
| model.safetensors |
|
| 28 |
| mtp.safetensors | 4.0MB |
|
| 29 |
|
| 30 |
### Example usage
|
| 31 |
|
| 32 |
-
#### Transformers (
|
| 33 |
|
| 34 |
Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
|
| 35 |
|
|
@@ -56,9 +56,9 @@ if __name__ == "__main__":
|
|
| 56 |
run_transformers(model_id="tiny-random/inkling-2")
|
| 57 |
```
|
| 58 |
|
| 59 |
-
#### SGLang serving (
|
| 60 |
|
| 61 |
-
Use the
|
| 62 |
|
| 63 |
```bash
|
| 64 |
SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
|
|
@@ -80,6 +80,6 @@ sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
|
|
| 80 |
|
| 81 |
### Change log
|
| 82 |
|
| 83 |
-
- 2026-11-
|
| 84 |
|
| 85 |
*This repo is generated by GPT-6.1-Sol*
|
|
|
|
| 9 |
|
| 10 |
| Feature | Source | Tiny |
|
| 11 |
|---|---|---|
|
| 12 |
+
| Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 16.40M BF16, 3.21K F32, 1.88M F8_E4M3, 940.03K U8 |
|
| 13 |
+
| Safetensors weight files (including MTP) | 393.48 GB (25 files) | 35.68 MB (2 files) |
|
| 14 |
| Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
|
| 15 |
| Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
|
| 16 |
| Routed experts / experts per token | 384 / 8 | 96 / 8 |
|
| 17 |
| Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
|
| 18 |
+
| Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
|
| 19 |
| Hidden / expert width | 4,608 / 2,304 | 32 / 32 |
|
| 20 |
| Q heads / KV heads / head dimension | 48 / 8 / 128 | 24 / 4 / 32 |
|
| 21 |
| Vision width / layers | 1,024 / 23 | 32 / 4 |
|
|
|
|
| 24 |
|
| 25 |
| File path | Size |
|
| 26 |
|------|------|
|
| 27 |
+
| model.safetensors | 31.7MB |
|
| 28 |
| mtp.safetensors | 4.0MB |
|
| 29 |
|
| 30 |
### Example usage
|
| 31 |
|
| 32 |
+
#### Transformers (unverified)
|
| 33 |
|
| 34 |
Install [Transformers with Inkling 2 support](https://github.com/huggingface/transformers/tree/c748111b3bb4d97df452f0fe60e63c480beb6fe6). The bundled loader expands packed expert weights to BF16.
|
| 35 |
|
|
|
|
| 56 |
run_transformers(model_id="tiny-random/inkling-2")
|
| 57 |
```
|
| 58 |
|
| 59 |
+
#### SGLang serving (unverified)
|
| 60 |
|
| 61 |
+
Use the pinned SGLang version and the bundled small-matrix kernel plugin.
|
| 62 |
|
| 63 |
```bash
|
| 64 |
SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
|
|
|
|
| 80 |
|
| 81 |
### Change log
|
| 82 |
|
| 83 |
+
- 2026-11-14: Updated tiny Inkling 2 with 96 mixed NVFP4/MXFP8 experts, independent token embeddings/output head, and BF16 MTP.
|
| 84 |
|
| 85 |
*This repo is generated by GPT-6.1-Sol*
|
config.json
CHANGED
|
@@ -54,7 +54,7 @@
|
|
| 54 |
"mlp_up_softcap": 25.0,
|
| 55 |
"route_scale": 8.0,
|
| 56 |
"use_gate_bias": true,
|
| 57 |
-
"tie_word_embeddings":
|
| 58 |
},
|
| 59 |
"vision_config": {
|
| 60 |
"temporal_patch_size": 2,
|
|
@@ -98,7 +98,7 @@
|
|
| 98 |
12
|
| 99 |
]
|
| 100 |
},
|
| 101 |
-
"tie_word_embeddings":
|
| 102 |
"auto_map": {
|
| 103 |
"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
|
| 104 |
}
|
|
|
|
| 54 |
"mlp_up_softcap": 25.0,
|
| 55 |
"route_scale": 8.0,
|
| 56 |
"use_gate_bias": true,
|
| 57 |
+
"tie_word_embeddings": false
|
| 58 |
},
|
| 59 |
"vision_config": {
|
| 60 |
"temporal_patch_size": 2,
|
|
|
|
| 98 |
12
|
| 99 |
]
|
| 100 |
},
|
| 101 |
+
"tie_word_embeddings": false,
|
| 102 |
"auto_map": {
|
| 103 |
"AutoModelForImageTextToText": "modeling_inkling2_tiny.Inkling2TinyForConditionalGeneration"
|
| 104 |
}
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6a3c70ce16c57bdfd587f0e6720c7e1e33384f4651cf4b14204ce71732a318ae
|
| 3 |
+
size 31674312
|
modeling_inkling2_tiny.py
CHANGED
|
@@ -93,10 +93,8 @@ class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
|
| 93 |
)
|
| 94 |
target_dtype = torch.float32 if keep_float32 else dtype
|
| 95 |
state[target] = weight.to(target_dtype).contiguous()
|
| 96 |
-
state["lm_head.weight"] = state["model.language_model.embed_tokens.weight"]
|
| 97 |
model.load_state_dict(state, strict=True, assign=True)
|
| 98 |
model.eval()
|
| 99 |
if device_map is not None:
|
| 100 |
model.to(device_map)
|
| 101 |
-
model.lm_head.weight = model.model.language_model.embed_tokens.weight
|
| 102 |
return model
|
|
|
|
| 93 |
)
|
| 94 |
target_dtype = torch.float32 if keep_float32 else dtype
|
| 95 |
state[target] = weight.to(target_dtype).contiguous()
|
|
|
|
| 96 |
model.load_state_dict(state, strict=True, assign=True)
|
| 97 |
model.eval()
|
| 98 |
if device_map is not None:
|
| 99 |
model.to(device_map)
|
|
|
|
| 100 |
return model
|
pyproject.toml
CHANGED
|
@@ -3,13 +3,13 @@ requires = ["setuptools>=68"]
|
|
| 3 |
build-backend = "setuptools.build_meta"
|
| 4 |
|
| 5 |
[project]
|
| 6 |
-
name = "sglang-inkling2-
|
| 7 |
-
version = "0.
|
| 8 |
-
description = "
|
| 9 |
requires-python = ">=3.10"
|
| 10 |
|
| 11 |
[project.entry-points."sglang.srt.plugins"]
|
| 12 |
-
|
| 13 |
|
| 14 |
[tool.setuptools]
|
| 15 |
-
py-modules = ["
|
|
|
|
| 3 |
build-backend = "setuptools.build_meta"
|
| 4 |
|
| 5 |
[project]
|
| 6 |
+
name = "sglang-inkling2-tiny-kernels"
|
| 7 |
+
version = "0.2.0"
|
| 8 |
+
description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
|
| 9 |
requires-python = ">=3.10"
|
| 10 |
|
| 11 |
[project.entry-points."sglang.srt.plugins"]
|
| 12 |
+
inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
|
| 13 |
|
| 14 |
[tool.setuptools]
|
| 15 |
+
py-modules = ["sglang_inkling2_tiny_kernels"]
|
sglang_inkling2_tiny_kernels.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: sglang-inkling2-tiny-kernels
|
| 3 |
+
Version: 0.2.0
|
| 4 |
+
Summary: Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint
|
| 5 |
+
Requires-Python: >=3.10
|
sglang_inkling2_tiny_kernels.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
README.md
|
| 2 |
+
pyproject.toml
|
| 3 |
+
sglang_inkling2_tiny_kernels.py
|
| 4 |
+
sglang_inkling2_tiny_kernels.egg-info/PKG-INFO
|
| 5 |
+
sglang_inkling2_tiny_kernels.egg-info/SOURCES.txt
|
| 6 |
+
sglang_inkling2_tiny_kernels.egg-info/dependency_links.txt
|
| 7 |
+
sglang_inkling2_tiny_kernels.egg-info/entry_points.txt
|
| 8 |
+
sglang_inkling2_tiny_kernels.egg-info/top_level.txt
|
sglang_inkling2_tiny_kernels.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
sglang_inkling2_tiny_kernels.egg-info/entry_points.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[sglang.srt.plugins]
|
| 2 |
+
inkling2_tiny_kernels = sglang_inkling2_tiny_kernels:register
|
sglang_inkling2_tiny_kernels.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
sglang_inkling2_tiny_kernels
|
sglang_inkling2_tied.py → sglang_inkling2_tiny_kernels.py
RENAMED
|
@@ -1,44 +1,10 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
-
import logging
|
| 4 |
|
| 5 |
-
logger = logging.getLogger(__name__)
|
| 6 |
|
| 7 |
|
| 8 |
-
def _config_init(original, config, *args, **kwargs):
|
| 9 |
-
tied = kwargs.get("tie_word_embeddings", False)
|
| 10 |
-
if tied:
|
| 11 |
-
kwargs = {**kwargs, "tie_word_embeddings": False}
|
| 12 |
-
original(config, *args, **kwargs)
|
| 13 |
-
if tied:
|
| 14 |
-
config.tie_word_embeddings = True
|
| 15 |
|
| 16 |
|
| 17 |
-
def _load_weights(original, model, weights):
|
| 18 |
-
if not model.text_config.tie_word_embeddings or model.llm is None:
|
| 19 |
-
return original(model, weights)
|
| 20 |
-
|
| 21 |
-
def with_head():
|
| 22 |
-
embedding = None
|
| 23 |
-
for name, tensor in weights:
|
| 24 |
-
if name == "model.llm.unembed.weight":
|
| 25 |
-
raise ValueError("A tied Inkling 2 checkpoint must omit unembed.weight")
|
| 26 |
-
if name == "model.llm.embed.weight":
|
| 27 |
-
embedding = tensor
|
| 28 |
-
yield name, tensor
|
| 29 |
-
if embedding is None:
|
| 30 |
-
raise ValueError("Tied Inkling 2 checkpoint has no embedding table")
|
| 31 |
-
yield "model.llm.unembed.weight", embedding
|
| 32 |
-
|
| 33 |
-
loaded = original(model, with_head())
|
| 34 |
-
embedding, head = model.llm.embed_tokens, model.llm.lm_head
|
| 35 |
-
if embedding.weight.shape != head.weight.shape or embedding.tp_size != head.tp_size:
|
| 36 |
-
raise ValueError("Tied embedding/head tensor-parallel partitions must match")
|
| 37 |
-
head.weight = embedding.weight
|
| 38 |
-
assert head.weight is embedding.weight
|
| 39 |
-
logger.info("Inkling 2 tied token tables active: one stored table and one shared parameter")
|
| 40 |
-
return loaded
|
| 41 |
-
|
| 42 |
|
| 43 |
class _PaddedMarlin(tuple):
|
| 44 |
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
|
@@ -140,12 +106,6 @@ def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
|
| 140 |
def register():
|
| 141 |
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 142 |
|
| 143 |
-
HookRegistry.register(
|
| 144 |
-
"sglang.srt.configs.inkling2.Inkling2ModelConfig.__init__", _config_init, HookType.AROUND
|
| 145 |
-
)
|
| 146 |
-
HookRegistry.register(
|
| 147 |
-
"sglang.srt.models.inkling2.Inkling2ForConditionalGeneration.load_weights", _load_weights, HookType.AROUND
|
| 148 |
-
)
|
| 149 |
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 150 |
for name in ("repack", "repack_mxfp8"):
|
| 151 |
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
|
|
|
| 1 |
+
"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
|
| 2 |
|
|
|
|
| 3 |
|
|
|
|
| 4 |
|
| 5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
class _PaddedMarlin(tuple):
|
| 10 |
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
|
|
|
| 106 |
def register():
|
| 107 |
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 110 |
for name in ("repack", "repack_mxfp8"):
|
| 111 |
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|