Preserve native mixed NVFP4/MXFP8 Inkling 2 without model-specific plugins
Browse files- .meta/create_model.py +16 -260
- .meta/env.json +17 -6
- .meta/model.log +74 -74
- README.md +13 -40
- config.json +12 -13
- model.safetensors +2 -2
- modeling_inkling2_tiny.py +0 -100
- mtp.safetensors +2 -2
- pyproject.toml +0 -15
- sglang_inkling2_tiny_kernels.py +0 -129
.meta/create_model.py
CHANGED
|
@@ -14,27 +14,32 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 14 |
precision = json.loads(
|
| 15 |
Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
|
| 16 |
)
|
| 17 |
-
# Two full 5-SWA/1-global cycles, sampling
|
| 18 |
source_layers = [*range(6), *range(42, 48)]
|
| 19 |
-
hidden_size =
|
| 20 |
-
intermediate_size =
|
| 21 |
text = raw["text_config"]
|
| 22 |
text.update(
|
| 23 |
hidden_size=hidden_size,
|
| 24 |
intermediate_size=intermediate_size,
|
| 25 |
num_hidden_layers=12,
|
| 26 |
-
num_attention_heads=
|
| 27 |
num_key_value_heads=4,
|
| 28 |
-
head_dim=
|
| 29 |
-
swa_num_attention_heads=
|
| 30 |
swa_num_key_value_heads=4,
|
| 31 |
-
swa_head_dim=
|
| 32 |
n_routed_experts=96,
|
| 33 |
tie_word_embeddings=False,
|
| 34 |
local_layer_ids=[i for i in range(12) if i % 6 != 5],
|
| 35 |
)
|
| 36 |
raw["tie_word_embeddings"] = False
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
raw["vision_config"].update(
|
| 39 |
hidden_size=32,
|
| 40 |
num_hidden_layers=4,
|
|
@@ -46,9 +51,9 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 46 |
mtp.update(
|
| 47 |
hidden_size=hidden_size,
|
| 48 |
intermediate_size=128,
|
| 49 |
-
num_attention_heads=
|
| 50 |
num_key_value_heads=4,
|
| 51 |
-
head_dim=
|
| 52 |
markov_rank=8,
|
| 53 |
target_layer_ids=[5, 12],
|
| 54 |
)
|
|
@@ -185,7 +190,7 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 185 |
draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
|
| 186 |
draft = {}
|
| 187 |
draft_module = torch.nn.Module()
|
| 188 |
-
replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144:
|
| 189 |
for name, tensor in sorted(draft_metadata.tensors.items()):
|
| 190 |
if name == "fuse_weight":
|
| 191 |
shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
|
|
@@ -250,255 +255,6 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
|
|
| 250 |
path = folder / name
|
| 251 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 252 |
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
| 253 |
-
plugin_code = '''"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
class _PaddedMarlin(tuple):
|
| 262 |
-
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
| 263 |
-
value = super().__new__(cls, packed)
|
| 264 |
-
value.input_width = input_width
|
| 265 |
-
value.output_width = output_width
|
| 266 |
-
value.padded_input = padded_input
|
| 267 |
-
value.padded_output = padded_output
|
| 268 |
-
return value
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
def _pad_matrix(tensor, rows, columns, fill=0):
|
| 272 |
-
output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
|
| 273 |
-
output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
|
| 274 |
-
return output
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
def _repack_marlin(original, weight, scales):
|
| 278 |
-
import torch
|
| 279 |
-
|
| 280 |
-
nvfp4 = weight.dtype == torch.uint8
|
| 281 |
-
packing = 2 if nvfp4 else 1
|
| 282 |
-
block = 16 if nvfp4 else 32
|
| 283 |
-
n, k = weight.shape[-2], weight.shape[-1] * packing
|
| 284 |
-
padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
|
| 285 |
-
if (padded_k, padded_n) == (k, n):
|
| 286 |
-
return original(weight, scales)
|
| 287 |
-
weight = _pad_matrix(weight, padded_n, padded_k // packing)
|
| 288 |
-
# Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
|
| 289 |
-
# Zero weight bytes make padded products zero in both precision families.
|
| 290 |
-
scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
|
| 291 |
-
packed = original(weight, scales)
|
| 292 |
-
return _PaddedMarlin(packed, k, n, padded_k, padded_n)
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
|
| 296 |
-
if not isinstance(packed, _PaddedMarlin):
|
| 297 |
-
return original(x, packed, out_size, *args, **kwargs)
|
| 298 |
-
if x.shape[-1] != packed.input_width or out_size != packed.output_width:
|
| 299 |
-
raise ValueError("Marlin padding metadata does not match the expert projection")
|
| 300 |
-
padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
|
| 301 |
-
padded[..., :x.shape[-1]].copy_(x)
|
| 302 |
-
if original.__name__ == "gemv" and packed.padded_input == 128:
|
| 303 |
-
# Eight K tiles fit one split of eight warps. The large-model heuristic
|
| 304 |
-
# otherwise chooses four splits/four warps for short decode batches.
|
| 305 |
-
kwargs = {**kwargs, "splits": 1, "warps": 8}
|
| 306 |
-
result = original(padded, packed, packed.padded_output, *args, **kwargs)
|
| 307 |
-
return result[..., :out_size].contiguous()
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
def _relative_fa4(original, *args, **kwargs):
|
| 311 |
-
q = args[0] if args else kwargs.get("q")
|
| 312 |
-
if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
|
| 313 |
-
and kwargs.get("tile_mn") is None):
|
| 314 |
-
# The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
|
| 315 |
-
# relative-bias kernel explicitly requires M=N=128. Use its tile API.
|
| 316 |
-
kwargs = {**kwargs, "tile_mn": (128, 128)}
|
| 317 |
-
return original(*args, **kwargs)
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
def _attn_res_config(original, width, *args, **kwargs):
|
| 321 |
-
threads, stages, vector, cluster = original(width, *args, **kwargs)
|
| 322 |
-
if width == 32:
|
| 323 |
-
# One compute warp covers the original 32 residual coordinates.
|
| 324 |
-
# Keep the same CuTe kernel, RMS normalization, and source weighting.
|
| 325 |
-
return 32, stages, 1, 1
|
| 326 |
-
return threads, stages, vector, cluster
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
def _small_all_reduce(original, x, group, **kwargs):
|
| 330 |
-
if x.ndim == 2 and x.shape[-1] == 32:
|
| 331 |
-
return group.all_reduce(x.clone())
|
| 332 |
-
return original(x, group, **kwargs)
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
def _small_multicast_input(original, group, shape, dtype):
|
| 336 |
-
if len(shape) == 2 and shape[-1] == 32:
|
| 337 |
-
import torch
|
| 338 |
-
|
| 339 |
-
return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
|
| 340 |
-
return original(group, shape, dtype)
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
def _small_multicast_reduce(original, x, group):
|
| 344 |
-
if x.ndim == 2 and x.shape[-1] == 32:
|
| 345 |
-
x.copy_(group.all_reduce(x))
|
| 346 |
-
return x
|
| 347 |
-
return original(x, group)
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
| 351 |
-
if cols == 32:
|
| 352 |
-
# Use the existing FP32 combine path instead of the GB300-tuned
|
| 353 |
-
# symmetric-memory one-shot collective for these tiny projections.
|
| 354 |
-
return None
|
| 355 |
-
return original(group, rows, cols, dtype, tag=tag)
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
def register():
|
| 359 |
-
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 360 |
-
|
| 361 |
-
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 362 |
-
for name in ("repack", "repack_mxfp8"):
|
| 363 |
-
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
| 364 |
-
for name in ("gemm", "gemv"):
|
| 365 |
-
HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
|
| 366 |
-
HookRegistry.register(
|
| 367 |
-
"sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
|
| 368 |
-
_relative_fa4, HookType.AROUND,
|
| 369 |
-
)
|
| 370 |
-
HookRegistry.register(
|
| 371 |
-
"sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
|
| 372 |
-
_attn_res_config, HookType.AROUND,
|
| 373 |
-
)
|
| 374 |
-
prefix = "sglang.kernels.ops.inkling2.comm"
|
| 375 |
-
for name, hook in (
|
| 376 |
-
("all_reduce", _small_all_reduce),
|
| 377 |
-
("multicast_all_reduce_input", _small_multicast_input),
|
| 378 |
-
("multicast_all_reduce", _small_multicast_reduce),
|
| 379 |
-
("oneshot_all_reduce_input", _small_oneshot_input),
|
| 380 |
-
):
|
| 381 |
-
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
| 382 |
-
'''
|
| 383 |
-
(folder / "sglang_inkling2_tiny_kernels.py").write_text(plugin_code)
|
| 384 |
-
hf_loader_code = '''"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 385 |
-
|
| 386 |
-
from pathlib import Path
|
| 387 |
-
|
| 388 |
-
import torch
|
| 389 |
-
from huggingface_hub import hf_hub_download
|
| 390 |
-
from safetensors.torch import load_file
|
| 391 |
-
from transformers import Inkling2Config, Inkling2ForConditionalGeneration
|
| 392 |
-
from transformers.conversion_mapping import get_model_conversion_mapping
|
| 393 |
-
from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
def dequantize(weight, scale, amax=None, denominator=None):
|
| 397 |
-
if weight.dtype == torch.uint8:
|
| 398 |
-
if amax is None or denominator is None:
|
| 399 |
-
raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
|
| 400 |
-
codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
|
| 401 |
-
levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
|
| 402 |
-
values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
|
| 403 |
-
return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
|
| 404 |
-
if weight.dtype == torch.float8_e4m3fn:
|
| 405 |
-
scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
|
| 406 |
-
return weight.float() * scales
|
| 407 |
-
raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
| 411 |
-
@classmethod
|
| 412 |
-
def from_pretrained(
|
| 413 |
-
cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
|
| 414 |
-
revision=None, cache_dir=None, local_files_only=False, token=None,
|
| 415 |
-
attn_implementation="eager", **kwargs,
|
| 416 |
-
):
|
| 417 |
-
if model_args:
|
| 418 |
-
raise ValueError("Inkling 2 does not take positional construction arguments")
|
| 419 |
-
options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
|
| 420 |
-
root = Path(pretrained_model_name_or_path)
|
| 421 |
-
|
| 422 |
-
def resolve(filename):
|
| 423 |
-
if root.is_dir():
|
| 424 |
-
return root / filename
|
| 425 |
-
return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
|
| 426 |
-
|
| 427 |
-
import json
|
| 428 |
-
|
| 429 |
-
raw = json.loads(resolve("config.json").read_text())
|
| 430 |
-
# The native config otherwise assumes the all-MXFP8 sibling. This
|
| 431 |
-
# adapter dequantizes each tensor according to its actual stored dtype.
|
| 432 |
-
raw.pop("quantization_config", None)
|
| 433 |
-
config = Inkling2Config(**raw)
|
| 434 |
-
config._attn_implementation = attn_implementation
|
| 435 |
-
dtype = kwargs.pop("torch_dtype", dtype)
|
| 436 |
-
if dtype in (None, "auto"):
|
| 437 |
-
dtype = torch.bfloat16
|
| 438 |
-
elif isinstance(dtype, str):
|
| 439 |
-
dtype = getattr(torch, dtype)
|
| 440 |
-
if dtype not in (torch.float32, torch.bfloat16, torch.float16):
|
| 441 |
-
raise ValueError("The Transformers adapter requires a floating compute dtype")
|
| 442 |
-
device_map = kwargs.pop("device_map", None)
|
| 443 |
-
if device_map == "auto":
|
| 444 |
-
device_map = "cuda" if torch.cuda.is_available() else "cpu"
|
| 445 |
-
if isinstance(device_map, dict):
|
| 446 |
-
if set(device_map) != {""}:
|
| 447 |
-
raise ValueError("Use a single device for this tiny Transformers adapter")
|
| 448 |
-
device_map = device_map[""]
|
| 449 |
-
allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
|
| 450 |
-
unexpected = set(kwargs) - allowed
|
| 451 |
-
if unexpected:
|
| 452 |
-
raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
|
| 453 |
-
model = Inkling2ForConditionalGeneration(config)
|
| 454 |
-
transforms = get_model_conversion_mapping(model)
|
| 455 |
-
renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
|
| 456 |
-
converters = [t for t in transforms if isinstance(t, WeightConverter)]
|
| 457 |
-
source = load_file(str(resolve("model.safetensors")))
|
| 458 |
-
state = {}
|
| 459 |
-
for name, weight in source.items():
|
| 460 |
-
if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
|
| 461 |
-
continue
|
| 462 |
-
if name + "_scale" in source:
|
| 463 |
-
weight = dequantize(
|
| 464 |
-
weight, source[name + "_scale"], source.get(name + "_amax"),
|
| 465 |
-
source.get(name + "_outer_scale_denominator"),
|
| 466 |
-
)
|
| 467 |
-
target, _ = rename_source_key(name, renamings, converters)
|
| 468 |
-
if name.endswith("w13_weight"):
|
| 469 |
-
# The source interleaves gate and up; Transformers chunks them.
|
| 470 |
-
e, rows, columns = weight.shape
|
| 471 |
-
weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
|
| 472 |
-
if target in state:
|
| 473 |
-
raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
|
| 474 |
-
keep_float32 = source[name].dtype == torch.float32 or any(
|
| 475 |
-
part in target for part in model._keep_in_fp32_modules_strict
|
| 476 |
-
)
|
| 477 |
-
target_dtype = torch.float32 if keep_float32 else dtype
|
| 478 |
-
state[target] = weight.to(target_dtype).contiguous()
|
| 479 |
-
model.load_state_dict(state, strict=True, assign=True)
|
| 480 |
-
model.eval()
|
| 481 |
-
if device_map is not None:
|
| 482 |
-
model.to(device_map)
|
| 483 |
-
return model
|
| 484 |
-
'''
|
| 485 |
-
(folder / "modeling_inkling2_tiny.py").write_text(hf_loader_code)
|
| 486 |
-
(folder / "pyproject.toml").write_text("""[build-system]
|
| 487 |
-
requires = ["setuptools>=68"]
|
| 488 |
-
build-backend = "setuptools.build_meta"
|
| 489 |
-
|
| 490 |
-
[project]
|
| 491 |
-
name = "sglang-inkling2-tiny-kernels"
|
| 492 |
-
version = "0.2.0"
|
| 493 |
-
description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
|
| 494 |
-
requires-python = ">=3.10"
|
| 495 |
-
|
| 496 |
-
[project.entry-points."sglang.srt.plugins"]
|
| 497 |
-
inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
|
| 498 |
-
|
| 499 |
-
[tool.setuptools]
|
| 500 |
-
py-modules = ["sglang_inkling2_tiny_kernels"]
|
| 501 |
-
""")
|
| 502 |
return str(model)
|
| 503 |
|
| 504 |
|
|
|
|
| 14 |
precision = json.loads(
|
| 15 |
Path(hf_hub_download(source_model_id, "hf_quant_config.json", revision=source_revision)).read_text()
|
| 16 |
)
|
| 17 |
+
# Two full 5-SWA/1-global cycles, sampling early and late source layers.
|
| 18 |
source_layers = [*range(6), *range(42, 48)]
|
| 19 |
+
hidden_size = 128
|
| 20 |
+
intermediate_size = 128
|
| 21 |
text = raw["text_config"]
|
| 22 |
text.update(
|
| 23 |
hidden_size=hidden_size,
|
| 24 |
intermediate_size=intermediate_size,
|
| 25 |
num_hidden_layers=12,
|
| 26 |
+
num_attention_heads=8,
|
| 27 |
num_key_value_heads=4,
|
| 28 |
+
head_dim=128,
|
| 29 |
+
swa_num_attention_heads=8,
|
| 30 |
swa_num_key_value_heads=4,
|
| 31 |
+
swa_head_dim=128,
|
| 32 |
n_routed_experts=96,
|
| 33 |
tie_word_embeddings=False,
|
| 34 |
local_layer_ids=[i for i in range(12) if i % 6 != 5],
|
| 35 |
)
|
| 36 |
raw["tie_word_embeddings"] = False
|
| 37 |
+
import transformers
|
| 38 |
+
|
| 39 |
+
raw.pop("auto_map", None)
|
| 40 |
+
raw.pop("quantization_config", None)
|
| 41 |
+
raw["transformers_version"] = transformers.__version__
|
| 42 |
+
raw["dtype"] = "bfloat16"
|
| 43 |
raw["vision_config"].update(
|
| 44 |
hidden_size=32,
|
| 45 |
num_hidden_layers=4,
|
|
|
|
| 51 |
mtp.update(
|
| 52 |
hidden_size=hidden_size,
|
| 53 |
intermediate_size=128,
|
| 54 |
+
num_attention_heads=8,
|
| 55 |
num_key_value_heads=4,
|
| 56 |
+
head_dim=128,
|
| 57 |
markov_rank=8,
|
| 58 |
target_layer_ids=[5, 12],
|
| 59 |
)
|
|
|
|
| 190 |
draft_metadata = parse_safetensors_file_metadata(source_model_id, "mtp.safetensors", revision=source_revision)
|
| 191 |
draft = {}
|
| 192 |
draft_module = torch.nn.Module()
|
| 193 |
+
replacements = {4608: hidden_size, 18432: mtp["intermediate_size"], 6144: 8 * 128, 1024: 4 * 128, 128: 128}
|
| 194 |
for name, tensor in sorted(draft_metadata.tensors.items()):
|
| 195 |
if name == "fuse_weight":
|
| 196 |
shape = (mtp["num_hidden_layers"] + 1, len(mtp["target_layer_ids"]))
|
|
|
|
| 255 |
path = folder / name
|
| 256 |
path.parent.mkdir(parents=True, exist_ok=True)
|
| 257 |
path.write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
return str(model)
|
| 259 |
|
| 260 |
|
.meta/env.json
CHANGED
|
@@ -15,16 +15,27 @@
|
|
| 15 |
},
|
| 16 |
"cuda": "12.6"
|
| 17 |
},
|
| 18 |
-
"
|
| 19 |
"python": "3.12.13",
|
| 20 |
"packages": {
|
| 21 |
-
"
|
|
|
|
|
|
|
|
|
|
| 22 |
"safetensors": "0.8.0",
|
| 23 |
-
"
|
| 24 |
-
"
|
| 25 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
},
|
| 27 |
-
"cuda": "
|
|
|
|
|
|
|
|
|
|
| 28 |
}
|
| 29 |
}
|
| 30 |
}
|
|
|
|
| 15 |
},
|
| 16 |
"cuda": "12.6"
|
| 17 |
},
|
| 18 |
+
"sglang": {
|
| 19 |
"python": "3.12.13",
|
| 20 |
"packages": {
|
| 21 |
+
"Pillow": "12.3.0",
|
| 22 |
+
"flashinfer-python": "0.7.4",
|
| 23 |
+
"huggingface_hub": "1.33.0",
|
| 24 |
+
"nvidia-cutlass-dsl": "4.8.0",
|
| 25 |
"safetensors": "0.8.0",
|
| 26 |
+
"sglang": "0.0.0.dev0",
|
| 27 |
+
"sglang-kernel": "0.4.10",
|
| 28 |
+
"tiktoken": "0.14.0",
|
| 29 |
+
"tml-renderers": "0.4.1",
|
| 30 |
+
"tokenizers": "0.23.2",
|
| 31 |
+
"torch": "2.14.1+cu130",
|
| 32 |
+
"transformers": "5.19.0",
|
| 33 |
+
"triton": "3.8.0"
|
| 34 |
},
|
| 35 |
+
"cuda": "13.0",
|
| 36 |
+
"hardware": {
|
| 37 |
+
"gpu_model": "NVIDIA H20"
|
| 38 |
+
}
|
| 39 |
}
|
| 40 |
}
|
| 41 |
}
|
.meta/model.log
CHANGED
|
@@ -2,8 +2,8 @@ Inkling2ForConditionalGeneration(
|
|
| 2 |
(model): Inkling2Model(
|
| 3 |
(language_model): Inkling2TextModel(
|
| 4 |
(embed_tokens): Inkling2NormedEmbedding(
|
| 5 |
-
201024,
|
| 6 |
-
(embed_norm): Inkling2RMSNorm((
|
| 7 |
)
|
| 8 |
(layers): ModuleList(
|
| 9 |
(0): Inkling2SlidingDecoderLayer(
|
|
@@ -11,20 +11,20 @@ Inkling2ForConditionalGeneration(
|
|
| 11 |
(norm): Inkling2UnweightedRMSNorm()
|
| 12 |
)
|
| 13 |
(self_attn): Inkling2SlidingAttention(
|
| 14 |
-
(q_proj): Linear(in_features=
|
| 15 |
-
(k_proj): Linear(in_features=
|
| 16 |
-
(v_proj): Linear(in_features=
|
| 17 |
-
(r_proj): Linear(in_features=
|
| 18 |
-
(o_proj): Linear(in_features=
|
| 19 |
(k_sconv): Inkling2ShortConvolution(
|
| 20 |
-
(conv1d): Conv1d(
|
| 21 |
)
|
| 22 |
(v_sconv): Inkling2ShortConvolution(
|
| 23 |
-
(conv1d): Conv1d(
|
| 24 |
)
|
| 25 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 26 |
-
(gate_proj): Linear(in_features=
|
| 27 |
-
(sink_proj): Linear(in_features=
|
| 28 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 29 |
(k_l2_clip): Inkling2L2Clip()
|
| 30 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
@@ -41,13 +41,13 @@ Inkling2ForConditionalGeneration(
|
|
| 41 |
(score_fn): Sigmoid()
|
| 42 |
)
|
| 43 |
)
|
| 44 |
-
(input_layernorm): Inkling2RMSNorm((
|
| 45 |
-
(post_attention_layernorm): Inkling2RMSNorm((
|
| 46 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 47 |
-
(conv1d): Conv1d(
|
| 48 |
)
|
| 49 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 50 |
-
(conv1d): Conv1d(
|
| 51 |
)
|
| 52 |
)
|
| 53 |
(1-4): 4 x Inkling2SlidingDecoderLayer(
|
|
@@ -58,20 +58,20 @@ Inkling2ForConditionalGeneration(
|
|
| 58 |
(norm): Inkling2UnweightedRMSNorm()
|
| 59 |
)
|
| 60 |
(self_attn): Inkling2SlidingAttention(
|
| 61 |
-
(q_proj): Linear(in_features=
|
| 62 |
-
(k_proj): Linear(in_features=
|
| 63 |
-
(v_proj): Linear(in_features=
|
| 64 |
-
(r_proj): Linear(in_features=
|
| 65 |
-
(o_proj): Linear(in_features=
|
| 66 |
(k_sconv): Inkling2ShortConvolution(
|
| 67 |
-
(conv1d): Conv1d(
|
| 68 |
)
|
| 69 |
(v_sconv): Inkling2ShortConvolution(
|
| 70 |
-
(conv1d): Conv1d(
|
| 71 |
)
|
| 72 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 73 |
-
(gate_proj): Linear(in_features=
|
| 74 |
-
(sink_proj): Linear(in_features=
|
| 75 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 76 |
(k_l2_clip): Inkling2L2Clip()
|
| 77 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
@@ -88,13 +88,13 @@ Inkling2ForConditionalGeneration(
|
|
| 88 |
(score_fn): Sigmoid()
|
| 89 |
)
|
| 90 |
)
|
| 91 |
-
(input_layernorm): Inkling2RMSNorm((
|
| 92 |
-
(post_attention_layernorm): Inkling2RMSNorm((
|
| 93 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 94 |
-
(conv1d): Conv1d(
|
| 95 |
)
|
| 96 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 97 |
-
(conv1d): Conv1d(
|
| 98 |
)
|
| 99 |
)
|
| 100 |
(5): Inkling2FullDecoderLayer(
|
|
@@ -105,20 +105,20 @@ Inkling2ForConditionalGeneration(
|
|
| 105 |
(norm): Inkling2UnweightedRMSNorm()
|
| 106 |
)
|
| 107 |
(self_attn): Inkling2FullAttention(
|
| 108 |
-
(q_proj): Linear(in_features=
|
| 109 |
-
(k_proj): Linear(in_features=
|
| 110 |
-
(v_proj): Linear(in_features=
|
| 111 |
-
(r_proj): Linear(in_features=
|
| 112 |
-
(o_proj): Linear(in_features=
|
| 113 |
(k_sconv): Inkling2DynamicShortConvolution(
|
| 114 |
-
(weight_proj): Linear(in_features=
|
| 115 |
)
|
| 116 |
(v_sconv): Inkling2DynamicShortConvolution(
|
| 117 |
-
(weight_proj): Linear(in_features=
|
| 118 |
)
|
| 119 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 120 |
-
(gate_proj): Linear(in_features=
|
| 121 |
-
(sink_proj): Linear(in_features=
|
| 122 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 123 |
(k_l2_clip): Inkling2L2Clip()
|
| 124 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
@@ -135,15 +135,15 @@ Inkling2ForConditionalGeneration(
|
|
| 135 |
(score_fn): Sigmoid()
|
| 136 |
)
|
| 137 |
)
|
| 138 |
-
(q_input_layernorm): Inkling2RMSNorm((
|
| 139 |
-
(k_input_layernorm): Inkling2RMSNorm((
|
| 140 |
-
(v_input_layernorm): Inkling2RMSNorm((
|
| 141 |
-
(post_attention_layernorm): Inkling2RMSNorm((
|
| 142 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 143 |
-
(conv1d): Conv1d(
|
| 144 |
)
|
| 145 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 146 |
-
(conv1d): Conv1d(
|
| 147 |
)
|
| 148 |
)
|
| 149 |
(6-10): 5 x Inkling2SlidingDecoderLayer(
|
|
@@ -154,20 +154,20 @@ Inkling2ForConditionalGeneration(
|
|
| 154 |
(norm): Inkling2UnweightedRMSNorm()
|
| 155 |
)
|
| 156 |
(self_attn): Inkling2SlidingAttention(
|
| 157 |
-
(q_proj): Linear(in_features=
|
| 158 |
-
(k_proj): Linear(in_features=
|
| 159 |
-
(v_proj): Linear(in_features=
|
| 160 |
-
(r_proj): Linear(in_features=
|
| 161 |
-
(o_proj): Linear(in_features=
|
| 162 |
(k_sconv): Inkling2ShortConvolution(
|
| 163 |
-
(conv1d): Conv1d(
|
| 164 |
)
|
| 165 |
(v_sconv): Inkling2ShortConvolution(
|
| 166 |
-
(conv1d): Conv1d(
|
| 167 |
)
|
| 168 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 169 |
-
(gate_proj): Linear(in_features=
|
| 170 |
-
(sink_proj): Linear(in_features=
|
| 171 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 172 |
(k_l2_clip): Inkling2L2Clip()
|
| 173 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
@@ -184,13 +184,13 @@ Inkling2ForConditionalGeneration(
|
|
| 184 |
(score_fn): Sigmoid()
|
| 185 |
)
|
| 186 |
)
|
| 187 |
-
(input_layernorm): Inkling2RMSNorm((
|
| 188 |
-
(post_attention_layernorm): Inkling2RMSNorm((
|
| 189 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 190 |
-
(conv1d): Conv1d(
|
| 191 |
)
|
| 192 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 193 |
-
(conv1d): Conv1d(
|
| 194 |
)
|
| 195 |
)
|
| 196 |
(11): Inkling2FullDecoderLayer(
|
|
@@ -201,20 +201,20 @@ Inkling2ForConditionalGeneration(
|
|
| 201 |
(norm): Inkling2UnweightedRMSNorm()
|
| 202 |
)
|
| 203 |
(self_attn): Inkling2FullAttention(
|
| 204 |
-
(q_proj): Linear(in_features=
|
| 205 |
-
(k_proj): Linear(in_features=
|
| 206 |
-
(v_proj): Linear(in_features=
|
| 207 |
-
(r_proj): Linear(in_features=
|
| 208 |
-
(o_proj): Linear(in_features=
|
| 209 |
(k_sconv): Inkling2DynamicShortConvolution(
|
| 210 |
-
(weight_proj): Linear(in_features=
|
| 211 |
)
|
| 212 |
(v_sconv): Inkling2DynamicShortConvolution(
|
| 213 |
-
(weight_proj): Linear(in_features=
|
| 214 |
)
|
| 215 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 216 |
-
(gate_proj): Linear(in_features=
|
| 217 |
-
(sink_proj): Linear(in_features=
|
| 218 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 219 |
(k_l2_clip): Inkling2L2Clip()
|
| 220 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
@@ -231,19 +231,19 @@ Inkling2ForConditionalGeneration(
|
|
| 231 |
(score_fn): Sigmoid()
|
| 232 |
)
|
| 233 |
)
|
| 234 |
-
(q_input_layernorm): Inkling2RMSNorm((
|
| 235 |
-
(k_input_layernorm): Inkling2RMSNorm((
|
| 236 |
-
(v_input_layernorm): Inkling2RMSNorm((
|
| 237 |
-
(post_attention_layernorm): Inkling2RMSNorm((
|
| 238 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 239 |
-
(conv1d): Conv1d(
|
| 240 |
)
|
| 241 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 242 |
-
(conv1d): Conv1d(
|
| 243 |
)
|
| 244 |
)
|
| 245 |
)
|
| 246 |
-
(norm): Inkling2RMSNorm((
|
| 247 |
(final_input): Inkling2BlockAttentionResidual(
|
| 248 |
(norm): Inkling2UnweightedRMSNorm()
|
| 249 |
)
|
|
@@ -275,11 +275,11 @@ Inkling2ForConditionalGeneration(
|
|
| 275 |
(merger): Inkling2VisionMerger(
|
| 276 |
(linear_1): Linear(in_features=128, out_features=32, bias=False)
|
| 277 |
(act): GELUActivation()
|
| 278 |
-
(linear_2): Linear(in_features=32, out_features=
|
| 279 |
)
|
| 280 |
)
|
| 281 |
)
|
| 282 |
-
(lm_head): Linear(in_features=
|
| 283 |
(logit_softcap): Inkling2Softcap()
|
| 284 |
(mtp): Module(
|
| 285 |
(layers): Module(
|
|
|
|
| 2 |
(model): Inkling2Model(
|
| 3 |
(language_model): Inkling2TextModel(
|
| 4 |
(embed_tokens): Inkling2NormedEmbedding(
|
| 5 |
+
201024, 128
|
| 6 |
+
(embed_norm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 7 |
)
|
| 8 |
(layers): ModuleList(
|
| 9 |
(0): Inkling2SlidingDecoderLayer(
|
|
|
|
| 11 |
(norm): Inkling2UnweightedRMSNorm()
|
| 12 |
)
|
| 13 |
(self_attn): Inkling2SlidingAttention(
|
| 14 |
+
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 15 |
+
(k_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 16 |
+
(v_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 17 |
+
(r_proj): Linear(in_features=128, out_features=128, bias=False)
|
| 18 |
+
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
|
| 19 |
(k_sconv): Inkling2ShortConvolution(
|
| 20 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 21 |
)
|
| 22 |
(v_sconv): Inkling2ShortConvolution(
|
| 23 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 24 |
)
|
| 25 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 26 |
+
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 27 |
+
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
|
| 28 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 29 |
(k_l2_clip): Inkling2L2Clip()
|
| 30 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
|
|
| 41 |
(score_fn): Sigmoid()
|
| 42 |
)
|
| 43 |
)
|
| 44 |
+
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 45 |
+
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 46 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 47 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 48 |
)
|
| 49 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 50 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 51 |
)
|
| 52 |
)
|
| 53 |
(1-4): 4 x Inkling2SlidingDecoderLayer(
|
|
|
|
| 58 |
(norm): Inkling2UnweightedRMSNorm()
|
| 59 |
)
|
| 60 |
(self_attn): Inkling2SlidingAttention(
|
| 61 |
+
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 62 |
+
(k_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 63 |
+
(v_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 64 |
+
(r_proj): Linear(in_features=128, out_features=128, bias=False)
|
| 65 |
+
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
|
| 66 |
(k_sconv): Inkling2ShortConvolution(
|
| 67 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 68 |
)
|
| 69 |
(v_sconv): Inkling2ShortConvolution(
|
| 70 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 71 |
)
|
| 72 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 73 |
+
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 74 |
+
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
|
| 75 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 76 |
(k_l2_clip): Inkling2L2Clip()
|
| 77 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
|
|
| 88 |
(score_fn): Sigmoid()
|
| 89 |
)
|
| 90 |
)
|
| 91 |
+
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 92 |
+
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 93 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 94 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 95 |
)
|
| 96 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 97 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 98 |
)
|
| 99 |
)
|
| 100 |
(5): Inkling2FullDecoderLayer(
|
|
|
|
| 105 |
(norm): Inkling2UnweightedRMSNorm()
|
| 106 |
)
|
| 107 |
(self_attn): Inkling2FullAttention(
|
| 108 |
+
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 109 |
+
(k_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 110 |
+
(v_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 111 |
+
(r_proj): Linear(in_features=128, out_features=128, bias=False)
|
| 112 |
+
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
|
| 113 |
(k_sconv): Inkling2DynamicShortConvolution(
|
| 114 |
+
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
|
| 115 |
)
|
| 116 |
(v_sconv): Inkling2DynamicShortConvolution(
|
| 117 |
+
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
|
| 118 |
)
|
| 119 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 120 |
+
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 121 |
+
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
|
| 122 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 123 |
(k_l2_clip): Inkling2L2Clip()
|
| 124 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
|
|
| 135 |
(score_fn): Sigmoid()
|
| 136 |
)
|
| 137 |
)
|
| 138 |
+
(q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 139 |
+
(k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 140 |
+
(v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 141 |
+
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 142 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 143 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 144 |
)
|
| 145 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 146 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 147 |
)
|
| 148 |
)
|
| 149 |
(6-10): 5 x Inkling2SlidingDecoderLayer(
|
|
|
|
| 154 |
(norm): Inkling2UnweightedRMSNorm()
|
| 155 |
)
|
| 156 |
(self_attn): Inkling2SlidingAttention(
|
| 157 |
+
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 158 |
+
(k_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 159 |
+
(v_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 160 |
+
(r_proj): Linear(in_features=128, out_features=128, bias=False)
|
| 161 |
+
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
|
| 162 |
(k_sconv): Inkling2ShortConvolution(
|
| 163 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 164 |
)
|
| 165 |
(v_sconv): Inkling2ShortConvolution(
|
| 166 |
+
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
|
| 167 |
)
|
| 168 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 169 |
+
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 170 |
+
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
|
| 171 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 172 |
(k_l2_clip): Inkling2L2Clip()
|
| 173 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
|
|
| 184 |
(score_fn): Sigmoid()
|
| 185 |
)
|
| 186 |
)
|
| 187 |
+
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 188 |
+
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 189 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 190 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 191 |
)
|
| 192 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 193 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 194 |
)
|
| 195 |
)
|
| 196 |
(11): Inkling2FullDecoderLayer(
|
|
|
|
| 201 |
(norm): Inkling2UnweightedRMSNorm()
|
| 202 |
)
|
| 203 |
(self_attn): Inkling2FullAttention(
|
| 204 |
+
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 205 |
+
(k_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 206 |
+
(v_proj): Linear(in_features=128, out_features=512, bias=False)
|
| 207 |
+
(r_proj): Linear(in_features=128, out_features=128, bias=False)
|
| 208 |
+
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
|
| 209 |
(k_sconv): Inkling2DynamicShortConvolution(
|
| 210 |
+
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
|
| 211 |
)
|
| 212 |
(v_sconv): Inkling2DynamicShortConvolution(
|
| 213 |
+
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
|
| 214 |
)
|
| 215 |
(rel_logits_proj): Inkling2RelativeLogits()
|
| 216 |
+
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
|
| 217 |
+
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
|
| 218 |
(q_l2_clip): Inkling2ScaledL2Clip()
|
| 219 |
(k_l2_clip): Inkling2L2Clip()
|
| 220 |
(rel_l2_clip): Inkling2ScaledL2Clip()
|
|
|
|
| 231 |
(score_fn): Sigmoid()
|
| 232 |
)
|
| 233 |
)
|
| 234 |
+
(q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 235 |
+
(k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 236 |
+
(v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 237 |
+
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 238 |
(post_attention_convolution): Inkling2ShortConvolution(
|
| 239 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 240 |
)
|
| 241 |
(post_mlp_convolution): Inkling2ShortConvolution(
|
| 242 |
+
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
|
| 243 |
)
|
| 244 |
)
|
| 245 |
)
|
| 246 |
+
(norm): Inkling2RMSNorm((128,), eps=1e-06)
|
| 247 |
(final_input): Inkling2BlockAttentionResidual(
|
| 248 |
(norm): Inkling2UnweightedRMSNorm()
|
| 249 |
)
|
|
|
|
| 275 |
(merger): Inkling2VisionMerger(
|
| 276 |
(linear_1): Linear(in_features=128, out_features=32, bias=False)
|
| 277 |
(act): GELUActivation()
|
| 278 |
+
(linear_2): Linear(in_features=32, out_features=128, bias=False)
|
| 279 |
)
|
| 280 |
)
|
| 281 |
)
|
| 282 |
+
(lm_head): Linear(in_features=128, out_features=201024, bias=False)
|
| 283 |
(logit_softcap): Inkling2Softcap()
|
| 284 |
(mtp): Module(
|
| 285 |
(layers): Module(
|
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
library_name:
|
| 3 |
pipeline_tag: image-text-to-text
|
| 4 |
base_model:
|
| 5 |
- thinkingmachines/Inkling-2-Small
|
|
@@ -9,66 +9,39 @@ This is a randomly initialized tiny model based on [thinkingmachines/Inkling-2-S
|
|
| 9 |
|
| 10 |
| Feature | Source | Tiny |
|
| 11 |
|---|---|---|
|
| 12 |
-
| Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 |
|
| 13 |
-
| Safetensors weight files (including MTP) | 393.48 GB (25 files) |
|
| 14 |
| Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
|
| 15 |
| Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
|
| 16 |
| Routed experts / experts per token | 384 / 8 | 96 / 8 |
|
| 17 |
| Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
|
| 18 |
| Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
|
| 19 |
-
| Hidden / expert width | 4,608 / 2,304 |
|
| 20 |
-
| Q heads / KV heads / head dimension | 48 / 8 / 128 |
|
| 21 |
| Vision width / layers | 1,024 / 23 | 32 / 4 |
|
| 22 |
| MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
|
| 23 |
| MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
|
| 24 |
|
| 25 |
| File path | Size |
|
| 26 |
|------|------|
|
| 27 |
-
| model.safetensors |
|
| 28 |
-
| mtp.safetensors |
|
| 29 |
|
| 30 |
### Example usage
|
| 31 |
|
| 32 |
-
####
|
| 33 |
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
```python
|
| 37 |
-
def run_transformers(model_id: str):
|
| 38 |
-
import torch
|
| 39 |
-
from PIL import Image
|
| 40 |
-
from transformers import AutoModelForImageTextToText, AutoProcessor
|
| 41 |
-
|
| 42 |
-
processor = AutoProcessor.from_pretrained(model_id)
|
| 43 |
-
model = AutoModelForImageTextToText.from_pretrained(
|
| 44 |
-
model_id, trust_remote_code=True, dtype=torch.bfloat16, device_map="cuda"
|
| 45 |
-
)
|
| 46 |
-
image = Image.new("RGB", (80, 80), (80, 120, 160))
|
| 47 |
-
inputs = processor(text=f"{processor.image_token}Describe this image.", images=image, return_tensors="pt").to(
|
| 48 |
-
model.device
|
| 49 |
-
)
|
| 50 |
-
with torch.inference_mode():
|
| 51 |
-
output = model.generate(**inputs, max_new_tokens=10, min_new_tokens=10, do_sample=False)
|
| 52 |
-
print(processor.batch_decode(output[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
if __name__ == "__main__":
|
| 56 |
-
run_transformers(model_id="tiny-random/inkling-2")
|
| 57 |
-
```
|
| 58 |
-
|
| 59 |
-
#### SGLang serving (unverified)
|
| 60 |
-
|
| 61 |
-
Use the pinned SGLang version and the bundled small-matrix kernel plugin.
|
| 62 |
|
| 63 |
```bash
|
| 64 |
SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
|
| 65 |
hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
|
| 66 |
-
|
| 67 |
-
sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
|
| 68 |
--dtype bfloat16 --attention-backend fa4 --enable-multimodal \
|
| 69 |
--context-length 2048 --mem-fraction-static 0.2 \
|
| 70 |
--disable-custom-all-reduce \
|
| 71 |
-
--
|
|
|
|
| 72 |
--reasoning-parser inkling --tool-call-parser inkling2
|
| 73 |
```
|
| 74 |
|
|
@@ -80,6 +53,6 @@ sglang serve --model-path ./inkling-2-tiny --tp 4 --ep 4 \
|
|
| 80 |
|
| 81 |
### Change log
|
| 82 |
|
| 83 |
-
- 2026-11-14:
|
| 84 |
|
| 85 |
*This repo is generated by GPT-6.1-Sol*
|
|
|
|
| 1 |
---
|
| 2 |
+
library_name: sglang
|
| 3 |
pipeline_tag: image-text-to-text
|
| 4 |
base_model:
|
| 5 |
- thinkingmachines/Inkling-2-Small
|
|
|
|
| 9 |
|
| 10 |
| Feature | Source | Tiny |
|
| 11 |
|---|---|---|
|
| 12 |
+
| Checkpoint tensor elements (including packed weights/scales) | 8.69B BF16, 565.33K F32, 128.42B F8_E4M3, 247.67B U8 | 63.01M BF16, 5.90K F32, 30.08M F8_E4M3, 15.04M U8 |
|
| 13 |
+
| Safetensors weight files (including MTP) | 393.48 GB (25 files) | 171.22 MB (2 files) |
|
| 14 |
| Decoder layers / attention residual block | 48 / 6 | 12 / 6 |
|
| 15 |
| Attention cycle | 5 SWA + 1 global | Same; two complete cycles |
|
| 16 |
| Routed experts / experts per token | 384 / 8 | 96 / 8 |
|
| 17 |
| Expert precision | 40 NVFP4 + 8 MXFP8 layers | 6 NVFP4 + 6 MXFP8 layers |
|
| 18 |
| Vocabulary / tied embeddings | 201,024 / false | 201,024 / false |
|
| 19 |
+
| Hidden / expert width | 4,608 / 2,304 | 128 / 128 |
|
| 20 |
+
| Q heads / KV heads / head dimension | 48 / 8 / 128 | 8 / 4 / 128 |
|
| 21 |
| Vision width / layers | 1,024 / 23 | 32 / 4 |
|
| 22 |
| MTP layers / block size / Markov rank | 5 / 8 / 256 | 5 / 8 / 8 |
|
| 23 |
| MTP feature taps | 5,11,17,23,29,35,41,48 | 5,12 (final norm sentinel) |
|
| 24 |
|
| 25 |
| File path | Size |
|
| 26 |
|------|------|
|
| 27 |
+
| model.safetensors | 162.2MB |
|
| 28 |
+
| mtp.safetensors | 9.0MB |
|
| 29 |
|
| 30 |
### Example usage
|
| 31 |
|
| 32 |
+
#### SGLang serving (✅ verified)
|
| 33 |
|
| 34 |
+
Use SGLang with Inkling 2 support. No model-specific package is required.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
```bash
|
| 37 |
SGLANG_BUILD_RUST_EXTS=none pip install 'git+https://github.com/sgl-project/sglang.git@f726172d760dea737c7f1e797c9f32efe14d1685#subdirectory=python'
|
| 38 |
hf download tiny-random/inkling-2 --local-dir inkling-2-tiny
|
| 39 |
+
sglang serve --model-path ./inkling-2-tiny --tp 2 --ep 2 \
|
|
|
|
| 40 |
--dtype bfloat16 --attention-backend fa4 --enable-multimodal \
|
| 41 |
--context-length 2048 --mem-fraction-static 0.2 \
|
| 42 |
--disable-custom-all-reduce \
|
| 43 |
+
--skip-server-warmup \
|
| 44 |
+
--cuda-graph-config '{"decode": {"bs": [16]}, "prefill": {"bs": [128, 2048]}}' \
|
| 45 |
--reasoning-parser inkling --tool-call-parser inkling2
|
| 46 |
```
|
| 47 |
|
|
|
|
| 53 |
|
| 54 |
### Change log
|
| 55 |
|
| 56 |
+
- 2026-11-14: Preserve source mixed NVFP4/MXFP8 expert precision, with 96 experts and independent token tables.
|
| 57 |
|
| 58 |
*This repo is generated by GPT-6.1-Sol*
|
config.json
CHANGED
|
@@ -7,12 +7,12 @@
|
|
| 7 |
"text_config": {
|
| 8 |
"model_max_length": 1048576,
|
| 9 |
"torch_dtype": "bfloat16",
|
| 10 |
-
"hidden_size":
|
| 11 |
"num_hidden_layers": 12,
|
| 12 |
"vocab_size": 201024,
|
| 13 |
-
"num_attention_heads":
|
| 14 |
"num_key_value_heads": 4,
|
| 15 |
-
"head_dim":
|
| 16 |
"d_rel": 16,
|
| 17 |
"rel_extent": 1024,
|
| 18 |
"q_bias": false,
|
|
@@ -40,8 +40,8 @@
|
|
| 40 |
"sconv_kernel_size": 4,
|
| 41 |
"unpadded_vocab_size": 200058,
|
| 42 |
"final_logit_softcapping": 30.0,
|
| 43 |
-
"swa_head_dim":
|
| 44 |
-
"swa_num_attention_heads":
|
| 45 |
"swa_num_key_value_heads": 4,
|
| 46 |
"sliding_window_size": 1024,
|
| 47 |
"n_layers_per_attn_res_block": 6,
|
|
@@ -49,7 +49,7 @@
|
|
| 49 |
"n_routed_experts": 96,
|
| 50 |
"num_experts_per_tok": 8,
|
| 51 |
"n_shared_experts": 0,
|
| 52 |
-
"intermediate_size":
|
| 53 |
"mlp_gate_softcap": 4.0,
|
| 54 |
"mlp_up_softcap": 25.0,
|
| 55 |
"route_scale": 8.0,
|
|
@@ -68,14 +68,14 @@
|
|
| 68 |
"mlp_ratio": 4.0,
|
| 69 |
"posemb_grid_size": 32,
|
| 70 |
"layer_norm_eps": 1e-05,
|
| 71 |
-
"text_hidden_size":
|
| 72 |
},
|
| 73 |
"mtp_config": {
|
| 74 |
-
"hidden_size":
|
| 75 |
"num_hidden_layers": 5,
|
| 76 |
-
"num_attention_heads":
|
| 77 |
"num_key_value_heads": 4,
|
| 78 |
-
"head_dim":
|
| 79 |
"intermediate_size": 128,
|
| 80 |
"max_position_embeddings": 1048576,
|
| 81 |
"torch_dtype": "bfloat16",
|
|
@@ -99,7 +99,6 @@
|
|
| 99 |
]
|
| 100 |
},
|
| 101 |
"tie_word_embeddings": false,
|
| 102 |
-
"
|
| 103 |
-
|
| 104 |
-
}
|
| 105 |
}
|
|
|
|
| 7 |
"text_config": {
|
| 8 |
"model_max_length": 1048576,
|
| 9 |
"torch_dtype": "bfloat16",
|
| 10 |
+
"hidden_size": 128,
|
| 11 |
"num_hidden_layers": 12,
|
| 12 |
"vocab_size": 201024,
|
| 13 |
+
"num_attention_heads": 8,
|
| 14 |
"num_key_value_heads": 4,
|
| 15 |
+
"head_dim": 128,
|
| 16 |
"d_rel": 16,
|
| 17 |
"rel_extent": 1024,
|
| 18 |
"q_bias": false,
|
|
|
|
| 40 |
"sconv_kernel_size": 4,
|
| 41 |
"unpadded_vocab_size": 200058,
|
| 42 |
"final_logit_softcapping": 30.0,
|
| 43 |
+
"swa_head_dim": 128,
|
| 44 |
+
"swa_num_attention_heads": 8,
|
| 45 |
"swa_num_key_value_heads": 4,
|
| 46 |
"sliding_window_size": 1024,
|
| 47 |
"n_layers_per_attn_res_block": 6,
|
|
|
|
| 49 |
"n_routed_experts": 96,
|
| 50 |
"num_experts_per_tok": 8,
|
| 51 |
"n_shared_experts": 0,
|
| 52 |
+
"intermediate_size": 128,
|
| 53 |
"mlp_gate_softcap": 4.0,
|
| 54 |
"mlp_up_softcap": 25.0,
|
| 55 |
"route_scale": 8.0,
|
|
|
|
| 68 |
"mlp_ratio": 4.0,
|
| 69 |
"posemb_grid_size": 32,
|
| 70 |
"layer_norm_eps": 1e-05,
|
| 71 |
+
"text_hidden_size": 128
|
| 72 |
},
|
| 73 |
"mtp_config": {
|
| 74 |
+
"hidden_size": 128,
|
| 75 |
"num_hidden_layers": 5,
|
| 76 |
+
"num_attention_heads": 8,
|
| 77 |
"num_key_value_heads": 4,
|
| 78 |
+
"head_dim": 128,
|
| 79 |
"intermediate_size": 128,
|
| 80 |
"max_position_embeddings": 1048576,
|
| 81 |
"torch_dtype": "bfloat16",
|
|
|
|
| 99 |
]
|
| 100 |
},
|
| 101 |
"tie_word_embeddings": false,
|
| 102 |
+
"transformers_version": "5.23.0.dev0",
|
| 103 |
+
"dtype": "bfloat16"
|
|
|
|
| 104 |
}
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ac7f0eaa84e50b53c4afce3ea7eecdc49ccca21bd467edb16f3d052d9099a3de
|
| 3 |
+
size 162239128
|
modeling_inkling2_tiny.py
DELETED
|
@@ -1,100 +0,0 @@
|
|
| 1 |
-
"""Load the mixed packed Inkling 2 checkpoint with the native Transformers model."""
|
| 2 |
-
|
| 3 |
-
from pathlib import Path
|
| 4 |
-
|
| 5 |
-
import torch
|
| 6 |
-
from huggingface_hub import hf_hub_download
|
| 7 |
-
from safetensors.torch import load_file
|
| 8 |
-
from transformers import Inkling2Config, Inkling2ForConditionalGeneration
|
| 9 |
-
from transformers.conversion_mapping import get_model_conversion_mapping
|
| 10 |
-
from transformers.core_model_loading import WeightConverter, WeightRenaming, rename_source_key
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
def dequantize(weight, scale, amax=None, denominator=None):
|
| 14 |
-
if weight.dtype == torch.uint8:
|
| 15 |
-
if amax is None or denominator is None:
|
| 16 |
-
raise ValueError("NVFP4 weights require per-expert amax and an outer-scale denominator")
|
| 17 |
-
codes = torch.stack((weight & 15, weight >> 4), dim=-1).flatten(-2)
|
| 18 |
-
levels = weight.new_tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.float32)
|
| 19 |
-
values = levels[(codes & 7).long()] * torch.where(codes < 8, 1.0, -1.0)
|
| 20 |
-
return values * scale.float().repeat_interleave(16, -1) * (amax / denominator).reshape(-1, 1, 1)
|
| 21 |
-
if weight.dtype == torch.float8_e4m3fn:
|
| 22 |
-
scales = torch.pow(2.0, scale.float() - 127).repeat_interleave(32, -1)
|
| 23 |
-
return weight.float() * scales
|
| 24 |
-
raise ValueError(f"Unsupported packed expert dtype: {weight.dtype}")
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
class Inkling2TinyForConditionalGeneration(Inkling2ForConditionalGeneration):
|
| 28 |
-
@classmethod
|
| 29 |
-
def from_pretrained(
|
| 30 |
-
cls, pretrained_model_name_or_path, *model_args, config=None, dtype=torch.bfloat16,
|
| 31 |
-
revision=None, cache_dir=None, local_files_only=False, token=None,
|
| 32 |
-
attn_implementation="eager", **kwargs,
|
| 33 |
-
):
|
| 34 |
-
if model_args:
|
| 35 |
-
raise ValueError("Inkling 2 does not take positional construction arguments")
|
| 36 |
-
options = dict(revision=revision, cache_dir=cache_dir, local_files_only=local_files_only, token=token)
|
| 37 |
-
root = Path(pretrained_model_name_or_path)
|
| 38 |
-
|
| 39 |
-
def resolve(filename):
|
| 40 |
-
if root.is_dir():
|
| 41 |
-
return root / filename
|
| 42 |
-
return Path(hf_hub_download(str(pretrained_model_name_or_path), filename, **options))
|
| 43 |
-
|
| 44 |
-
import json
|
| 45 |
-
|
| 46 |
-
raw = json.loads(resolve("config.json").read_text())
|
| 47 |
-
# The native config otherwise assumes the all-MXFP8 sibling. This
|
| 48 |
-
# adapter dequantizes each tensor according to its actual stored dtype.
|
| 49 |
-
raw.pop("quantization_config", None)
|
| 50 |
-
config = Inkling2Config(**raw)
|
| 51 |
-
config._attn_implementation = attn_implementation
|
| 52 |
-
dtype = kwargs.pop("torch_dtype", dtype)
|
| 53 |
-
if dtype in (None, "auto"):
|
| 54 |
-
dtype = torch.bfloat16
|
| 55 |
-
elif isinstance(dtype, str):
|
| 56 |
-
dtype = getattr(torch, dtype)
|
| 57 |
-
if dtype not in (torch.float32, torch.bfloat16, torch.float16):
|
| 58 |
-
raise ValueError("The Transformers adapter requires a floating compute dtype")
|
| 59 |
-
device_map = kwargs.pop("device_map", None)
|
| 60 |
-
if device_map == "auto":
|
| 61 |
-
device_map = "cuda" if torch.cuda.is_available() else "cpu"
|
| 62 |
-
if isinstance(device_map, dict):
|
| 63 |
-
if set(device_map) != {""}:
|
| 64 |
-
raise ValueError("Use a single device for this tiny Transformers adapter")
|
| 65 |
-
device_map = device_map[""]
|
| 66 |
-
allowed = {"_from_auto", "_from_pipeline", "trust_remote_code", "weights_only", "adapter_kwargs", "_commit_hash"}
|
| 67 |
-
unexpected = set(kwargs) - allowed
|
| 68 |
-
if unexpected:
|
| 69 |
-
raise TypeError(f"Unsupported loading arguments: {sorted(unexpected)}")
|
| 70 |
-
model = Inkling2ForConditionalGeneration(config)
|
| 71 |
-
transforms = get_model_conversion_mapping(model)
|
| 72 |
-
renamings = [t for t in transforms if isinstance(t, WeightRenaming)]
|
| 73 |
-
converters = [t for t in transforms if isinstance(t, WeightConverter)]
|
| 74 |
-
source = load_file(str(resolve("model.safetensors")))
|
| 75 |
-
state = {}
|
| 76 |
-
for name, weight in source.items():
|
| 77 |
-
if name.endswith(("_scale", "_amax", "_outer_scale_denominator")):
|
| 78 |
-
continue
|
| 79 |
-
if name + "_scale" in source:
|
| 80 |
-
weight = dequantize(
|
| 81 |
-
weight, source[name + "_scale"], source.get(name + "_amax"),
|
| 82 |
-
source.get(name + "_outer_scale_denominator"),
|
| 83 |
-
)
|
| 84 |
-
target, _ = rename_source_key(name, renamings, converters)
|
| 85 |
-
if name.endswith("w13_weight"):
|
| 86 |
-
# The source interleaves gate and up; Transformers chunks them.
|
| 87 |
-
e, rows, columns = weight.shape
|
| 88 |
-
weight = weight.reshape(e, rows // 2, 2, columns).transpose(1, 2).reshape(e, rows, columns)
|
| 89 |
-
if target in state:
|
| 90 |
-
raise ValueError(f"Checkpoint keys map to the same parameter: {target}")
|
| 91 |
-
keep_float32 = source[name].dtype == torch.float32 or any(
|
| 92 |
-
part in target for part in model._keep_in_fp32_modules_strict
|
| 93 |
-
)
|
| 94 |
-
target_dtype = torch.float32 if keep_float32 else dtype
|
| 95 |
-
state[target] = weight.to(target_dtype).contiguous()
|
| 96 |
-
model.load_state_dict(state, strict=True, assign=True)
|
| 97 |
-
model.eval()
|
| 98 |
-
if device_map is not None:
|
| 99 |
-
model.to(device_map)
|
| 100 |
-
return model
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
mtp.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5b2d1d9b26b66f9d222696d5103486d30849f34734ac59ef172bd1d2622b0285
|
| 3 |
+
size 8983080
|
pyproject.toml
DELETED
|
@@ -1,15 +0,0 @@
|
|
| 1 |
-
[build-system]
|
| 2 |
-
requires = ["setuptools>=68"]
|
| 3 |
-
build-backend = "setuptools.build_meta"
|
| 4 |
-
|
| 5 |
-
[project]
|
| 6 |
-
name = "sglang-inkling2-tiny-kernels"
|
| 7 |
-
version = "0.2.0"
|
| 8 |
-
description = "Small-matrix SGLang kernels for the tiny Inkling 2 checkpoint"
|
| 9 |
-
requires-python = ">=3.10"
|
| 10 |
-
|
| 11 |
-
[project.entry-points."sglang.srt.plugins"]
|
| 12 |
-
inkling2_tiny_kernels = "sglang_inkling2_tiny_kernels:register"
|
| 13 |
-
|
| 14 |
-
[tool.setuptools]
|
| 15 |
-
py-modules = ["sglang_inkling2_tiny_kernels"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
sglang_inkling2_tiny_kernels.py
DELETED
|
@@ -1,129 +0,0 @@
|
|
| 1 |
-
"""Small-matrix kernels for the tiny Inkling 2 checkpoint."""
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
class _PaddedMarlin(tuple):
|
| 10 |
-
def __new__(cls, packed, input_width, output_width, padded_input, padded_output):
|
| 11 |
-
value = super().__new__(cls, packed)
|
| 12 |
-
value.input_width = input_width
|
| 13 |
-
value.output_width = output_width
|
| 14 |
-
value.padded_input = padded_input
|
| 15 |
-
value.padded_output = padded_output
|
| 16 |
-
return value
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
def _pad_matrix(tensor, rows, columns, fill=0):
|
| 20 |
-
output = tensor.new_full((*tensor.shape[:-2], rows, columns), fill)
|
| 21 |
-
output[..., :tensor.shape[-2], :tensor.shape[-1]].copy_(tensor)
|
| 22 |
-
return output
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def _repack_marlin(original, weight, scales):
|
| 26 |
-
import torch
|
| 27 |
-
|
| 28 |
-
nvfp4 = weight.dtype == torch.uint8
|
| 29 |
-
packing = 2 if nvfp4 else 1
|
| 30 |
-
block = 16 if nvfp4 else 32
|
| 31 |
-
n, k = weight.shape[-2], weight.shape[-1] * packing
|
| 32 |
-
padded_k, padded_n = (k + 127) // 128 * 128, (n + 127) // 128 * 128
|
| 33 |
-
if (padded_k, padded_n) == (k, n):
|
| 34 |
-
return original(weight, scales)
|
| 35 |
-
weight = _pad_matrix(weight, padded_n, padded_k // packing)
|
| 36 |
-
# Extra MXFP8 blocks encode a scale of 1 (UE8M0 exponent 127).
|
| 37 |
-
# Zero weight bytes make padded products zero in both precision families.
|
| 38 |
-
scales = _pad_matrix(scales, padded_n, padded_k // block, 0 if nvfp4 else 127)
|
| 39 |
-
packed = original(weight, scales)
|
| 40 |
-
return _PaddedMarlin(packed, k, n, padded_k, padded_n)
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
def _marlin_matmul(original, x, packed, out_size, *args, **kwargs):
|
| 44 |
-
if not isinstance(packed, _PaddedMarlin):
|
| 45 |
-
return original(x, packed, out_size, *args, **kwargs)
|
| 46 |
-
if x.shape[-1] != packed.input_width or out_size != packed.output_width:
|
| 47 |
-
raise ValueError("Marlin padding metadata does not match the expert projection")
|
| 48 |
-
padded = x.new_zeros((*x.shape[:-1], packed.padded_input))
|
| 49 |
-
padded[..., :x.shape[-1]].copy_(x)
|
| 50 |
-
if original.__name__ == "gemv" and packed.padded_input == 128:
|
| 51 |
-
# Eight K tiles fit one split of eight warps. The large-model heuristic
|
| 52 |
-
# otherwise chooses four splits/four warps for short decode batches.
|
| 53 |
-
kwargs = {**kwargs, "splits": 1, "warps": 8}
|
| 54 |
-
result = original(padded, packed, packed.padded_output, *args, **kwargs)
|
| 55 |
-
return result[..., :out_size].contiguous()
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
def _relative_fa4(original, *args, **kwargs):
|
| 59 |
-
q = args[0] if args else kwargs.get("q")
|
| 60 |
-
if (q is not None and q.shape[-1] == 32 and kwargs.get("rel_bias") is not None
|
| 61 |
-
and kwargs.get("tile_mn") is None):
|
| 62 |
-
# The width-32 heuristic chooses M=192 on Hopper, whereas Inkling's
|
| 63 |
-
# relative-bias kernel explicitly requires M=N=128. Use its tile API.
|
| 64 |
-
kwargs = {**kwargs, "tile_mn": (128, 128)}
|
| 65 |
-
return original(*args, **kwargs)
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
def _attn_res_config(original, width, *args, **kwargs):
|
| 69 |
-
threads, stages, vector, cluster = original(width, *args, **kwargs)
|
| 70 |
-
if width == 32:
|
| 71 |
-
# One compute warp covers the original 32 residual coordinates.
|
| 72 |
-
# Keep the same CuTe kernel, RMS normalization, and source weighting.
|
| 73 |
-
return 32, stages, 1, 1
|
| 74 |
-
return threads, stages, vector, cluster
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def _small_all_reduce(original, x, group, **kwargs):
|
| 78 |
-
if x.ndim == 2 and x.shape[-1] == 32:
|
| 79 |
-
return group.all_reduce(x.clone())
|
| 80 |
-
return original(x, group, **kwargs)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
def _small_multicast_input(original, group, shape, dtype):
|
| 84 |
-
if len(shape) == 2 and shape[-1] == 32:
|
| 85 |
-
import torch
|
| 86 |
-
|
| 87 |
-
return torch.empty(shape, dtype=dtype, device=torch.cuda.current_device())
|
| 88 |
-
return original(group, shape, dtype)
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def _small_multicast_reduce(original, x, group):
|
| 92 |
-
if x.ndim == 2 and x.shape[-1] == 32:
|
| 93 |
-
x.copy_(group.all_reduce(x))
|
| 94 |
-
return x
|
| 95 |
-
return original(x, group)
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
def _small_oneshot_input(original, group, rows, cols, dtype, tag=""):
|
| 99 |
-
if cols == 32:
|
| 100 |
-
# Use the existing FP32 combine path instead of the GB300-tuned
|
| 101 |
-
# symmetric-memory one-shot collective for these tiny projections.
|
| 102 |
-
return None
|
| 103 |
-
return original(group, rows, cols, dtype, tag=tag)
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
def register():
|
| 107 |
-
from sglang.srt.plugins.hook_registry import HookRegistry, HookType
|
| 108 |
-
|
| 109 |
-
prefix = "sglang.kernels.ops.inkling2.marlin_sm90"
|
| 110 |
-
for name in ("repack", "repack_mxfp8"):
|
| 111 |
-
HookRegistry.register(f"{prefix}.{name}", _repack_marlin, HookType.AROUND)
|
| 112 |
-
for name in ("gemm", "gemv"):
|
| 113 |
-
HookRegistry.register(f"{prefix}.{name}", _marlin_matmul, HookType.AROUND)
|
| 114 |
-
HookRegistry.register(
|
| 115 |
-
"sglang.kernels.ops.attention.flash_attn.cute.interface._flash_attn_fwd",
|
| 116 |
-
_relative_fa4, HookType.AROUND,
|
| 117 |
-
)
|
| 118 |
-
HookRegistry.register(
|
| 119 |
-
"sglang.kernels.ops.inkling2.attn_res._attn_res_fwd_cutedsl_config",
|
| 120 |
-
_attn_res_config, HookType.AROUND,
|
| 121 |
-
)
|
| 122 |
-
prefix = "sglang.kernels.ops.inkling2.comm"
|
| 123 |
-
for name, hook in (
|
| 124 |
-
("all_reduce", _small_all_reduce),
|
| 125 |
-
("multicast_all_reduce_input", _small_multicast_input),
|
| 126 |
-
("multicast_all_reduce", _small_multicast_reduce),
|
| 127 |
-
("oneshot_all_reduce_input", _small_oneshot_input),
|
| 128 |
-
):
|
| 129 |
-
HookRegistry.register(f"{prefix}.{name}", hook, HookType.AROUND)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|