Inkling-2-Mini-nvfp4 / runtime /inkling2-mini.patch
prince-canuma's picture
Upload validated MLX Inkling 2 Mini with native mixed precision and bundled draft
26f71ee verified
Raw History Blame Contribute Delete
27.3 kB
diff --git a/mlx_vlm/convert.py b/mlx_vlm/convert.py
index e7b032f5..0b884aea 100644
--- a/mlx_vlm/convert.py
+++ b/mlx_vlm/convert.py
@@ -318,6 +318,19 @@ def convert(
model, config, processor = fetch_from_hub(
model_path, lazy=True, trust_remote_code=trust_remote_code
)
+ # Native packed exports can declare only a generic quantization family in
+ # config.json, with the actual per-layer precision in a sidecar. Preserve
+ # that precision when saving the sanitized MLX weights.
+ metadata_path = model_path / "hf_quant_config.json"
+ if (
+ "quantization" not in config
+ and hasattr(model, "quantization_from_metadata")
+ and metadata_path.exists()
+ ):
+ with metadata_path.open() as f:
+ native_quantization = model.quantization_from_metadata(json.load(f))
+ if native_quantization is not None:
+ config["quantization"] = config["quantization_config"] = native_quantization
model_quant_predicate = getattr(model, "quant_predicate", None)
diff --git a/mlx_vlm/models/inkling2/config.py b/mlx_vlm/models/inkling2/config.py
index 8e3b7b20..f6e4f1e4 100644
--- a/mlx_vlm/models/inkling2/config.py
+++ b/mlx_vlm/models/inkling2/config.py
@@ -40,6 +40,34 @@ class TextConfig(BaseModelConfig):
mlp_gate_softcap: float = 4.0
mlp_up_softcap: float = 25.0
final_logit_softcapping: float = 30.0
+ loop_config: Optional[dict] = None
+
+ def __post_init__(self):
+ if self.loop_config is None:
+ return
+ cfg = self.loop_config
+ start, end = cfg["core_start"], cfg["core_end"]
+ block = self.n_layers_per_attn_res_block
+ if (
+ not 0 < start < end <= self.num_hidden_layers
+ or start % block
+ or end % block
+ or cfg["num_passes"] < 1
+ or cfg["gate_rank"] < 1
+ or cfg.get("is_varloop", False)
+ ):
+ raise ValueError("Invalid or unsupported Inkling 2 loop configuration")
+
+ @property
+ def layer_schedule(self):
+ if self.loop_config is None:
+ return list(range(self.num_hidden_layers))
+ start, end = self.loop_config["core_start"], self.loop_config["core_end"]
+ return (
+ list(range(start))
+ + list(range(start, end)) * self.loop_config["num_passes"]
+ + list(range(end, self.num_hidden_layers))
+ )
def layer_is_sliding(self, index):
if self.layer_types is not None:
diff --git a/mlx_vlm/models/inkling2/inkling2.py b/mlx_vlm/models/inkling2/inkling2.py
index d2173a54..08c04909 100644
--- a/mlx_vlm/models/inkling2/inkling2.py
+++ b/mlx_vlm/models/inkling2/inkling2.py
@@ -114,6 +114,16 @@ class Model(nn.Module):
if sub.startswith("mlp.experts."):
experts.setdefault(layer, {})[sub[len("mlp.experts.") :]] = value
continue
+ if sub.startswith("controllers."):
+ _, name, leaf = sub.split(".")
+ name = {"attn": "q", "k": "k", "v": "v", "mlp": "mlp"}[name]
+ leaf = {
+ "down_weight": "down_proj",
+ "forget_weight": "forget_gate_proj",
+ "gain_weight": "message_gate_proj",
+ }[leaf]
+ out[base + name + "_message_gate." + leaf + ".weight"] = value
+ continue
if sub.startswith("attn."):
sub = sub[len("attn.") :]
name, _, leaf = sub.partition(".")
@@ -156,6 +166,8 @@ class Model(nn.Module):
out["language_model.model." + target] = value
elif key.startswith("model.llm."):
sub = key[len("model.llm.") :]
+ if sub.startswith("middle_cycle_cross_pass_"):
+ continue
target = (
"language_model.lm_head.weight"
if sub == "unembed.weight"
@@ -181,6 +193,15 @@ class Model(nn.Module):
continue
else:
out[key] = value
+ query_key = "model.llm.middle_cycle_cross_pass_queries"
+ kv_key = "model.llm.middle_cycle_cross_pass_kv_queries"
+ if query_key in weights or kv_key in weights:
+ hidden = self.config.text_config.hidden_size
+ queries = weights[query_key].reshape(-1, 2, hidden)
+ kv_queries = weights[kv_key].reshape(-1, 2, hidden)
+ out["language_model.model.message_input.query"] = mx.concatenate(
+ [queries[:, :1], kv_queries, queries[:, 1:]], axis=1
+ )
for layer, tensors in experts.items():
base = f"language_model.model.layers.{layer}.mlp.switch_mlp."
for name, projections in (
diff --git a/mlx_vlm/models/inkling2/language.py b/mlx_vlm/models/inkling2/language.py
index 805dcc1f..023226db 100644
--- a/mlx_vlm/models/inkling2/language.py
+++ b/mlx_vlm/models/inkling2/language.py
@@ -537,6 +537,44 @@ class AttentionResidual(nn.Module):
)
+class MessageGate(nn.Module):
+ """Mix a previous-pass message into the current attention-residual input."""
+
+ def __init__(self, config):
+ super().__init__()
+ rank = config.loop_config["gate_rank"]
+ self.eps = config.rms_norm_eps
+ self.down_proj = nn.Linear(2 * config.hidden_size, rank, bias=False)
+ self.forget_gate_proj = nn.Linear(rank, config.hidden_size, bias=False)
+ self.message_gate_proj = nn.Linear(rank, config.hidden_size, bias=False)
+
+ def __call__(self, hidden, message):
+ hidden, message = hidden.astype(mx.float32), message.astype(mx.float32)
+ inputs = mx.concatenate(
+ [mx.fast.rms_norm(x, None, self.eps) for x in (hidden, message)], axis=-1
+ )
+ gate = self.down_proj(inputs.astype(self.down_proj.weight.dtype)).astype(
+ mx.float32
+ )
+ forget = mx.tanh(gate @ self.forget_gate_proj.weight.astype(mx.float32).T)
+ gain = mx.tanh(gate @ self.message_gate_proj.weight.astype(mx.float32).T)
+ return (1 - forget) * hidden + gain * message
+
+
+class MessageAttentionResidual(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ loop = config.loop_config
+ self.query = mx.zeros(
+ (loop["core_end"] - loop["core_start"], 4, config.hidden_size), mx.float32
+ )
+
+ def __call__(self, blocks, partial, layer_index):
+ return _attention_residual(
+ mx.stack([*blocks, partial], axis=-2), self.query[layer_index], True
+ )
+
+
class DecoderLayer(nn.Module):
def __init__(self, config, layer_idx):
super().__init__()
@@ -563,8 +601,12 @@ class DecoderLayer(nn.Module):
self.mlp_sconv = ShortConvolution(
config.hidden_size, config.sconv_kernel_size, 3
)
+ loop = config.loop_config
+ if loop and loop["core_start"] <= layer_idx < loop["core_end"]:
+ for name in ("q", "k", "v", "mlp"):
+ self[name + "_message_gate"] = MessageGate(config)
- def __call__(self, x, blocks, cache=None):
+ def __call__(self, x, blocks, cache=None, messages=None):
conv = cache[1] if cache is not None else None
mask = _cache_padding_mask(conv, x.shape[1])
mixed = (
@@ -572,7 +614,7 @@ class DecoderLayer(nn.Module):
if self.attention_input is not None
else x[..., None, :]
)
- if self.is_sliding:
+ if self.is_sliding and messages is None:
attn = self.self_attn(
self.input_layernorm(mixed[..., 0, :]).astype(x.dtype),
cache=cache,
@@ -580,17 +622,21 @@ class DecoderLayer(nn.Module):
)
else:
parts = [mixed[..., min(i, mixed.shape[-2] - 1), :] for i in range(3)]
- parts = [
- norm(part).astype(x.dtype)
- for norm, part in zip(
- (
- self.q_input_layernorm,
- self.k_input_layernorm,
- self.v_input_layernorm,
- ),
- parts,
+ if messages is not None:
+ parts = [
+ self[name + "_message_gate"](part, messages[..., i, :])
+ for i, (name, part) in enumerate(zip(("q", "k", "v"), parts))
+ ]
+ norms = (
+ (self.input_layernorm,) * 3
+ if self.is_sliding
+ else (
+ self.q_input_layernorm,
+ self.k_input_layernorm,
+ self.v_input_layernorm,
)
- ]
+ )
+ parts = [norm(part).astype(x.dtype) for norm, part in zip(norms, parts)]
attn = self.self_attn(*parts, cache=cache, conv_mask=mask)
residual = self.attn_sconv(attn, conv, mask).astype(mx.float32)
if self.is_block_boundary:
@@ -598,6 +644,8 @@ class DecoderLayer(nn.Module):
else:
residual = residual + x.astype(mx.float32)
hidden = self.mlp_input(blocks, residual)[..., 0, :]
+ if messages is not None:
+ hidden = self.mlp_message_gate(hidden, messages[..., 3, :])
hidden = self.mlp(self.post_attention_layernorm(hidden).astype(x.dtype))
out = (residual + self.mlp_sconv(hidden, conv, mask).astype(mx.float32)).astype(
x.dtype
@@ -610,11 +658,14 @@ class DecoderLayer(nn.Module):
class TextModel(nn.Module):
def __init__(self, config):
super().__init__()
+ self.config = config
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
self.embed_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.layers = [DecoderLayer(config, i) for i in range(config.num_hidden_layers)]
self.final_input = AttentionResidual(config)
self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ if config.loop_config:
+ self.message_input = MessageAttentionResidual(config)
def embed(self, inputs):
return self.embed_norm(self.embed_tokens(inputs))
@@ -632,7 +683,7 @@ class TextModel(nn.Module):
if capture_layer_ids is not None and (
len(capture_layer_ids) != len(set(capture_layer_ids))
or any(
- type(i) is not int or not 0 <= i <= len(self.layers)
+ type(i) is not int or not 0 <= i <= len(self.config.layer_schedule)
for i in capture_layer_ids
)
):
@@ -640,19 +691,35 @@ class TextModel(nn.Module):
requested = set(capture_layer_ids or [])
x = self.embed(inputs) if input_embeddings is None else input_embeddings
blocks = []
- for i, (layer, c) in enumerate(
- zip(self.layers, cache or [None] * len(self.layers))
- ):
+ schedule = self.config.layer_schedule
+ if cache is not None and len(cache) != len(schedule):
+ raise ValueError("Inkling 2 requires a separate cache for each layer visit")
+ caches = cache if cache is not None else [None] * len(schedule)
+ loop = self.config.loop_config
+ pre_loop = previous = None
+ for i, (layer_index, c) in enumerate(zip(schedule, caches)):
+ layer = self.layers[layer_index]
+ messages = None
+ if loop and layer_index == loop["core_start"]:
+ if pre_loop is None:
+ pre_loop = (x, list(blocks))
+ else:
+ previous = (blocks[len(pre_loop[1]) + 1 :], x)
+ x, blocks = pre_loop[0], list(pre_loop[1])
+ if previous is not None and layer_index < loop["core_end"]:
+ messages = self.message_input(
+ *previous, layer_index - loop["core_start"]
+ )
# DSpark taps the partial block entering the layer, not its output.
if i in requested:
captures[i] = x
- x = layer(x, blocks, c)
+ x = layer(x, blocks, c, messages=messages)
if speculative_verify and (i + 1) % 4 == 0:
mx.async_eval(x)
h = self.final_input(blocks, x)[..., 0, :]
normalized = self.norm(h).astype(x.dtype)
- if len(self.layers) in requested:
- captures[len(self.layers)] = normalized
+ if len(schedule) in requested:
+ captures[len(schedule)] = normalized
result = h if skip_final_norm else normalized
if capture_layer_ids is not None:
return result, [captures[i] for i in capture_layer_ids]
@@ -830,7 +897,7 @@ class LanguageModel(InklingLanguageModel):
),
ArraysCache(4),
)
- for layer in self.model.layers
+ for layer in (self.model.layers[i] for i in self.config.layer_schedule)
]
def speculative_verify_hidden(self, inputs, cache):
diff --git a/mlx_vlm/models/inkling2/processing_inkling2.py b/mlx_vlm/models/inkling2/processing_inkling2.py
index ffefc0a1..d640a644 100644
--- a/mlx_vlm/models/inkling2/processing_inkling2.py
+++ b/mlx_vlm/models/inkling2/processing_inkling2.py
@@ -206,4 +206,4 @@ class Inkling2Processor(ProcessorMixin):
return ["input_ids", "attention_mask", "pixel_values", "image_grid_thw"]
-install_auto_processor_patch(["inkling2"], Inkling2Processor)
+install_auto_processor_patch(["inkling2", "inkling2_loop"], Inkling2Processor)
diff --git a/mlx_vlm/prompt_utils.py b/mlx_vlm/prompt_utils.py
index 79a6067b..387035a1 100644
--- a/mlx_vlm/prompt_utils.py
+++ b/mlx_vlm/prompt_utils.py
@@ -122,6 +122,7 @@ MODEL_CONFIG = {
"inkling": MessageFormat.LIST_WITH_IMAGE_FIRST,
"inkling_mm_model": MessageFormat.LIST_WITH_IMAGE_FIRST,
"inkling2": MessageFormat.LIST_WITH_IMAGE_FIRST,
+ "inkling2_loop": MessageFormat.LIST_WITH_IMAGE_FIRST,
# Prompt-only models
"florence2": MessageFormat.PROMPT_ONLY,
"plamo2vl": MessageFormat.PROMPT_ONLY,
diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py
index 262597a0..15726d22 100644
--- a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py
+++ b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py
@@ -64,6 +64,6 @@ class ModelConfig(BaseModelConfig):
len(ids) < 2
or ids != sorted(set(ids))
or ids[0] < 0
- or ids[-1] != text["num_hidden_layers"]
+ or ids[-1] != len(self.text_config.layer_schedule)
):
raise ValueError("Invalid Inkling 2 DSpark target_layer_ids")
diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py
index 197193be..b4ff9f21 100644
--- a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py
+++ b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py
@@ -211,7 +211,8 @@ class Model(nn.Module):
args = getattr(target, "config", None)
if (
args is None
- or args.model_type != "inkling2_text"
+ or args.model_type not in ("inkling2_text", "inkling2_loop_text")
+ or args.loop_config != self.config.text_config.loop_config
or any(
getattr(args, name) != getattr(self.config.text_config, name)
for name in ("hidden_size", "vocab_size", "num_hidden_layers")
diff --git a/mlx_vlm/speculative/drafters/mtp_split.py b/mlx_vlm/speculative/drafters/mtp_split.py
index 2c586aa2..a149c0ed 100644
--- a/mlx_vlm/speculative/drafters/mtp_split.py
+++ b/mlx_vlm/speculative/drafters/mtp_split.py
@@ -351,6 +351,8 @@ MTP_SPLITTERS: Dict[str, str] = {
"inkling_mm_model": "mlx_vlm.speculative.drafters.inkling_mtp.split:InklingMTPSplitter",
"inkling2": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter",
"inkling2_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter",
+ "inkling2_loop": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter",
+ "inkling2_loop_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter",
}
diff --git a/mlx_vlm/tests/model_cases.json b/mlx_vlm/tests/model_cases.json
index ce909d1a..3c21508c 100644
--- a/mlx_vlm/tests/model_cases.json
+++ b/mlx_vlm/tests/model_cases.json
@@ -1213,6 +1213,79 @@
"checks": ["forward_cache", "language", "vision", "input_embeddings"],
"vision": {"input_shape": [4, 1176], "feature_layer": 0, "grid_thw": [[1, 2, 2]]}
},
+ {
+ "id": "inkling2_loop",
+ "language": {
+ "logits_dtype": "float32"
+ },
+ "module": "inkling2",
+ "config": {
+ "image_token_id": 60,
+ "text_config": {
+ "hidden_size": 32,
+ "num_hidden_layers": 5,
+ "vocab_size": 64,
+ "unpadded_vocab_size": 64,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 8,
+ "swa_num_attention_heads": 4,
+ "swa_num_key_value_heads": 2,
+ "swa_head_dim": 8,
+ "sliding_window_size": 4,
+ "local_layer_ids": [
+ 0,
+ 2,
+ 4
+ ],
+ "d_rel": 4,
+ "rel_extent": 4,
+ "sconv_kernel_size": 4,
+ "intermediate_size": 32,
+ "n_routed_experts": 4,
+ "num_experts_per_tok": 2,
+ "n_layers_per_attn_res_block": 2,
+ "loop_config": {
+ "core_start": 2,
+ "core_end": 4,
+ "num_passes": 3,
+ "gate_rank": 8,
+ "is_varloop": false
+ }
+ },
+ "vision_config": {
+ "native_patch_size": 2,
+ "adapter_merge_size": 2,
+ "hidden_size": 16,
+ "num_hidden_layers": 2,
+ "num_attention_heads": 2,
+ "projection_hidden_dim": 32,
+ "posemb_grid_size": 2
+ },
+ "model_type": "inkling2_loop"
+ },
+ "checks": [
+ "forward_cache",
+ "language",
+ "vision"
+ ],
+ "vision": {
+ "input_shape": [
+ 2,
+ 4,
+ 4,
+ 3
+ ],
+ "feature_layer": null,
+ "grid_thw": [
+ [
+ 1,
+ 1,
+ 2
+ ]
+ ]
+ }
+ },
{
"id": "inkling2",
"language": {"logits_dtype": "float32"},
diff --git a/mlx_vlm/tests/test_models.py b/mlx_vlm/tests/test_models.py
index cc269be4..2adb290d 100644
--- a/mlx_vlm/tests/test_models.py
+++ b/mlx_vlm/tests/test_models.py
@@ -7696,11 +7696,12 @@ class TestGemma4PerLayerConfig:
@pytest.fixture
-def inkling2_model():
+def inkling2_model(request):
from mlx_vlm.models.inkling2 import Model, ModelConfig
mx.random.seed(194)
- model = Model(ModelConfig.from_dict(model_case("inkling2")["config"]))
+ case = getattr(request, "param", "inkling2")
+ model = Model(ModelConfig.from_dict(model_case(case)["config"]))
for path, value in tree_flatten(model.parameters()):
if any(name in path for name in (".query", ".rel_proj", "gate_weight")):
model.load_weights(
@@ -7722,6 +7723,16 @@ def _inkling2_native_weights(model):
key = f"model.llm.attn_res.attn_queries.{int(layer) - 1}"
elif sub == "mlp_input.query":
key = f"model.llm.attn_res.mlp_queries.{layer}"
+ elif "_message_gate." in sub:
+ gate, leaf, _ = sub.split(".")
+ name = gate.removesuffix("_message_gate")
+ name = "attn" if name == "q" else name
+ leaf = {
+ "down_proj": "down_weight",
+ "forget_gate_proj": "forget_weight",
+ "message_gate_proj": "gain_weight",
+ }[leaf]
+ key = f"model.llm.layers.{layer}.controllers.{name}.{leaf}"
else:
for src, dst in (
("self_attn.q_proj", "attn.wq_du"),
@@ -7752,6 +7763,14 @@ def _inkling2_native_weights(model):
key = f"model.llm.layers.{layer}.{sub}"
elif key.startswith("language_model.model."):
sub = key[len("language_model.model.") :]
+ if sub == "message_input.query":
+ weights["model.llm.middle_cycle_cross_pass_queries"] = value[
+ :, [0, 3], :
+ ].reshape(-1, value.shape[-1])
+ weights["model.llm.middle_cycle_cross_pass_kv_queries"] = value[
+ :, 1:3, :
+ ].reshape(-1, value.shape[-1])
+ continue
sub = sub.replace("embed_tokens.weight", "embed.weight")
sub = sub.replace("final_input.query", "attn_res.output_query")
key = "model.llm." + sub
@@ -7786,6 +7805,7 @@ def _inkling2_native_weights(model):
return weights
+@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True)
def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_path):
from mlx_vlm.convert import convert
@@ -7803,7 +7823,9 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p
# Match the release's cast groups: queries and router bias stay FP32,
# while dense weights (including router/conv weights) are stored in BF16.
for key, value in raw.items():
- if not (".attn_res." in key or key.endswith(".mlp.gate.bias")):
+ if not (
+ ".attn_res." in key or "cross_pass" in key or key.endswith(".mlp.gate.bias")
+ ):
raw[key] = value.astype(mx.bfloat16)
elif key.endswith(".mlp.gate.bias"):
raw[key] = mx.arange(value.size, dtype=mx.float32) / 129
@@ -7860,6 +7882,11 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p
config = json.loads((tmp_path / "config.json").read_text())
config.pop("quantization")
config.pop("quantization_config")
+ if model.config.text_config.loop_config:
+ config["quantization_config"] = {
+ "quant_method": "modelopt",
+ "quant_algo": "NVFP4",
+ }
(tmp_path / "config.json").write_text(json.dumps(config))
mx.save_safetensors(str(tmp_path / "model.safetensors"), raw)
loaded = load_model(tmp_path, strict=True)
@@ -7909,7 +7936,6 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p
target = old[2:] if ".switch_mlp." in key else old
assert mx.array_equal(value, target).item(), key
converted = tmp_path / "converted"
- config["quantization"] = model.quantization_from_metadata(metadata)
expected_dtypes = {key: value.dtype for key, value in expected.items()}
with patch("mlx_vlm.convert.fetch_from_hub", return_value=(loaded, config, None)):
convert(str(tmp_path), str(converted), dtype="bfloat16")
@@ -7952,6 +7978,7 @@ def test_inkling2_bf16_head_keeps_fp32_projection_scores(inkling2_model, softcap
@pytest.mark.parametrize(
"dtype,tolerance", [(mx.float32, 2e-5), (mx.float16, 1e-2), (mx.bfloat16, 8e-2)]
)
+@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True)
def test_inkling2_cached_chunks_and_speculative_rollback(
inkling2_model, dtype, tolerance
):
@@ -7986,6 +8013,7 @@ def test_inkling2_cached_chunks_and_speculative_rollback(
@pytest.mark.parametrize("dtype", [mx.float32, mx.float16, mx.bfloat16])
+@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True)
def test_inkling2_multiple_images_and_embedding_slots(inkling2_model, dtype):
model = inkling2_model
model.update(tree_map(lambda a: a.astype(dtype), model.parameters()))
diff --git a/mlx_vlm/tests/test_speculative.py b/mlx_vlm/tests/test_speculative.py
index f5950086..e4ab9688 100644
--- a/mlx_vlm/tests/test_speculative.py
+++ b/mlx_vlm/tests/test_speculative.py
@@ -60,13 +60,15 @@ DATA = json.loads(Path(__file__).with_name("model_cases.json").read_text())[
]
-def inkling2_dspark_case(dtype=mx.float32):
+def inkling2_dspark_case(dtype=mx.float32, family="inkling2"):
from mlx_vlm.models.inkling2 import Model, ModelConfig
from mlx_vlm.speculative.drafters.inkling2_dspark import Model as Draft
from mlx_vlm.speculative.drafters.inkling2_dspark import ModelConfig as DraftConfig
- config = deepcopy(model_case("inkling2")["config"])
+ config = deepcopy(model_case(family)["config"])
config["mtp_config"] = deepcopy(DATA["inkling2_dspark"])
+ if family == "inkling2_loop":
+ config["mtp_config"]["target_layer_ids"] = [1, 3, 7, 9]
target = Model(ModelConfig.from_dict(config))
draft = Draft(
DraftConfig(
@@ -461,11 +463,12 @@ def test_inkling2_dspark_markov_head_reference(dtype):
@parametrize("temperature", [0, 0.7])
@parametrize("prefill_step_size", [None, 2])
@parametrize("dtype", [mx.float32, mx.bfloat16])
+@parametrize("family", ["inkling2", "inkling2_loop"])
def test_inkling2_dspark_standard_generation_and_reset(
- temperature, prefill_step_size, dtype, monkeypatch
+ temperature, prefill_step_size, dtype, family, monkeypatch
):
mx.random.seed(73)
- target, draft = inkling2_dspark_case(dtype)
+ target, draft = inkling2_dspark_case(dtype, family=family)
calls = []
forward = type(target.language_model).__call__
diff --git a/mlx_vlm/utils.py b/mlx_vlm/utils.py
index 86e079e0..b43e7aa8 100644
--- a/mlx_vlm/utils.py
+++ b/mlx_vlm/utils.py
@@ -79,6 +79,7 @@ MODEL_REMAPPING = {
"iquestcoder": "llama",
"nemotron-nas": "nemotron_nas",
"inkling_mm_model": "inkling",
+ "inkling2_loop": "inkling2",
"lille-130m": "lille_130m",
}
@@ -547,7 +548,7 @@ python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir>
if quantization_config is not None:
config["quantization_config"] = quantization_config
- if quantization_config is None:
+ if quantization_config is None or hasattr(model, "quantization_from_metadata"):
hf_quant_path = model_path / "hf_quant_config.json"
if hf_quant_path.exists():
with open(hf_quant_path) as f:
@@ -566,6 +567,7 @@ python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir>
config["quantization"] = config["quantization_config"] = (
native_quantization
)
+ quantization_config = None
if quantization_config is not None:
quant_method = quantization_config.get("quant_method")
diff --git a/mlx_vlm/version.py b/mlx_vlm/version.py
index 777f190d..2811cedf 100644
--- a/mlx_vlm/version.py
+++ b/mlx_vlm/version.py
@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.8.0+inkling2mini"