diff --git a/conversion/__init__.py b/conversion/__init__.py index 6a34da41fe4aa8b133371551e391a4d5b4068839..4759e95c98e8329025ddb7f22f25fc20258aa545 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -139,6 +139,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "HrmTextForCausalLM": "hrm_text", "HYV3ForCausalLM": "hunyuan", "HYV4ForCausalLM": "hy_v4", + "HYV4NextForCausalLM": "hy_v4_flash", + "HYV4NextVLForConditionalGeneration": "hy_v4_flash", "IQuestCoderForCausalLM": "llama", "InternLM2ForCausalLM": "internlm", "InternLM3ForCausalLM": "internlm", @@ -357,6 +359,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", "HunYuanVLForConditionalGeneration": "hunyuan", + "HYV4NextVLForConditionalGeneration": "hy_v4_flash_vision", "Idefics3ForConditionalGeneration": "smolvlm", "InternVisionModel": "internvl", "JanusForConditionalGeneration": "januspro", diff --git a/conversion/hy_v4_flash.py b/conversion/hy_v4_flash.py new file mode 100644 index 0000000000000000000000000000000000000000..2b7085221f7257c518b94387dd3225fd57b2699e --- /dev/null +++ b/conversion/hy_v4_flash.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + +import numpy as np +import torch + +from .base import ModelBase, gguf, logger +from .hy_v4 import HYV4Model + +# the hash multipliers are not stored in the checkpoint: the reference derives them from the +# Engram seed, the 1-based layer id and the compressed vocabulary size +ENGRAM_SEED_STRIDE = 10007 + + +@ModelBase.register("HYV4NextForCausalLM", "HYV4NextVLForConditionalGeneration") +@ModelBase.example("tencent/Hy4-Flash") +class HYV4FlashModel(HYV4Model): + """HY_V4_FLASH: hy_v4's iHC + gated MLA + learnable sink with a hybrid attention stack. + + The stack mixes "sparse" full-attention layers with sliding-window layers, in the order given + by layer_types rather than a fixed alternation. Both are MLA over the same latent geometry and + differ only in head count and rope base, so the head count is exported per layer and the window + layers get their own rope frequency. + + Added over hy_v4: sandwich norms (after the attention output and after the FFN) and routed + experts that run in a latent space reached by routed_expert_down_proj / _up_proj, while the + router and the shared expert stay at hidden_size. + + The MTP (nextn) draft block is exported as one extra sparse layer with no hyper-connections of + its own; it is fed the next token's embedding and the trunk's final hidden state. + + Dropped: the vision tower, which is converted separately as an mmproj. + """ + + model_arch = gguf.MODEL_ARCH.HY_V4_FLASH + + # the draft block is exported, and _engram_modify re-enters the normal path under a block index + # past the trunk. DeepseekV2Model.modify_tensors drops exactly that range when skip_mtp is set, + # which would silently lose most of the block. + skip_mtp = False + + # dropped subsystems, see the class docstring + DROP_PREFIXES = ("vit.",) + DROP_SUBSTRINGS = () + + # compressed_indexer tail -> the tensor the graph reads it as + INDEXER_MAP = { + "token_key_projection.weight": gguf.MODEL_TENSOR.INDEXER_ATTN_K, + "token_key_norm.weight": gguf.MODEL_TENSOR.INDEXER_K_NORM, + "key_value_projection.weight": gguf.MODEL_TENSOR.INDEXER_KPOOL_VALUE, + "key_gate_projection.weight": gguf.MODEL_TENSOR.INDEXER_KPOOL_GATE, + "intra_group_ape": gguf.MODEL_TENSOR.INDEXER_KPOOL_APE, + "key_norm.weight": gguf.MODEL_TENSOR.INDEXER_KPOOL_NORM, + "query_projection.weight": gguf.MODEL_TENSOR.INDEXER_ATTN_Q_B, + "head_weight_projection.weight": gguf.MODEL_TENSOR.INDEXER_PROJ, + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # the draft block is an extra block as far as tensor naming goes; block_count is what the + # loader reads as n_layer_all and it subtracts the nextn count back off + if not getattr(self, "no_mtp", False): + self.block_count += self._n_nextn() + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + # Engram weights arrive one at a time but are written fused, so they wait here + self._engram_buf: dict[int, dict[str, torch.Tensor]] = {} + self._engram_tables: dict[int, torch.Tensor] = {} + + @classmethod + def filter_tensors(cls, item): + name = item[0] + if name.startswith(cls.DROP_PREFIXES) or any(s in name for s in cls.DROP_SUBSTRINGS): + return None + if name.startswith("model.mtp_layers."): + # hy_v4 drops the draft block; this model exports it, so skip that filter + return super(HYV4Model, cls).filter_tensors(item) + return super().filter_tensors(item) + + def _layer_types(self) -> list[str]: + cached = getattr(self, "_layer_types_cache", None) + if cached is not None: + return cached + layer_types = self.hparams.get("layer_types") + n_layer = self.hparams["num_hidden_layers"] + if not layer_types or len(layer_types) < n_layer: + raise ValueError("HY_V4_FLASH needs a layer_types entry per layer") + unknown = set(layer_types[:n_layer]) - {"sparse_attention", "sliding_attention"} + if unknown: + raise ValueError(f"HY_V4_FLASH unknown layer_types values: {sorted(unknown)}") + self._layer_types_cache = list(layer_types[:n_layer]) + return self._layer_types_cache + + # ---- Engram ------------------------------------------------------------------------------ + # + # Four n-gram hash modules, one per Engram layer. All of their buckets are packed into a single + # table, so each module's head offsets are rebased into it, matching deepseek41's layout. + + def _engram_enabled(self) -> bool: + return bool(self.hparams.get("engram_enable")) and bool(self.hparams.get("engram_layer_ids")) + + def _engram_ids(self) -> list[int]: + """The config's engram_layer_ids, 1-based, in ascending order.""" + ids = sorted(int(i) for i in self.hparams["engram_layer_ids"]) + n_layer = self.hparams["num_hidden_layers"] + for i in ids: + # 1-based: the tensors live at model.layers.{i-1}.engram_layer.* + if not 1 <= i <= n_layer: + raise ValueError(f"HY_V4_FLASH engram_layer_ids has {i}, outside 1..{n_layer}") + return ids + + def _engram_primes(self) -> dict[int, list[int]]: + """Per 1-based layer id, the 16 bucket counts flattened from their [ngram][head] nesting.""" + raw = self.hparams.get("engram_layer_head_vocab_sizes") or {} + n_gram = self.hparams["engram_max_ngram_size"] + n_head = self.hparams["engram_n_head_per_ngram"] + out = {} + for i in self._engram_ids(): + # the keys are strings in the config + groups = raw.get(str(i)) or raw.get(i) + if groups is None: + raise ValueError(f"HY_V4_FLASH engram_layer_head_vocab_sizes has no entry for layer {i}") + if len(groups) != n_gram - 1 or any(len(g) != n_head for g in groups): + raise ValueError( + f"HY_V4_FLASH engram layer {i}: expected {n_gram - 1} groups of {n_head} bucket " + f"counts, got {[len(g) for g in groups]}") + out[i] = [int(p) for g in groups for p in g] + return out + + def _engram_vocab_map(self) -> tuple[list[int], int]: + """The token -> compressed id map, extended by identity, and the compressed vocab size. + + The map covers only the source tokenizer, which is shorter than the model's vocabulary. The + reference extends it by identity, which puts the trailing reserved ids outside the + compressed range; they hash to a row anyway because the hash is taken modulo the bucket + count, so they stay in range but are not reachable by any trained token. + """ + path = Path(self.dir_model) / "engram_vocab_map.npz" + if not path.is_file(): + raise FileNotFoundError(f"HY_V4_FLASH engram needs {path}") + with np.load(path) as z: + table = z["lookup_table"].astype(np.int64) + meta = json.loads(bytes(z["metadata"]).decode("utf-8").rstrip("\x00")) + + compressed = int(meta["compressed_vocab_size"]) + if int(table.max()) + 1 > compressed: + raise ValueError("HY_V4_FLASH engram lookup table exceeds its compressed vocab size") + if int(meta["engram_seed"]) != int(self.hparams.get("engram_seed", 0)): + raise ValueError("HY_V4_FLASH engram seed disagrees between the config and the vocab map") + + n_vocab = self.hparams["vocab_size"] + token_map = list(map(int, table)) + token_map += list(range(len(token_map), n_vocab)) + return token_map, compressed + + def _engram_multipliers(self, layer_id: int, compressed: int) -> list[int]: + seed = int(self.hparams.get("engram_seed", 0)) + n_gram = self.hparams["engram_max_ngram_size"] + bound = max(1, (np.iinfo(np.int64).max // compressed) // 2) + rng = np.random.default_rng(seed + ENGRAM_SEED_STRIDE*layer_id) + return [int(m) for m in rng.integers(0, bound, size=n_gram)*2 + 1] + + def _engram_set_gguf_parameters(self): + hparams = self.hparams + ids = self._engram_ids() + primes = self._engram_primes() + token_map, compressed = self._engram_vocab_map() + + n_gram = hparams["engram_max_ngram_size"] + n_head = hparams["engram_n_head_per_ngram"] + head_dim, rem = divmod(hparams["engram_n_embed_per_ngram"], n_head) + if rem: + raise ValueError("HY_V4_FLASH engram_n_embed_per_ngram is not a multiple of the head count") + + # offsets rebase each module's buckets into the single shared table; row ids are int32, so + # the loader rejects a table whose last bucket ends past INT32_MAX + offsets: list[int] = [] + multipliers: list[int] = [] + base = 0 + for i in ids: + for p in primes[i]: + offsets.append(base) + base += p + multipliers += self._engram_multipliers(i, compressed) + if base > np.iinfo(np.int32).max: + raise ValueError(f"HY_V4_FLASH engram table needs {base} rows, past the int32 row id limit") + + w = self.gguf_writer + w.add_ple_layers([i - 1 for i in ids]) + w.add_ple_ngram_size(n_gram) + w.add_ple_heads_per_ngram(n_head) + w.add_embedding_length_per_layer_input(head_dim) + w.add_ple_layer_multipliers(multipliers) + w.add_ple_head_offsets(offsets) + w.add_ple_head_vocab_sizes([p for i in ids for p in primes[i]]) + w.add_engram_pad_token_id(hparams["engram_pad_id"]) + w.add_engram_token_map(token_map) + + logger.info( + "HY_V4_FLASH engram: %d module(s) at 0-based layers %s, %d-gram x %d heads x %d dims, " + "%d table rows, compressed vocab %d", + len(ids), [i - 1 for i in ids], n_gram, n_head, head_dim, base, compressed) + + def _n_nextn(self) -> int: + return int(self.hparams.get("num_nextn_predict_layers", 0) or 0) + + def _layer_type_of(self, bid: int) -> str: + """Layer type of any block, including the draft one. + + A draft block defaults to a sparse layer that owns an indexer, which is the shape the + checkpoints are exported with, and `mtp_layer_types` overrides it per draft layer. The + graph only drives a sparse draft block, so refuse anything else rather than write a file + whose head count and rope would silently disagree with it. + """ + n_layer = self.hparams["num_hidden_layers"] + if bid < n_layer: + return self._layer_types()[bid] + + types = self.hparams.get("mtp_layer_types") or [] + if not types: + return "sparse_attention" + + t = types[min(bid - n_layer, len(types) - 1)] + if t != "sparse_attention": + raise ValueError( + f"HY_V4_FLASH: mtp_layer_types gives {t!r} for draft block {bid}, " + "but only 'sparse_attention' draft blocks are supported") + return t + + def _rope_for(self, layer_type: str) -> dict: + params = self.hparams.get("rope_parameters") + if not isinstance(params, dict) or layer_type not in params: + raise ValueError(f"HY_V4_FLASH needs rope_parameters[{layer_type!r}]") + return params[layer_type] + + def set_gguf_parameters(self): + hparams = self.hparams + layer_types = self._layer_types() + is_swa = [t == "sliding_attention" for t in layer_types] + + if not any(is_swa) or all(is_swa): + raise ValueError("HY_V4_FLASH expects a mix of sparse and sliding attention layers") + + # The graph applies the sandwich post-norms on every layer unconditionally, so refuse a + # config that asks for either of them to be skipped: the tensors ship in both cases, so + # the file would convert cleanly and then quietly compute something else. + if not hparams.get("sandwich_norm", False): + raise ValueError("HY_V4_FLASH expects sandwich_norm, the graph always applies the post-norms") + if hparams.get("dsa_skip_attn_post_norm", False): + raise ValueError( + "HY_V4_FLASH: dsa_skip_attn_post_norm is not supported - the graph applies " + "attn_post_norm on the sparse layers too") + + # the base reads rope_parameters["full_attention"], which this config does not have: + # its non-sliding entry is keyed "sparse_attention". Without this the sparse layers + # would silently fall back to the default rope base. + sparse_rope = self._rope_for("sparse_attention") + window_rope = self._rope_for("sliding_attention") + + # hy_v4 warns that it drops the draft block. This model exports it, so keep the key out of + # sight for that one call rather than let the warning mislead. + n_nextn = self.hparams.pop("num_nextn_predict_layers", None) + try: + super().set_gguf_parameters() + finally: + if n_nextn is not None: + self.hparams["num_nextn_predict_layers"] = n_nextn + + self.gguf_writer.add_rope_freq_base(sparse_rope["rope_theta"]) + + # per-layer head count: the window layers carry swa_num_attention_heads + n_head = hparams["num_attention_heads"] + n_head_swa = hparams["swa_num_attention_heads"] + self.gguf_writer.remove_key( + gguf.Keys.Attention.HEAD_COUNT.format(arch=self.gguf_writer.arch)) + # the arrays are read over n_layer_all, so the draft block needs an entry too: it is a + # sparse layer, so the global head count and no window + is_swa_all = is_swa + [False]*self._n_nextn() + self.gguf_writer.add_head_count( + [n_head_swa if swa else n_head for swa in is_swa_all]) + + # the base already derives rope.freq_base_swa from rope_parameters["sliding_attention"] + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern(is_swa_all) + + # routed experts run here, the router and the shared expert stay at hidden_size + latent = hparams.get("routed_expert_hidden_size") + if latent: + self.gguf_writer.add_expert_latent_length(latent) + + if self._n_nextn() and not getattr(self, "no_mtp", False): + self.gguf_writer.add_nextn_predict_layers(self._n_nextn()) + + if self._indexer_enabled(): + # the base writes head_count / key_length / top_k / types off indexer_is_full(); the + # pool size is this model's compress_ratio, and the current partial group is always kept + ratio = hparams["compressed_indexer_compress_ratio"] + self.gguf_writer.add_indexer_kpool(ratio) + self.gguf_writer.add_indexer_kpool_select_tail(True) + + # the reference's own config check: the candidate budget is the group budget expanded + group_topk = hparams.get("compressed_indexer_group_topk") + candidate = hparams.get("compressed_indexer_candidate_topk") + if group_topk and candidate and group_topk * ratio != candidate: + raise ValueError( + f"HY_V4_FLASH indexer: candidate_topk {candidate} != " + f"group_topk {group_topk} x compress_ratio {ratio}") + if hparams["index_topk"] % ratio: + raise ValueError( + f"HY_V4_FLASH indexer: index_topk {hparams['index_topk']} is not a multiple " + f"of compress_ratio {ratio}") + logger.info( + "HY_V4_FLASH indexer: %d pooled groups of %d, keeping the best %d plus the " + "current group (index_topk %d)", + sum(self.indexer_is_full()), ratio, hparams["index_topk"] // ratio, + hparams["index_topk"]) + + if self._engram_enabled(): + self._engram_set_gguf_parameters() + + self._check_mla_geometry() + + logger.info( + "HY_V4_FLASH: %d sparse (%d heads, rope_theta %g) + %d sliding " + "(%d heads, rope_theta %g, window %d) layers", + is_swa.count(False), n_head, sparse_rope["rope_theta"], + is_swa.count(True), n_head_swa, window_rope["rope_theta"], + hparams["sliding_window"], + ) + if hparams.get("engram_layer_ids") and not self._engram_enabled(): + logger.warning( + "HY_V4_FLASH: engram_enable is off, dropping %d Engram layer(s)", + len(hparams["engram_layer_ids"])) + if hparams.get("use_compressed_indexer") and not self._indexer_enabled(): + logger.warning( + "HY_V4_FLASH: dropping the compressed indexer - the sparse layers will " + "attend their whole context instead of the top %s tokens", + hparams.get("index_topk"), + ) + + def _check_mla_geometry(self): + """The window layers must share the sparse layers' MLA geometry. + + Only the head count may differ: the graph keeps one latent cache for both layer types + and reads a single set of MLA dimensions. + """ + hparams = self.hparams + for sparse_key, window_key in ( + ("qk_nope_head_dim", "swa_qk_nope_head_dim"), + ("qk_rope_head_dim", "swa_qk_rope_head_dim"), + ("v_head_dim", "swa_v_head_dim"), + ("qk_head_dim", "swa_qk_head_dim"), + ("kv_lora_rank", "swa_kv_lora_rank"), + ): + if window_key not in hparams: + continue + if hparams[window_key] != hparams[sparse_key]: + raise NotImplementedError( + f"HY_V4_FLASH needs one MLA geometry for both layer types, but " + f"{window_key}={hparams[window_key]} != {sparse_key}={hparams[sparse_key]}" + ) + + def _indexer_enabled(self) -> bool: + return bool(self.hparams.get("use_dsa")) and bool(self.hparams.get("use_compressed_indexer")) + + def indexer_is_full(self) -> list[bool] | None: + """Which layers own a k-pool indexer. + + Derived from layer_types, not from indexer_types: the config marks all 42 layers "full", + which there means the layer carries an indexer *config*, while only the sparse layers + actually hold compressed_indexer weights. llama.cpp's flag means "owns one". + """ + if not self._indexer_enabled(): + return None + n_all = self.hparams["num_hidden_layers"] + self._n_nextn() + return [self._layer_type_of(i) == "sparse_attention" for i in range(n_all)] + + def prepare_tensors(self): + """Validate against this model's own indexer, then skip hy_v4's check. + + hy_v4 looks for the lightning indexer's `self_attn.indexer.*`; this model instead carries + `self_attn.compressed_indexer.*` on its sparse layers, so the inherited check would reject + every file. + """ + self.hparams["num_key_value_heads"] = self.hparams["num_attention_heads"] + + is_full = self.indexer_is_full() + if is_full is not None: + want = set(self.INDEXER_MAP) + n_layer = self.hparams["num_hidden_layers"] + present: dict[int, set[str]] = {} + for name in self.model_tensors: + head, sep, tail = name.partition(".self_attn.compressed_indexer.") + if not sep: + continue + # the draft block carries its own indexer, under a different prefix + if head.startswith("model.layers."): + present.setdefault(int(head.split(".")[2]), set()).add(tail) + elif head.startswith("model.mtp_layers."): + present.setdefault(n_layer + int(head.split(".")[2]), set()).add(tail) + for il, expect in enumerate(is_full): + seen = present.get(il, set()) + if expect and seen != want: + raise ValueError( + f"HY_V4_FLASH layer {il} owns an indexer but is missing " + f"{sorted(want - seen)}" + (f" and carries unexpected {sorted(seen - want)}" + if seen - want else "")) + if not expect and seen: + raise ValueError( + f"HY_V4_FLASH layer {il} owns no indexer but carries {sorted(seen)}") + + # straight to the grandparent: hy_v4's own validation does not apply here + super(HYV4Model, self).prepare_tensors() + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int): + # the graph feeds these straight to ggml_add / ggml_mul as the second operand, which has no + # f16 kernel. deepseek41 keeps engram_gate f32 for the same reason; all three are tiny. + if new_name.endswith((".engram_gate.weight", ".engram_kv_b.weight", ".engram_hc_scale.weight")): + return gguf.GGMLQuantizationType.F32 + # same for the pooled-key position bias, which is a ggml_add operand. glm keeps every + # indexer_compressor_ tensor f32; here the norms already fall under the base class's + # *_norm.weight rule and the rest are matmul operands, so only the ape needs it. + if new_name.endswith(".indexer_compressor_ape.weight"): + return gguf.GGMLQuantizationType.F32 + # The router picks which experts fire, so it stays f32 like every other arch's. The base + # class's rule for it matches the tensor name against the *source* layer id, which no + # longer agrees with the name once the draft block has been renumbered into the trunk's + # range - so without this the draft block's router alone would ship at the source dtype. + if new_name.endswith((".ffn_gate_inp.weight", ".ffn_gate_inp_shexp.weight")): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def _engram_modify(self, data_torch: torch.Tensor, name: str, + bid: int | None) -> Iterable[tuple[str, torch.Tensor]]: + """Fuse a layer's Engram weights into the tensors the graph reads. + + The per-hc key projections and the single value projection become one matmul plus one bias, + and the two gate norms multiply into one, both matching deepseek41's fused layout. Nothing + is emitted until a group is complete, because the fused tensor needs every part. + """ + if bid is None: + raise ValueError(f"HY_V4_FLASH: engram tensor without a layer id: {name}") + + hc = self.hparams["hc_mult"] + n_head = self.hparams["engram_n_head_per_ngram"] * (self.hparams["engram_max_ngram_size"] - 1) + + buf = self._engram_buf.setdefault(bid, {}) + tail = name.split(".engram_layer.", 1)[1] + buf[tail] = data_torch + + tn = self.format_tensor_name + T = gguf.MODEL_TENSOR + + def ready(*names) -> bool: + return all(n in buf for n in names) + + def take(*names): + return [buf.pop(n) for n in names] + + # the fused matmul: hc key projections then the value projection, which is the order the + # graph's views assume + kv_parts = [f"key_projs.{c}.weight" for c in range(hc)] + ["value_proj.weight"] + if ready(*kv_parts): + yield tn(T.ENGRAM_KV, bid), torch.cat(take(*kv_parts), dim=0) + + kvb_parts = [f"key_projs.{c}.bias" for c in range(hc)] + ["value_proj.bias"] + if ready(*kvb_parts): + yield tn(T.ENGRAM_KV_B, bid), torch.stack(take(*kvb_parts), dim=0) + + # engram_gate folds the key and query norms, as deepseek41 does + gate_parts = [f"norm_key.{c}.weight" for c in range(hc)] + [f"norm_query.{c}.weight" for c in range(hc)] + if ready(*gate_parts): + parts = take(*gate_parts) + yield tn(T.ENGRAM_GATE, bid), torch.stack( + [parts[c]*parts[hc + c] for c in range(hc)], dim=0) + + norm_parts = [f"lookup_per_head_norms.{c}.weight" for c in range(n_head)] + if ready(*norm_parts): + yield tn(T.ENGRAM_LOOKUP_NORM, bid), torch.stack(take(*norm_parts), dim=0) + + scale_parts = [f"unified_per_hc_scales.{c}" for c in range(hc)] + if ready(*scale_parts): + yield tn(T.ENGRAM_HC_SCALE, bid), torch.stack(take(*scale_parts), dim=0) + + if ready("unified_pre_gamma"): + yield tn(T.ENGRAM_PRE_GAMMA, bid), take("unified_pre_gamma")[0] + + # every module's buckets share one table, concatenated in layer order, so it can only be + # written once the last module has arrived + if ready("multi_head_embedding.embedding.weight"): + self._engram_tables[bid] = take("multi_head_embedding.embedding.weight")[0] + want = [i - 1 for i in self._engram_ids()] + if all(i in self._engram_tables for i in want): + tables = [self._engram_tables.pop(i) for i in want] + yield (tn(T.PER_LAYER_TOKEN_EMBD), + tables[0] if len(tables) == 1 else torch.cat(tables, dim=0)) + + NEXTN_MAP = { + "eh_proj.weight": gguf.MODEL_TENSOR.NEXTN_EH_PROJ, + "enorm.weight": gguf.MODEL_TENSOR.NEXTN_ENORM, + "hnorm.weight": gguf.MODEL_TENSOR.NEXTN_HNORM, + "final_layernorm.weight": gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + } + + def modify_tensors(self, data_torch: torch.Tensor, name: str, bid: int | None) -> Iterable[tuple[str, torch.Tensor]]: + if name.startswith("model.mtp_layers."): + if not self._n_nextn() or getattr(self, "no_mtp", False): + return + parts = name.split(".", 3) + nbid = self.hparams["num_hidden_layers"] + int(parts[2]) + tail = parts[3] + key = self.NEXTN_MAP.get(tail) + if key is not None: + yield self.format_tensor_name(key, nbid), data_torch + return + # the rest of the block is an ordinary sparse layer, so re-enter under its block index + yield from self.modify_tensors(data_torch, f"model.layers.{nbid}.{tail}", nbid) + return + + if ".compressed_indexer." in name: + # dropped wholesale when the indexer is off, so the GGUF stays loadable as a + # full-attention model + if not self._indexer_enabled(): + return + tail = name.split(".compressed_indexer.", 1)[1] + key = self.INDEXER_MAP.get(tail) + if key is None: + raise ValueError(f"HY_V4_FLASH: unmapped compressed_indexer tensor {name}") + # intra_group_ape is numpy [kpool, head_dim], which gguf writes as ne (head_dim, kpool) + # already, so no transpose is needed anywhere here + yield self.format_tensor_name(key, bid), data_torch + return + + if ".engram_layer." in name: + if not self._engram_enabled(): + return + yield from self._engram_modify(data_torch, name, bid) + return + + # the base splits kv_b_proj with one global head count; here the sliding-window + # layers carry swa_num_attention_heads over the same MLA geometry, so the split + # has to read the head count of this layer + if name.endswith("kv_b_proj.weight") and bid is not None: + hparams = self.hparams + n_head = (hparams["swa_num_attention_heads"] + if self._layer_type_of(bid) == "sliding_attention" + else hparams["num_attention_heads"]) + v_head_dim = hparams["v_head_dim"] + qk_nope_head_dim = hparams["qk_nope_head_dim"] + + expected = n_head * (v_head_dim + qk_nope_head_dim) + if data_torch.shape[0] != expected: + raise ValueError( + f"HY_V4_FLASH layer {bid}: {name} has {data_torch.shape[0]} rows, " + f"expected {expected} = {n_head} heads x ({qk_nope_head_dim} + {v_head_dim})" + ) + + kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1]) + k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1) + k_b = k_b.transpose(1, 2) + + tn = self.format_tensor_name + yield from ModelBase.modify_tensors(self, k_b, tn(gguf.MODEL_TENSOR.ATTN_K_B, bid), bid) + yield from ModelBase.modify_tensors(self, v_b, tn(gguf.MODEL_TENSOR.ATTN_V_B, bid), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/hy_v4_flash_vision.py b/conversion/hy_v4_flash_vision.py new file mode 100644 index 0000000000000000000000000000000000000000..f65313396b4434d7c91f7fe3cfb9500c66ba091a --- /dev/null +++ b/conversion/hy_v4_flash_vision.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import Callable, Iterable, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import MmprojModel, ModelBase, gguf + + +@ModelBase.register("HYV4NextVLForConditionalGeneration") +@ModelBase.example("tencent/Hy4-Flash") +class HYV4FlashVisionModel(MmprojModel): + """Vision tower of HY_V4_FLASH: a SigLIP-shaped encoder with a rmsnorm_mlp_adapter projector. + + The encoder is pre-norm with LayerNorm, biased qkv and a gelu ffn, and has no rope: positions + come from a learned 128x128 grid that the encoder bilinearly resamples to the patch grid. + + The projector merges each 2x2 patch block with a strided conv2d, runs a 1x1 conv2d to the text + hidden size and appends one learned newline embedding to every output row. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None + # there is no image_size, the position-embedding grid is max_image_size / patch_size + self.hparams_vision["image_size"] = self.hparams_vision.get("max_image_size", 2048) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + vcfg = self.hparams_vision + projector_type = vcfg.get("projector_type") + if projector_type != "rmsnorm_mlp_adapter": + raise ValueError(f"unsupported projector_type {projector_type!r}") + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HY_V4_FLASH) + self.gguf_writer.add_vision_use_gelu(True) + # rms_norm_eps is also the LayerNorm eps of the encoder blocks + self.gguf_writer.add_vision_attention_layernorm_eps(vcfg.get("rms_norm_eps", 1e-5)) + self.gguf_writer.add_vision_spatial_merge_size(vcfg.get("spatial_merge_size", 2)) + self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"])) + self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"])) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not name.startswith("vit."): + return None + + return super().filter_tensors(item) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # row 0 of the position embedding is an unused CLS slot, the reference resamples rows 1: + if "position_embedding" in name: + data_torch = data_torch[1:] + yield from super().modify_tensors(data_torch, name, bid) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # force conv weights to F32 or F16 to avoid BF16 IM2COL issues on Metal + if ("mm.0." in new_name or "mm.2." in new_name) and new_name.endswith(".weight"): + return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 3520b434918d5a33ce423f7b916747c01427e3cf..dfb09f9dd982a18735473c064e845a470684de6a 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2125,6 +2125,8 @@ static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggm static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; + const char * batch_inputs_env = getenv("GGML_SCHED_BATCH_HOST_INPUTS"); + const int batch_host_inputs = batch_inputs_env ? atoi(batch_inputs_env) : 0; int prev_backend_id = -1; @@ -2132,6 +2134,9 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; + const auto split_device_type = batch_host_inputs ? ggml_backend_dev_type(ggml_backend_get_device(split_backend)) : GGML_BACKEND_DEVICE_TYPE_CPU; + const bool batch_split_inputs = batch_host_inputs && + (split_device_type == GGML_BACKEND_DEVICE_TYPE_GPU || split_device_type == GGML_BACKEND_DEVICE_TYPE_IGPU); // ensure the previous split's async work has completed before we start // this split, the allocator may have reused buffer regions across splits @@ -2146,6 +2151,42 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s // copy the input tensors to the split backend // the weights in host memory are copied last, so that the copy callback can read the other inputs of the split for (int input_id = 0; input_id < split->n_inputs; input_id++) { + if (batch_split_inputs) { + std::vector srcs; + std::vector dsts; + for (int j = input_id; j < split->n_inputs; ++j) { + ggml_tensor * input = split->inputs[j]; + ggml_backend_buffer_t buf = input->view_src ? input->view_src->buffer : input->buffer; + if (ggml_backend_sched_is_host_weight(input) || !buf || !ggml_backend_buffer_is_host(buf)) { + break; + } + ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, input); + if (!(input->flags & GGML_TENSOR_FLAG_INPUT) && + (batch_host_inputs < 2 || !input_backend || + ggml_backend_dev_type(ggml_backend_get_device(input_backend)) != GGML_BACKEND_DEVICE_TYPE_CPU)) { + break; + } + srcs.push_back(input); + dsts.push_back(tensor_copy(input, split_backend_id, sched->cur_copy)); + } + if (srcs.size() > 1) { + for (size_t j = 0; j < srcs.size(); ++j) { + ggml_tensor * input = split->inputs[input_id + j]; + if (!(input->flags & GGML_TENSOR_FLAG_INPUT)) { + ggml_backend_synchronize(ggml_backend_sched_get_tensor_backend(sched, input)); + } + } + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + // Finish all copies before the caller can change these inputs. + ggml_backend_tensor_copy_n(srcs.data(), dsts.data(), srcs.size()); + input_id += (int) srcs.size() - 1; + continue; + } + } if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) { ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]); } diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index a46c4d3f096fb4e00bd0a6f000a545fd3fd33be3..6702a5d01ca295e4409340c07dd45c0cd45873b0 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1271,36 +1271,6 @@ struct ggml_tensor_extra_gpu { #define USE_CUDA_GRAPH #endif -struct ggml_cuda_graph { -#ifdef USE_CUDA_GRAPH - ~ggml_cuda_graph() { - for (cudaGraphExec_t instance : instances) { - CUDA_CHECK(cudaGraphExecDestroy(instance)); - } - } - // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued - std::vector instances; - size_t num_nodes = 0; - std::vector nodes; - bool disable_due_to_gpu_arch = false; - bool warmup_complete = false; - uint64_t uid = 0; - int64_t last_used_time = 0; - struct node_properties { - ggml_tensor node; - void * node_src_data_ptrs[GGML_MAX_SRC]; - int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; - size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; - }; - std::vector node_props; - - bool is_enabled() const { - static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); - return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env); - } -#endif -}; - struct ggml_cuda_concurrent_event { std::vector join_events; cudaEvent_t fork_event = nullptr; @@ -1445,13 +1415,46 @@ struct ggml_cuda_concurrent_event { }; struct ggml_cuda_stream_context { + bool moe_initialized = false; std::unordered_map concurrent_events; void reset() { + moe_initialized = false; concurrent_events.clear(); } }; +struct ggml_cuda_graph { + ggml_cuda_stream_context concurrent_stream_context; +#ifdef USE_CUDA_GRAPH + ~ggml_cuda_graph() { + for (cudaGraphExec_t instance : instances) { + CUDA_CHECK(cudaGraphExecDestroy(instance)); + } + } + // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued + std::vector instances; + size_t num_nodes = 0; + std::vector nodes; + bool disable_due_to_gpu_arch = false; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + std::vector node_props; + + bool is_enabled() const { + static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr); + return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env); + } +#endif +}; + struct ggml_backend_cuda_context { int device; std::string name; @@ -1535,7 +1538,17 @@ struct ggml_backend_cuda_context { cudaStream_t stream() { return stream(device, curr_stream_no); } - ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } + ggml_cuda_stream_context & stream_context(const void * graph_key = nullptr) { +#ifdef USE_CUDA_GRAPH + static const bool per_graph = [] { const char * v = getenv("GGML_CUDA_MOE_CONCURRENT"); return v && atoi(v) != 0; }(); + if (per_graph && graph_key) { + return cuda_graph(graph_key)->concurrent_stream_context; + } +#else + GGML_UNUSED(graph_key); +#endif + return concurrent_stream_context; + } cublasHandle_t cublas_handle() { if (cublas_handles[device][curr_stream_no] == nullptr) { diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 90a06b97bbc6e9f3936827cbf61e8c191303e043..22c4f70e0322e13d93ed381df7cd877994129caa 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -41,6 +41,21 @@ static __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne cpy_1(cx + x_offset, cdst + dst_offset); } +static __global__ void cpy_f32_rows_vec4(const char * src, float4 * dst, const uint32_t nvec, + const uint3 ne0_vec, const uint3 ne1, const uint3 ne2, const size_t nb1, const size_t nb2, const size_t nb3) { + ggml_cuda_pdl_lc(); + const uint32_t i = blockIdx.x*blockDim.x + threadIdx.x; + if (i >= nvec) { + return; + } + const uint2 p0 = fast_div_modulo(i, ne0_vec); + const uint2 p1 = fast_div_modulo(p0.x, ne1); + const uint2 p2 = fast_div_modulo(p1.x, ne2); + const size_t offset = size_t(p0.y)*sizeof(float4) + size_t(p1.y)*nb1 + size_t(p2.y)*nb2 + size_t(p2.x)*nb3; + ggml_cuda_pdl_sync(); + dst[i] = *(const float4 *) (src + offset); +} + template static __global__ void cpy_scalar_transpose(const char * cx, char * cdst, const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, @@ -478,7 +493,19 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch, mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream)); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { - if (can_be_transposed) { + static const bool use_vec = [] { + const char * value = getenv("GGML_CUDA_CPY_F32_VEC"); + return value && atoi(value) != 0; + }(); + if (use_vec && ne >= 4096 && ne/4 <= INT_MAX && nb00 == sizeof(float) && ne00 % 4 == 0 && + ne01 <= INT_MAX && ne02 <= INT_MAX && ggml_is_contiguous(src1) && + (nb01 | nb02 | nb03 | (uintptr_t) src0_ddc | (uintptr_t) src1_ddc) % sizeof(float4) == 0) { + const uint32_t nvec = ne/4; + const dim3 blocks((nvec + CUDA_CPY_BLOCK_SIZE - 1)/CUDA_CPY_BLOCK_SIZE); + const ggml_cuda_kernel_launch_params params(blocks, CUDA_CPY_BLOCK_SIZE, 0, main_stream); + ggml_cuda_kernel_launch(cpy_f32_rows_vec4, params, src0_ddc, (float4 *) src1_ddc, nvec, + init_fastdiv_values(ne00/4), init_fastdiv_values(ne01), init_fastdiv_values(ne02), size_t(nb01), size_t(nb02), size_t(nb03)); + } else if (can_be_transposed) { ggml_cpy_scalar_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else { diff --git a/ggml/src/ggml-cuda/dsv4-hc.cu b/ggml/src/ggml-cuda/dsv4-hc.cu index 2d0b3cfeaed7b07bb5467d150c98e410beb1ea1a..f40605b8fd6b6726a3a2106dd960501eaad46d0b 100644 --- a/ggml/src/ggml-cuda/dsv4-hc.cu +++ b/ggml/src/ggml-cuda/dsv4-hc.cu @@ -4,6 +4,110 @@ static constexpr int DSV4_HC = 4; +template +static __global__ void dsv4_hc_gate_consumer_f32(const float * mixes, const float * factor, const float * bias, + const float * x, const float * residual, float * dst, int n_embd, + float s0, float b0, float s1, float b1, bool extra, float output_scale) { + __shared__ float gates[DSV4_HC]; + ggml_cuda_pdl_lc(); + ggml_cuda_pdl_sync(); + if (threadIdx.x < DSV4_HC) { + const int i = threadIdx.x; + const float z = __fadd_rn(__fmul_rn(mixes[i], factor[0]), bias[i]); + float v = 1.0f / (1.0f + expf(-z)); + v = fmaf(s0, v, b0); + gates[i] = extra ? fmaf(s1, v, b1) : v; + } + __syncthreads(); + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if constexpr (post) { + if (i < DSV4_HC*n_embd) { + float sum = x[i % n_embd] * gates[i / n_embd]; + sum += residual[i]; + dst[i] = sum; + } + } else { + if (i < n_embd) { + float sum = 0.0f; + for (int h = 0; h < DSV4_HC; ++h) { + sum += x[i + h*n_embd] * gates[h]; + } + dst[i] = output_scale * sum; + } + } +} + +bool ggml_cuda_op_dsv4_hc_gate_consumer(ggml_backend_cuda_context & ctx, const ggml_tensor * mul, + const ggml_tensor * add, const ggml_tensor * scale, const ggml_tensor * scale_extra, ggml_tensor * dst) { + const bool post = dst->op == GGML_OP_DSV4_HC_POST; + if (!post && dst->op != GGML_OP_DSV4_HC_PRE) { + return false; + } + const ggml_tensor * gate = scale_extra ? scale_extra : scale; + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * residual = post ? dst->src[1] : nullptr; + const int64_t n_embd = x->ne[0]; + if (ggml_nelements(gate) != DSV4_HC || n_embd > INT_MAX / DSV4_HC || + dst->src[post ? 2 : 1] != gate || dst->type != GGML_TYPE_F32 || x->type != GGML_TYPE_F32 || + !ggml_is_contiguous(x) || !ggml_is_contiguous(dst)) { + return false; + } + if (post) { + if (dst->src[3] || ggml_nelements(x) != n_embd || residual->type != GGML_TYPE_F32 || + !ggml_is_contiguous(residual) || residual->ne[0] != n_embd || residual->ne[1] != DSV4_HC || + ggml_nelements(residual) != DSV4_HC*n_embd || ggml_nelements(dst) != DSV4_HC*n_embd) { + return false; + } + } else if (ggml_get_op_params_i32(dst, 1) != 0 || x->ne[1] != DSV4_HC || + ggml_nelements(x) != DSV4_HC*n_embd || ggml_nelements(dst) != n_embd) { + return false; + } + const int64_t n = ggml_nelements(dst); + const ggml_cuda_kernel_launch_params params((n + 255) / 256, 256, 0, ctx.stream()); + auto kernel = post ? dsv4_hc_gate_consumer_f32 : dsv4_hc_gate_consumer_f32; + ggml_cuda_kernel_launch(kernel, params, + (const float *) mul->src[0]->data, (const float *) mul->src[1]->data, (const float *) add->src[1]->data, + (const float *) x->data, residual ? (const float *) residual->data : nullptr, (float *) dst->data, (int) n_embd, + ggml_get_op_params_f32(scale, 0), ggml_get_op_params_f32(scale, 1), + scale_extra ? ggml_get_op_params_f32(scale_extra, 0) : 1.0f, + scale_extra ? ggml_get_op_params_f32(scale_extra, 1) : 0.0f, scale_extra != nullptr, + post ? 1.0f : ggml_get_op_params_f32(dst, 0)); + return true; +} + +static __global__ void dsv4_hc_gate_f32(const float * x, const float * scale, const float * bias, + float * dst, int64_t n, int64_t stride, float s0, float b0, float s1, float b1, bool extra) { + ggml_cuda_pdl_lc(); + const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) { + return; + } + ggml_cuda_pdl_sync(); + const int64_t col = i % DSV4_HC; + // Keep the rounding of the separate MUL and ADD kernels. + const float z = __fadd_rn(__fmul_rn(x[(i / DSV4_HC)*stride + col], scale[0]), bias[col]); + float v = 1.0f / (1.0f + expf(-z)); + v = fmaf(s0, v, b0); + if (extra) { + v = fmaf(s1, v, b1); + } + dst[i] = v; +} + +void ggml_cuda_op_dsv4_hc_gate(ggml_backend_cuda_context & ctx, const ggml_tensor * mul, + const ggml_tensor * add, ggml_tensor * scale, ggml_tensor * scale_extra) { + const ggml_tensor * x = mul->src[0]; + ggml_tensor * dst = scale_extra ? scale_extra : scale; + const int64_t n = ggml_nelements(dst); + const ggml_cuda_kernel_launch_params params((n + 255) / 256, 256, 0, ctx.stream()); + ggml_cuda_kernel_launch(dsv4_hc_gate_f32, params, + (const float *) x->data, (const float *) mul->src[1]->data, (const float *) add->src[1]->data, + (float *) dst->data, n, x->nb[1] / sizeof(float), + ggml_get_op_params_f32(scale, 0), ggml_get_op_params_f32(scale, 1), + scale_extra ? ggml_get_op_params_f32(scale_extra, 0) : 1.0f, + scale_extra ? ggml_get_op_params_f32(scale_extra, 1) : 0.0f, scale_extra != nullptr); +} + static __device__ void dsv4_hc_comb_norm_cols(float * comb, float eps) { for (int idst = 0; idst < DSV4_HC; ++idst) { diff --git a/ggml/src/ggml-cuda/dsv4-hc.cuh b/ggml/src/ggml-cuda/dsv4-hc.cuh index 1885e76659d5a67bbc5c829e91f1df0ffecf6099..af2516a240cfcdd15c9135cfa91d3cfb65ffd23b 100644 --- a/ggml/src/ggml-cuda/dsv4-hc.cuh +++ b/ggml/src/ggml-cuda/dsv4-hc.cuh @@ -3,6 +3,10 @@ void ggml_cuda_op_dsv4_hc_comb(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +void ggml_cuda_op_dsv4_hc_gate(ggml_backend_cuda_context & ctx, const ggml_tensor * mul, + const ggml_tensor * add, ggml_tensor * scale, ggml_tensor * scale_extra); +bool ggml_cuda_op_dsv4_hc_gate_consumer(ggml_backend_cuda_context & ctx, const ggml_tensor * mul, + const ggml_tensor * add, const ggml_tensor * scale, const ggml_tensor * scale_extra, ggml_tensor * dst); // gate_scale_in/gate_scale_out: the SCALE nodes of a fused SCALE -> SIGMOID -> SCALE chain feeding src[2] void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_tensor * gate_scale_in = nullptr, const ggml_tensor * gate_scale_out = nullptr); diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index beb877941c75ef8646f87c61d0e75d1da65b881a..9b06591930db2aa518a095c3d3e0e114fac122ff 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1156,7 +1156,13 @@ void launch_fattn( dim3 blocks_num; if (stream_k) { // Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven. - const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask; + static const bool whole_tiles_576 = [] { + const char * value = getenv("GGML_CUDA_FA_576_WHOLE_TILES"); + return value && atoi(value) != 0; + }(); + const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && scan_mask && + ((cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload) || + (whole_tiles_576 && (cc == GGML_CUDA_CC_BLACKWELL || cc == GGML_CUDA_CC_DGX_SPARK) && Q->ne[0] == 576 && Q->ne[1] > 16 && !use_sparse)); auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 06956ce7a3d9f5651735b89513bbdb9097b647a4..2f358d02ab5009e8a7160bf0fe0783eb0f0c4e12 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -929,6 +929,22 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co static bool ggml_backend_cuda_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) { ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context; + bool all_host = true; + for (size_t i = 0; i < n_tensors; i++) { + ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; + all_host &= ggml_backend_buffer_is_host(src_buf); + } + if (all_host) { + ggml_cuda_set_device(dst_ctx->device); + for (size_t i = 0; i < n_tensors; i++) { + if (ggml_nbytes(srcs[i]) != 0) { + CUDA_CHECK(cudaMemcpyAsync(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i]), cudaMemcpyHostToDevice, cudaStreamPerThread)); + } + } + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; + } + for (size_t i = 0; i < n_tensors; i++) { ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer; if (!ggml_backend_buffer_is_cuda(src_buf)) { @@ -2057,6 +2073,13 @@ static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggm return false; } + if (x_up->type == GGML_TYPE_NVFP4) { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_SPARK_MMQ_FUSION"); + if (!enabled || up->op != GGML_OP_MUL_MAT_ID || !blackwell_mma_available(ggml_cuda_info().devices[device].cc)) { + return false; + } + } + // The fused kernel is only compiled for weights whose rows fill whole tiles (no fallback). if (ggml_cuda_mmq_needs_fallback(x_up->ne[1])) { return false; @@ -2105,11 +2128,15 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t return false; } + if (x_up->type == GGML_TYPE_NVFP4) { + return ggml_cuda_mmq_spark_fusion_supported(up, gate); + } + // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient. return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], ncols, nchannels, device); } -static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_mmvq_cache * cache = nullptr) { GGML_TENSOR_BINARY_OP_LOCALS if (ggml_cuda_op_mul_mat_use_fwht(dst) && ggml_cuda_op_fwht(ctx, src1, dst)) { @@ -2154,7 +2181,7 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor return; } if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { - ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst, nullptr, cache); return; } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { @@ -2197,7 +2224,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c return true; } -static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_mmvq_cache * cache) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; const ggml_tensor * ids = dst->src[2]; @@ -2217,7 +2244,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * if (ggml_is_quantized(src0->type)) { const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc); if (ne2 <= mmvq_mmid_max) { - ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst, nullptr, cache); return; } } else { @@ -2356,7 +2383,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * nb1, nb2, nb3, stream); } -static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) { +static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst, ggml_cuda_mmvq_cache * cache) { switch (dst->op) { case GGML_OP_ARGMAX: ggml_cuda_argmax(ctx, dst); @@ -2549,10 +2576,10 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg ggml_cuda_op_rms_norm_back(ctx, dst); break; case GGML_OP_MUL_MAT: - ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst); + ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst, cache); break; case GGML_OP_MUL_MAT_ID: - ggml_cuda_mul_mat_id(ctx, dst); + ggml_cuda_mul_mat_id(ctx, dst, cache); break; case GGML_OP_OUT_PROD: ggml_cuda_out_prod(ctx, dst); @@ -3940,7 +3967,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } // try and fuse nodes and return the number of nodes to skip -static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { +static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i, ggml_cuda_mmvq_cache * cache) { if (ggml_cuda_fusion_disabled()) { return 0; @@ -3948,7 +3975,51 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; - if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && + static const bool fuse_hc_gate = ggml_cuda_env_flag("GGML_CUDA_FUSE_HC_GATE"); + if (fuse_hc_gate && node->op == GGML_OP_MUL) { + // A bias view can be visited between MUL and ADD; it has no device work. + const int view = i + 1 < cgraph->n_nodes && cgraph->nodes[i + 1]->op == GGML_OP_VIEW ? 1 : 0; + const int idxs[] = { i, i + 1 + view, i + 2 + view, i + 3 + view, i + 4 + view }; + const ggml_op ops[] = { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_SCALE }; + const bool extra = ggml_can_fuse_ext(cgraph, idxs, ops, 5); + if (extra || ggml_can_fuse_ext(cgraph, idxs, ops, 4)) { + ggml_tensor * add = cgraph->nodes[idxs[1]]; + ggml_tensor * sigmoid = cgraph->nodes[idxs[2]]; + ggml_tensor * scale = cgraph->nodes[idxs[3]]; + const int output_idx = idxs[extra ? 4 : 3]; + ggml_tensor * dst = cgraph->nodes[output_idx]; + const ggml_tensor * x = node->src[0]; + const ggml_tensor * factor = node->src[1]; + const ggml_tensor * bias = add->src[1]; + if (add->src[0] == node && sigmoid->src[0] == add && scale->src[0] == sigmoid && + (!extra || dst->src[0] == scale) && ggml_get_unary_op(sigmoid) == GGML_UNARY_OP_SIGMOID && + x->type == GGML_TYPE_F32 && factor->type == GGML_TYPE_F32 && bias->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && sigmoid->type == GGML_TYPE_F32 && + scale->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + x->ne[0] == 4 && x->ne[2] == 1 && x->ne[3] == 1 && ggml_is_contiguous_rows(x) && + ggml_nelements(factor) == 1 && bias->ne[0] == 4 && ggml_nelements(bias) == 4 && ggml_is_contiguous(bias) && + ggml_are_same_shape(x, dst) && ggml_is_contiguous(dst) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, output_idx - i + 1, &output_idx, 1)) { + static const bool fuse_consumer = ggml_cuda_env_flag("GGML_CUDA_FUSE_HC_CONSUMER"); + const int consumer_idx = output_idx + 1; + if (fuse_consumer && consumer_idx < cgraph->n_nodes) { + ggml_tensor * consumer = cgraph->nodes[consumer_idx]; + if (consumer->op == GGML_OP_DSV4_HC_PRE || consumer->op == GGML_OP_DSV4_HC_POST) { + if (ggml_node_has_n_uses(cgraph, output_idx, 1) && + (consumer->flags & GGML_TENSOR_FLAG_COMPUTE) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, consumer_idx - i + 1, &consumer_idx, 1) && + ggml_cuda_op_dsv4_hc_gate_consumer(*cuda_ctx, node, add, scale, extra ? dst : nullptr, consumer)) { + return consumer_idx - i; + } + } + } + ggml_cuda_op_dsv4_hc_gate(*cuda_ctx, node, add, scale, extra ? dst : nullptr); + return output_idx - i; + } + } + } + + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context(cgraph->n_nodes ? cgraph->nodes[0] : nullptr).concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { const int outputs[] = { i + 2, i + 5 }; @@ -3963,7 +4034,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion.shared_up = shared->src[1]->src[0]; fusion.shared_gate = shared->src[0]->src[0]; fusion.shared_dst = shared; - ggml_cuda_mul_mat_vec_q(*cuda_ctx, up->src[0], up->src[1], up->src[2], routed, &fusion); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, up->src[0], up->src[1], up->src[2], routed, &fusion, cache); return 5; } } @@ -4311,7 +4382,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = n_ops; break; @@ -4404,8 +4475,15 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + if (!with_bias && ggml_cuda_mmq_spark_fusion_supported(up_n, gate_n)) { + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = n_ops; break; @@ -4475,7 +4553,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = 5; break; @@ -4514,7 +4592,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.glu_op = ggml_get_glu_op(glu); fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = 3; break; @@ -4614,7 +4692,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fusion_data.x_scale = scale; if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = n_ops; break; @@ -4679,7 +4757,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data); + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data, cache); fused_mul_mat_vec = true; fused_node_count = 2; break; @@ -4750,13 +4828,83 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 0; } +// Reuse only vectors whose storage is unchanged between all consumers in this graph. +static void ggml_cuda_prepare_mmvq_cache(ggml_backend_cuda_context & ctx, const ggml_cgraph * graph, ggml_cuda_mmvq_cache & cache, + const std::unordered_map * stream_mapping = nullptr, int stream = 0) { + struct uses { int first; int last; }; + std::unordered_map inputs; + for (int i = 0; i < graph->n_nodes; ++i) { + const ggml_tensor * node = graph->nodes[i]; + if (stream_mapping) { + auto it = stream_mapping->find(node); + if ((it == stream_mapping->end() ? 0 : it->second) != stream) { + continue; + } + } + if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) { + continue; + } + const ggml_tensor * x = node->src[1]; + if (!ggml_is_quantized(node->src[0]->type) || x->type != GGML_TYPE_F32 || + x->ne[0] < 32 || x->ne[0] > 65536 || ggml_nelements(x) != x->ne[0] || !ggml_is_contiguous(x)) { + continue; + } + auto result = inputs.emplace(ggml_cuda_mmvq_cache_key(x), uses{i, i}); + result.first->second.last = i; + } + size_t size = 0; + for (const auto & input : inputs) { + const ggml_tensor * x = input.first; + const uses range = input.second; + if (range.first == range.last) { + continue; + } + const uintptr_t start = (uintptr_t) x->data; + const size_t bytes = ggml_nbytes(x); + auto overlaps = [&](const ggml_tensor * dst) { + const uintptr_t other = (uintptr_t) dst->data; + const size_t other_bytes = ggml_nbytes(dst); + return other <= start ? start - other < other_bytes : other - start < bytes; + }; + bool unchanged = true; + for (int i = range.first; i <= range.last && unchanged; ++i) { + const ggml_tensor * node = graph->nodes[i]; + if (ggml_cuda_is_view_or_noop(node)) { + continue; + } + unchanged = !overlaps(node); + // These operations write through an explicit destination source. + if (node->op == GGML_OP_CPY || node->op == GGML_OP_SET_ROWS) { + unchanged = unchanged && !overlaps(node->src[node->op == GGML_OP_CPY ? 1 : 2]); + } + // MMVQ can clear the allocation padding of temporary weights. + if ((node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) && + ggml_backend_buffer_get_usage(node->src[0]->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_is_quantized(node->src[0]->type)) { + unchanged = false; + } + } + if (unchanged) { + const size_t quantized_bytes = GGML_PAD(x->ne[0], MATRIX_ROW_PADDING) / QK8_1 * sizeof(block_q8_1); + if (size + quantized_bytes > 8*1024*1024) { + break; + } + cache.entries.emplace(x, ggml_cuda_mmvq_cache::entry{size}); + size += quantized_bytes; + } + } + if (size) { + cache.data.alloc(ctx.pool(), size); + } +} + static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, const void * graph_key) { bool graph_evaluated_or_captured = false; // flag used to determine whether it is an integrated_gpu const bool integrated = ggml_cuda_info().devices[cuda_ctx->device].integrated; - ggml_cuda_stream_context & stream_ctx = cuda_ctx->stream_context(); + ggml_cuda_stream_context & stream_ctx = cuda_ctx->stream_context(cgraph->n_nodes ? cgraph->nodes[0] : nullptr); bool is_concurrent_event_active = false; ggml_cuda_concurrent_event * concurrent_event = nullptr; bool should_launch_concurrent_events = false; @@ -4855,6 +5003,27 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud stream_ctx.concurrent_events.clear(); } + std::array mmvq_caches; + static const bool reuse_q8 = ggml_cuda_env_flag("GGML_CUDA_REUSE_Q8"); + static const bool moe_concurrent = ggml_cuda_env_flag("GGML_CUDA_MOE_CONCURRENT"); + const bool prepare_cache = reuse_q8 && (!should_launch_concurrent_events || moe_concurrent); + ggml_cuda_mmvq_cache * cache = nullptr; + if (prepare_cache) { + std::unordered_map node_streams; + int streams = 1; + if (should_launch_concurrent_events) { + for (const auto & item : stream_ctx.concurrent_events) { + streams = std::max(streams, item.second.n_streams + 1); + node_streams.insert(item.second.stream_mapping.begin(), item.second.stream_mapping.end()); + } + } + GGML_ASSERT(streams <= GGML_CUDA_MAX_STREAMS); + for (int stream = 0; stream < streams; ++stream) { + ggml_cuda_prepare_mmvq_cache(*cuda_ctx, cgraph, mmvq_caches[stream], &node_streams, stream); + } + cache = &mmvq_caches[0]; + } + for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; #ifdef USE_CUDA_GRAPH @@ -4906,7 +5075,20 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud continue; } - int nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, cgraph, i); + if (prepare_cache) { + cache = &mmvq_caches[cuda_ctx->curr_stream_no]; + } + ggml_cgraph fusion_graph = *cgraph; + if (is_concurrent_event_active) { + for (int j = i + 1; j < cgraph->n_nodes; ++j) { + auto mapping = concurrent_event->stream_mapping.find(cgraph->nodes[j]); + if (mapping == concurrent_event->stream_mapping.end() || mapping->second != cuda_ctx->curr_stream_no) { + fusion_graph.n_nodes = j; + break; + } + } + } + int nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, &fusion_graph, i, cache); if (nodes_to_skip != 0) { #ifdef GGML_CUDA_DEBUG @@ -4935,7 +5117,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud GGML_UNUSED(integrated); #endif // NDEBUG - bool ok = ggml_cuda_compute_forward(*cuda_ctx, node); + bool ok = ggml_cuda_compute_forward(*cuda_ctx, node, cache); if (!ok) { GGML_LOG_ERROR("%s: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } @@ -4996,6 +5178,142 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co } #endif // USE_CUDA_GRAPH +// Graph eviction drops stream metadata. Rebuild only when the existing allocation is safe. +static bool ggml_cuda_moe_stream_ranges_safe(const ggml_cuda_concurrent_event & event) { + const auto overlaps = [](const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->data || !b->data || ggml_is_empty(a) || ggml_is_empty(b)) { + return false; + } + const uintptr_t x = (uintptr_t) a->data; + const uintptr_t y = (uintptr_t) b->data; + return x <= y ? y - x < ggml_nbytes(a) : x - y < ggml_nbytes(b); + }; + for (const auto & [writer, stream] : event.stream_mapping) { + if (ggml_is_empty(writer) || writer->op == GGML_OP_NONE || writer->op == GGML_OP_VIEW || + writer->op == GGML_OP_RESHAPE || writer->op == GGML_OP_TRANSPOSE || writer->op == GGML_OP_PERMUTE) { + continue; + } + if (!writer->data || writer->op == GGML_OP_CPY || writer->op == GGML_OP_SET_ROWS) { + return false; + } + for (const auto & [other, other_stream] : event.stream_mapping) { + if (stream == other_stream) { + continue; + } + if (overlaps(writer, other)) { + return false; + } + for (const ggml_tensor * input : other->src) { + if (overlaps(writer, input)) { + return false; + } + } + } + } + return true; +} + +static void ggml_cuda_prepare_moe_concurrency(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, + ggml_backend_graph_optimize_params * params) { + auto & stream_context = cuda_ctx->stream_context(cgraph->n_nodes ? cgraph->nodes[0] : nullptr); + stream_context.moe_initialized = true; + std::unordered_map node_indices; + for (int i = 0; i < cgraph->n_nodes; ++i) { + node_indices[cgraph->nodes[i]] = i; + } + const auto is_noop = [](const ggml_tensor * node) { + return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || + node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE; + }; + std::vector> concurrent_node_ranges; + for (int join_idx = 0; join_idx < cgraph->n_nodes; ++join_idx) { + ggml_tensor * join = cgraph->nodes[join_idx]; + if (join->op != GGML_OP_ADD || ggml_nrows(join) != 1) { + continue; + } + const ggml_tensor * shared = join->src[1]; + if (!shared || shared->op != GGML_OP_MUL_MAT || !strstr(shared->name, "ffn_shexp")) { + continue; + } + const ggml_tensor * glu = shared->src[1]; + if (glu->op != GGML_OP_GLU || !glu->src[0] || !glu->src[1] || + glu->src[0]->op != GGML_OP_MUL_MAT || glu->src[1]->op != GGML_OP_MUL_MAT || + glu->src[0]->src[1] != glu->src[1]->src[1]) { + continue; + } + const ggml_tensor * root = glu->src[0]->src[1]; + auto root_it = node_indices.find(root); + if (root_it == node_indices.end() || ggml_nrows(root) != 1) { + continue; + } + const int fork_idx = root_it->second; + if (fork_idx >= join_idx || (!concurrent_node_ranges.empty() && fork_idx <= concurrent_node_ranges.back().second)) { + continue; + } + std::unordered_map branches; + bool indexed = false; + for (int branch = 0; branch < 2; ++branch) { + std::vector pending{join->src[branch]}; + while (!pending.empty()) { + const ggml_tensor * node = pending.back(); + pending.pop_back(); + auto pos = node_indices.find(node); + if (pos == node_indices.end() || pos->second <= fork_idx || pos->second >= join_idx) { + continue; + } + int & mask = branches[node]; + if (mask & (1 << branch)) { + continue; + } + mask |= 1 << branch; + indexed = indexed || (branch == 0 && node->op == GGML_OP_MUL_MAT_ID); + for (const ggml_tensor * input : node->src) { + if (input) { + pending.push_back(input); + } + } + } + } + if (!indexed) { + continue; + } + bool valid = true; + ggml_cuda_concurrent_event event(2); + event.join_node = join; + for (int j = fork_idx + 1; j < join_idx; ++j) { + const ggml_tensor * node = cgraph->nodes[j]; + const int mask = branches[node]; + if (mask == 3 || (mask == 0 && !is_noop(node))) { + valid = false; + break; + } + event.stream_mapping[node] = mask == 2 ? 2 : 1; + event.original_order.push_back(node); + } + if (!valid) { + continue; + } + // Both branches retain their intermediates until the join, including view sources. + if (params) { + for (int j = fork_idx; j < join_idx; ++j) { + params->add_alloc_dep(params->user_data, cgraph->nodes[j], join); + for (ggml_tensor * input : cgraph->nodes[j]->src) { + if (input) { + params->add_alloc_dep(params->user_data, input, join); + } + } + } + } else if (!ggml_cuda_moe_stream_ranges_safe(event)) { + continue; + } + stream_context.concurrent_events.emplace(root, std::move(event)); + concurrent_node_ranges.emplace_back(fork_idx, join_idx); + } + if (!params) { + GGML_LOG_DEBUG("%s: restored %zu concurrent MoE regions\n", __func__, stream_context.concurrent_events.size()); + } +} + static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; @@ -5014,6 +5332,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, if (graph->is_enabled()) { const bool graph_compatible = ggml_cuda_graph_check_compability(cgraph); if (graph_compatible) { + if (ggml_cuda_env_flag("GGML_CUDA_MOE_CONCURRENT") && !graph->concurrent_stream_context.moe_initialized) { + ggml_cuda_prepare_moe_concurrency(cuda_ctx, cgraph, nullptr); + } const bool properties_changed = ggml_cuda_graph_update_required(cuda_ctx, cgraph); if (!graph->warmup_complete) { @@ -5224,11 +5545,12 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph return env != nullptr && atoi(env) == 1; }(); - if (!enable_graph_optimization) { + static const bool enable_moe_concurrent = ggml_cuda_env_flag("GGML_CUDA_MOE_CONCURRENT"); + if (!enable_graph_optimization && !enable_moe_concurrent) { return; } - ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context(); + ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context(cgraph->n_nodes ? cgraph->nodes[0] : nullptr); stream_context.reset(); if (!use_cuda_graph) { @@ -5293,6 +5615,16 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph // store {fork_idx, join_idx} std::vector> concurrent_node_ranges; + if (enable_moe_concurrent) { + ggml_cuda_prepare_moe_concurrency(cuda_ctx, cgraph, params); + for (const auto & [root, event] : stream_context.concurrent_events) { + concurrent_node_ranges.emplace_back(node_indices.at(root), node_indices.at(event.join_node)); + } + } + if (!enable_graph_optimization) { + return; + } + for (const auto & [root_node, count] : fan_out) { if (count >= min_fan_out && count <= max_fan_out) { const int root_node_idx = node_indices[root_node]; @@ -5410,7 +5742,7 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph concurrent_event.original_order.push_back(cgraph->nodes[i]); } - std::unordered_map & concurrent_events = cuda_ctx->stream_context().concurrent_events; + std::unordered_map & concurrent_events = cuda_ctx->stream_context(cgraph->n_nodes ? cgraph->nodes[0] : nullptr).concurrent_events; GGML_ASSERT(concurrent_events.find(root_node) == concurrent_events.end()); concurrent_events.emplace(root_node, std::move(concurrent_event)); GGML_LOG_DEBUG("Adding stream at node %s %p\n", root_node->name, root_node); diff --git a/ggml/src/ggml-cuda/lightning-indexer.cu b/ggml/src/ggml-cuda/lightning-indexer.cu index 429d2e200c8405a223887e27d103c5ab6e643ad2..be3e2d4ebb8301f70ce6a22abd0a84ddc5c7cdd0 100644 --- a/ggml/src/ggml-cuda/lightning-indexer.cu +++ b/ggml/src/ggml-cuda/lightning-indexer.cu @@ -697,6 +697,49 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_F32) GGML_ABORT("fatal error"); } + } else if (n_embd == 128 && n_head == 16) { + // hy_v4_flash: 16 indexer heads, which both the 8-head wmma tile and the N_HEAD/4 inner + // loop of the vector kernel still cover +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { + // use wmma kernel + constexpr int K_VECS_PER_BLOCK = 32; + constexpr int WARPS_PER_BLOCK = 8; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 16, k, GGML_TYPE_Q8_0) + GGML_ABORT("fatal error"); + } else { +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + { +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + // use vector kernel + constexpr int K_VECS_PER_WARP = 8; + constexpr int WARPS_PER_BLOCK = 8; + constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_BF16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 16, k, GGML_TYPE_F32) + GGML_ABORT("fatal error"); + } } else if (n_embd == 128 && n_head == 4 && n_batch >= LIGHTNING_INDEXER_TILE_TOKENS) { // too few heads for a wmma tile, the tile kernel shares the keys across the tokens constexpr int WARPS_PER_BLOCK = 8; @@ -763,7 +806,7 @@ bool ggml_cuda_lightning_indexer_supported(int device, const ggml_tensor * dst) return false; } - if (neq1 != 64 && neq1 != 32 && neq1 != 4) { + if (neq1 != 64 && neq1 != 32 && neq1 != 16 && neq1 != 4) { return false; } diff --git a/ggml/src/ggml-cuda/mmq-config-blackwell.cuh b/ggml/src/ggml-cuda/mmq-config-blackwell.cuh index 9fbe32b6972b706baee8e9218dcf1570419d9f20..e0a84630ec9e4aff38bfdd14b0b9b92f69d2eb61 100644 --- a/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +++ b/ggml/src/ggml-cuda/mmq-config-blackwell.cuh @@ -19,7 +19,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); - CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); @@ -27,7 +27,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); - CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 4951db72ebfc9a795a8602cac911bf181f132ddf..1e8f564e0bdbc6f13a2bfd73fb8e2fd29321c5e0 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -1782,9 +1782,8 @@ template static __de } } -template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - GGML_UNUSED(x_gate); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; @@ -1800,7 +1799,6 @@ template static __device__ __forceinline_ const int kbx = txi % threads_per_row; const int row_in_warp = txi / threads_per_row; - const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx; uint32_t * x_u32_scale = x_u32 + 64 + kbx; #pragma unroll @@ -1811,7 +1809,7 @@ template static __device__ __forceinline_ i = min(i, i_max); } - const block_nvfp4 * bxi = bxi_base + i * stride; + const block_nvfp4 * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0 + kbx, i, stride); const uint32_t * src_qs = reinterpret_cast(bxi->qs); diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 49e0ce9050c021f4ffa03924f7bc510f4a96cc41..71cf86debf65dc9d926ab87147f6226d677ca507 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -1,5 +1,6 @@ #include "common.cuh" #include "mmq.cuh" +#include "mmvq.cuh" #include "quantize.cuh" #include "mmid.cuh" @@ -172,7 +173,10 @@ void ggml_cuda_mul_mat_q( const ggml_tensor * gate = fusion ? fusion->gate : nullptr; ggml_cuda_mm_fusion_args_device fusion_device = {}; if (fusion) { - GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale); + GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias); + GGML_ASSERT(src0->type == GGML_TYPE_NVFP4 || (!fusion->x_scale && !fusion->gate_scale)); + fusion_device.x_scale = fusion->x_scale ? fusion->x_scale->data : nullptr; + fusion_device.gate_scale = fusion->gate_scale ? fusion->gate_scale->data : nullptr; GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0)); fusion_device.gate = gate->data; fusion_device.glu_op = fusion->glu_op; @@ -224,9 +228,9 @@ void ggml_cuda_mul_mat_q( const bool fallback = ggml_cuda_mmq_needs_fallback(ne01); - // With fusion dst is the GLU output, not a MUL_MAT; the fused types always quantize src1 to Q8_1. - // TODO support for Q4 precision if both tensors allow it - const ggml_prec prec_src1 = gate ? GGML_PREC_Q8 : ggml_cuda_mmq_get_prec_src1(src0, dst, cc); + // Native FP4 fusion is selected only after checking both matmul precision settings. + const ggml_prec prec_src1 = gate ? (src0->type == GGML_TYPE_NVFP4 ? GGML_PREC_Q4 : GGML_PREC_Q8) : + ggml_cuda_mmq_get_prec_src1(src0, dst, cc); const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4; const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq); @@ -245,9 +249,48 @@ void ggml_cuda_mul_mat_q( if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; } + + if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { + static const auto read_tile_hint = [](const char * name, int fallback) { + const char * env = getenv(name); + if (!env) { + return fallback; + } + char * end = nullptr; + const long value = strtol(env, &end, 10); + if (end == env || *end || (value != -1 && value != 0 && (value < 8 || value > 128 || value % 8))) { + GGML_LOG_WARN("Invalid %s: %s; using %d\n", name, env, fallback); + return fallback; + } + return (int) value; + }; + static const int tile_plain = read_tile_hint("GGML_CUDA_SPARK_MMQ_TILE", 0); + static const int tile_fused = read_tile_hint("GGML_CUDA_SPARK_MMQ_TILE_FUSED", tile_plain); + const int tile_hint = gate ? tile_fused : tile_plain; + if (tile_hint == -1) { + const int64_t hint = (8*ne12*n_expert_used + 3*ne02 - 1) / (3*ne02); + ncols_opt = std::min(128, std::max(32, GGML_PAD(hint, 8))); + } else if (tile_hint > 0) { + ncols_opt = tile_hint; + } + } + } + + if (!ids && src0->type == GGML_TYPE_Q8_0 && cc == GGML_CUDA_CC_BLACKWELL && ne11 > 128) { + static const int tile_q8 = [] { + const char * value = getenv("GGML_CUDA_Q8_MMQ_TILE"); + return value ? atoi(value) : 0; + }(); + if (tile_q8 >= 8 && tile_q8 <= 128 && tile_q8 % 8 == 0) { + ncols_opt = tile_q8; + } } J_best = ggml_cuda_mmq_get_J_best(src0->type, ncols_opt, fallback, cc, smpbo, prec_src1, &nthreads_best); + if (ids && src0->type == GGML_TYPE_NVFP4 && getenv("GGML_CUDA_SPARK_MMQ_TRACE")) { + GGML_LOG_INFO("SPARK_MMQ weight=%s tokens=%lld opt=%lld J=%d threads=%d fused=%d\n", src0->name, + (long long) ne12, (long long) ncols_opt, J_best, nthreads_best, gate != nullptr); + } } GGML_ASSERT(J_best > 0); @@ -531,3 +574,30 @@ bool ggml_cuda_mmq_fusion_is_efficient(const ggml_type type, const int64_t nrows const int64_t tiles_efficiency_percent = 100 * ntiles / (nsm*tiles_nwaves); return tiles_efficiency_percent >= 90; } + +bool ggml_cuda_mmq_spark_fusion_supported(const ggml_tensor * up, const ggml_tensor * gate) { + static const bool enabled = []() { + const char * env = getenv("GGML_CUDA_SPARK_MMQ_FUSION"); + return env && std::atoi(env) != 0; + }(); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (!enabled || !blackwell_mma_available(cc) || up->op != GGML_OP_MUL_MAT_ID || + up->src[0]->type != GGML_TYPE_NVFP4 || gate->src[0]->type != GGML_TYPE_NVFP4 || + up->src[1]->type != GGML_TYPE_F32 || ggml_cuda_mmq_needs_fallback(up->src[0]->ne[1]) || + up->src[1]->ne[2] <= get_mmvq_mmid_max_batch(GGML_TYPE_NVFP4, cc)) { + return false; + } + if (!ggml_cuda_should_use_mmq(GGML_TYPE_NVFP4, cc, up->src[1]->ne[2], up->src[0]->ne[2])) { + return false; + } + for (const ggml_tensor * mm : {up, gate}) { + const ggml_tensor * weight = mm->src[0]; + if (ggml_get_op_params_i32(mm, 1) != GGML_HINT_NONE || + ggml_cuda_mmq_get_prec_src1(weight, mm, cc) != GGML_PREC_Q4 || + (ggml_backend_buffer_get_usage(weight->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(weight) != ggml_backend_buffer_get_alloc_size(weight->buffer, weight) && weight->view_src)) { + return false; + } + } + return true; +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 308968be9dcb5c6cc2dcc410b4ae3f91d8e7830c..4e0be7ecbfe0fabe8dc2bd97c6e75d2893f75702 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -16,7 +16,7 @@ typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, const ch typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, - const ggml_glu_op glu_op, const float glu_limit); + const ggml_glu_op glu_op, const float glu_limit, const float2 lane_scale); enum mmq_q8_1_ds_layout { MMQ_Q8_1_DS_LAYOUT_D4, @@ -434,9 +434,9 @@ static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config // ------------------------------------------------------------ -// Weight types with a fused gate/up/GLU path (see ggml_cuda_mmq_get_util_funcs). +// Weight types with a fused gate/up/GLU path. SPARK-4Bit requires native Q4 activations. static constexpr __host__ __device__ bool ggml_cuda_mmq_fusion_supported(const ggml_type type) { - return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS; + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_NVFP4; } // Fused gate/up/GLU: the weight tile alternates chunks of this many up and gate rows, so rows r and r + 8 of an MMA tile (same thread) pair up. @@ -462,8 +462,8 @@ static __device__ __forceinline__ const block_t * ggml_cuda_mmq_x_row( template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, - const ggml_glu_op glu_op, const float glu_limit) { - GGML_UNUSED_VARS(glu_op, glu_limit); + const ggml_glu_op glu_op, const float glu_limit, const float2 lane_scale) { + GGML_UNUSED_VARS(glu_op, glu_limit, lane_scale); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; @@ -507,7 +507,7 @@ template tile_C; @@ -543,7 +543,13 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( if constexpr (has_fusion) { const int sum_index = (j0/tile_C::J + n)*tile_C::ne + l; - dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, sum[sum_index + tile_C::ne/2], sum[sum_index], glu_limit); + float up = sum[sum_index]; + float gate = sum[sum_index + tile_C::ne/2]; + if constexpr (type == GGML_TYPE_NVFP4) { + up = (up * y_scale[j]) * lane_scale.x; + gate = (gate * y_scale[j]) * lane_scale.y; + } + dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, gate, up, glu_limit); } else if constexpr (type == GGML_TYPE_NVFP4) { if (y_scale_used) { dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; @@ -558,7 +564,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( } } - GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit); + GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit, lane_scale); } // ------------------------------------------------------------------------------------------------------------------------------------- @@ -737,9 +743,9 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func if (prec_src1 == GGML_PREC_Q4) { return ggml_cuda_mmq_util_funcs( -1, - ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, + ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, ggml_cuda_mmq_vec_dot_fp4_fp4_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); } break; default: @@ -919,7 +925,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, const int stride_row_x, const int ncols_y, const int stride_col_dst, - const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { + const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop, const float2 lane_scale = {1.0f, 1.0f}) { static_assert(!(fixup && has_fusion), "the fused gate/up path does not split K"); static_assert(!(fallback && has_fusion), "the fused gate/up path needs nrows_x % 128 == 0"); @@ -987,16 +993,16 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( } if (fixup) { - write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit); + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit, lane_scale); } else { - write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j, glu_op, glu_limit); + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j, glu_op, glu_limit, lane_scale); } } // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template +template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1)) static __global__ void mul_mat_q( const char * __restrict__ x, const char * __restrict__ x_gate, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, @@ -1005,7 +1011,7 @@ static __global__ void mul_mat_q( const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, - const uint3 ntx) { + const uint3 ntx, const float * x_scale, const float * gate_scale) { #ifdef TURING_MMA_AVAILABLE constexpr bool fusion_available = true; @@ -1043,76 +1049,89 @@ static __global__ void mul_mat_q( __syncthreads(); // The fused kernel needs the whole K range of a tile in one block, so it always uses tiling. - if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) { + if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1) || compact_grid) { // The fused kernel writes the GLU of I/2 up rows and I/2 gate rows per tile. constexpr int I_dst = has_fusion ? I/2 : I; const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); const int wt = tmp2.x; const int zt = tmp2.y; - const int jt = blockIdx.x; const int it = blockIdx.y; + const int col_count = ids_dst ? expert_bounds[zt + 1] - expert_bounds[zt] : ncols_dst; + const int jt_stop = compact_grid ? (col_count + J - 1)/J : blockIdx.x + 1; + for (int jt = blockIdx.x; jt < jt_stop; jt += gridDim.x) { + if constexpr (compact_grid) { + __syncthreads(); + } - // Defaults for regular matrix multiplication: - int col_low = 0; - int col_high = ncols_dst; - int col_diff = ncols_dst; - int offset_y = wt*stride_sample_y + zt*stride_channel_y; - int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; - int offset_y_scale; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; - } else { - GGML_UNUSED(offset_y_scale); - } - - if (ids_dst) { - col_low = expert_bounds[zt + 0]; - col_high = expert_bounds[zt + 1]; - col_diff = col_high - col_low; - - offset_y = 0; - offset_dst = 0; + // Defaults for regular matrix multiplication: + int col_low = 0; + int col_high = ncols_dst; + int col_diff = ncols_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale = 0; + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); } - if (jt*J >= col_diff) { - return; - } + if (ids_dst) { + col_low = expert_bounds[zt + 0]; + col_high = expert_bounds[zt + 1]; + col_diff = col_high - col_low; - // __syncthreads(); // There is no previous tile that could cause a race condition. -#pragma unroll - for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { - const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + offset_y = 0; + offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } - if (j0 + nwarps*warp_size > J && j >= J) { - break; + if (jt*J >= col_diff) { + return; } - ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = jt*J + j < col_diff ? ids_dst[col_low + jt*J + j] : 0; + } + __syncthreads(); } - __syncthreads(); - } - offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); - offset_dst += it*I_dst; - const float * y_scale_tile = nullptr; - if constexpr (type == GGML_TYPE_NVFP4) { - offset_y_scale += col_low + jt*J; - y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; - } + offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); + offset_dst += it*I_dst; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt*J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } - const int tile_x_max_i = nrows_x - it*I_dst - 1; - const int tile_y_max_j = col_diff - jt*J - 1; + const int tile_x_max_i = nrows_x - it*I_dst - 1; + const int tile_y_max_j = col_diff - jt*J - 1; - const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x; + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x; - constexpr bool fixup = false; - mul_mat_q_process_tile - (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, - glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); + float2 lane_scale = {1.0f, 1.0f}; + if constexpr (has_fusion && type == GGML_TYPE_NVFP4) { + const int expert = fastdiv(zt, channel_ratio); + lane_scale = {x_scale ? x_scale[expert] : 1.0f, gate_scale ? gate_scale[expert] : 1.0f}; + } + constexpr bool fixup = false; + mul_mat_q_process_tile + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z, lane_scale); + if constexpr (!compact_grid) { + break; + } + } return; } @@ -1473,7 +1492,14 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const int ntx = (args.ncols_max + config.J - 1) / config.J; const int ntzw = args.nchannels_y * args.nsamples_y; // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2. - const dim3 block_nums_xy_tiling(ntx, nty, ntzw); + static const int compact_hint = [] { + const char * value = getenv("GGML_CUDA_SPARK_MMQ_COMPACT_GRID"); + return value ? atoi(value) : 0; + }(); + const bool compact_grid = type == GGML_TYPE_NVFP4 && prec_src1 == GGML_PREC_Q4 && + blackwell_mma_available(cc) && args.ids_dst && compact_hint > 0; + const int grid_x = compact_grid ? std::min(ntx, compact_hint) : ntx; + const dim3 block_nums_xy_tiling(grid_x, nty, ntzw); GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); @@ -1487,14 +1513,21 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); - if (has_fusion || !config.stream_k) { + if (has_fusion || !config.stream_k || compact_grid) { GGML_ASSERT(nty <= 65535); // gridDim.y limit - mul_mat_q<<>> + auto kernel = mul_mat_q; + if constexpr (type == GGML_TYPE_NVFP4 && prec_src1 == GGML_PREC_Q4) { + if (compact_grid) { + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + kernel = mul_mat_q; + } + } + kernel<<>> (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntx_fd, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale); return; } @@ -1525,7 +1558,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); + ntx_fd, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale); if (!fixup_needed) { return; @@ -1597,7 +1630,7 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, template static void mul_mat_q_switch_fusion(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - if constexpr (!fallback && ggml_cuda_mmq_fusion_supported(type)) { + if constexpr (!fallback && ggml_cuda_mmq_fusion_supported(type) && (type != GGML_TYPE_NVFP4 || prec_src1 == GGML_PREC_Q4)) { if (args.fusion.gate) { mul_mat_q_switch_J(ctx, args, stream); return; @@ -1662,3 +1695,5 @@ void ggml_cuda_mul_mat_q( bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device); + +bool ggml_cuda_mmq_spark_fusion_supported(const ggml_tensor * up, const ggml_tensor * gate); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 1a9cf55cb9158e601c3b96db1f5b724aae442039..ceb86796e6315390142c7c67769e05584b5f145c 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -618,8 +618,8 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int return 1; } -template -__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1) +template +__launch_bounds__((spark_2w ? 2 : calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters))*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q( const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y, @@ -636,8 +636,8 @@ static __global__ void mul_mat_vec_q( constexpr int qi = ggml_cuda_type_traits::qi; constexpr int vdr = get_vdr_mmvq(type); constexpr mmvq_parameter_table_id table_id = get_device_table_id(); - constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); - constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); + constexpr int nwarps = spark_2w ? 2 : calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); + constexpr int rows_per_cuda_block = spark_2w ? 1 : calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); @@ -1198,6 +1198,27 @@ static void mul_mat_vec_q_switch_ncols_dst( return idle * 8 <= iters_wide * 2; }; + if constexpr (type == GGML_TYPE_NVFP4) { + static const bool use_2w = [] { const char * v = getenv("GGML_CUDA_SPARK_MMVQ_2W"); return v && atoi(v) != 0; }(); + if (use_2w && cc >= GGML_CUDA_CC_BLACKWELL && cc < GGML_CUDA_CC_RUBIN && has_ids && ncols_dst == 1 && + ncols_x == 1536 && !fusion.shared_up) { + const ggml_cuda_kernel_launch_params params(dim3(nrows_x, nchannels_dst, nsamples_dst), dim3(warp_size, 2), 0, stream); + const bool fused = fusion.gate || fusion.x_bias || fusion.gate_bias || fusion.x_scale || fusion.gate_scale; + if (fused) { + ggml_cuda_kernel_launch(mul_mat_vec_q, params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); + } else { + ggml_cuda_kernel_launch(mul_mat_vec_q, params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); + } + return; + } + } + if (has_ids && ncols_dst > 1) { // Multi-token MUL_MAT_ID path - dedicated MoE kernel mul_mat_vec_q_moe_launch( @@ -1454,7 +1475,7 @@ static void mul_mat_vec_q_switch_type( void ggml_cuda_mul_mat_vec_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, - const ggml_cuda_mm_fusion_args_host * fusion) { + const ggml_cuda_mm_fusion_args_host * fusion, ggml_cuda_mmvq_cache * cache) { GGML_ASSERT( src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. @@ -1549,12 +1570,25 @@ void ggml_cuda_mul_mat_vec_q( } const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); - ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1); - { + ggml_cuda_pool_alloc src1_q8_1(ctx.pool()); + char * src1_quantized = nullptr; + bool quantize = true; + if (cache) { + auto it = cache->entries.find(ggml_cuda_mmvq_cache_key(src1)); + if (it != cache->entries.end()) { + src1_quantized = cache->data.get() + it->second.offset; + quantize = !it->second.ready; + it->second.ready = true; + } + } + if (!src1_quantized) { + src1_quantized = src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1); + } + if (quantize) { const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + quantize_row_q8_1_cuda(src1_d, nullptr, src1_quantized, src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); } const int64_t s01 = src0->nb[1] / ts_src0; @@ -1580,7 +1614,7 @@ void ggml_cuda_mul_mat_vec_q( const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0; mul_mat_vec_q_switch_type( - src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00, + src0->data, src0->type, src1_quantized, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, ne03, ne3, s03, s13, s3, ids_stride, stream); diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh index 5605bf7a4e60348c047844c771e7171849ba2eec..b5a39a6b191b5eeee7a49bdc9758c6f0ed1b0a09 100644 --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -8,8 +8,25 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11); // based on the quantization type and GPU architecture (compute capability). int get_mmvq_mmid_max_batch(ggml_type type, int cc); +static inline const ggml_tensor * ggml_cuda_mmvq_cache_key(const ggml_tensor * x) { + while (x->view_src && x->data == x->view_src->data && x->type == x->view_src->type && + ggml_are_same_shape(x, x->view_src) && ggml_is_contiguous(x->view_src)) { + x = x->view_src; + } + return x; +} + +struct ggml_cuda_mmvq_cache { + struct entry { + size_t offset; + bool ready = false; + }; + std::unordered_map entries; + ggml_cuda_pool_alloc data; +}; + void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr); + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr, ggml_cuda_mmvq_cache * cache = nullptr); void ggml_cuda_op_mul_mat_vec_q( ggml_backend_cuda_context & ctx, diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4d057295c128c2b47f45564c635f597bd1f392ea..79be197138c349e315c87bb572d3df1c4bf41b1f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -702,6 +702,7 @@ class MODEL_ARCH(IntEnum): HUNYUAN_VL = auto() HY_V3 = auto() HY_V4 = auto() + HY_V4_FLASH = auto() SMOLLM3 = auto() GPT_OSS = auto() LFM2 = auto() @@ -1050,6 +1051,8 @@ class MODEL_TENSOR(IntEnum): INDEXER_COMPRESSOR_NORM = auto() INDEXER_KPOOL_GATE = auto() INDEXER_KPOOL_APE = auto() + INDEXER_KPOOL_VALUE = auto() + INDEXER_KPOOL_NORM = auto() ATTN_V_GATE = auto() # k2-horizon MoVA router ATTN_V_EXP = auto() # k2-horizon MoVA value experts # vision @@ -1377,6 +1380,10 @@ class MODEL_TENSOR(IntEnum): DSPARK_MASK_EMBD = auto() # per-slot mask embeddings ENGRAM_KV = auto() ENGRAM_GATE = auto() + ENGRAM_KV_B = auto() + ENGRAM_LOOKUP_NORM = auto() + ENGRAM_PRE_GAMMA = auto() + ENGRAM_HC_SCALE = auto() DFLASH_ATTN_CONV_BASE = auto() DFLASH_ATTN_CONV_PROJ = auto() DFLASH_FFN_CONV_BASE = auto() @@ -1562,6 +1569,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.HUNYUAN_VL: "hunyuan_vl", MODEL_ARCH.HY_V3: "hy_v3", MODEL_ARCH.HY_V4: "hy_v4", + MODEL_ARCH.HY_V4_FLASH: "hy_v4_flash", MODEL_ARCH.SMOLLM3: "smollm3", MODEL_ARCH.GPT_OSS: "gpt-oss", MODEL_ARCH.LFM2: "lfm2", @@ -1908,6 +1916,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.INDEXER_COMPRESSOR_NORM: "blk.{bid}.indexer_compressor_norm", MODEL_TENSOR.INDEXER_KPOOL_GATE: "blk.{bid}.indexer_compressor_gate", MODEL_TENSOR.INDEXER_KPOOL_APE: "blk.{bid}.indexer_compressor_ape", + MODEL_TENSOR.INDEXER_KPOOL_VALUE: "blk.{bid}.indexer_compressor_value", + MODEL_TENSOR.INDEXER_KPOOL_NORM: "blk.{bid}.indexer_compressor_norm", # vision MODEL_TENSOR.V_MMPROJ: "mm.{bid}", MODEL_TENSOR.V_MMPROJ_FC: "mm.model.fc", @@ -2259,6 +2269,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.DSPARK_MASK_EMBD: "mask_embd", MODEL_TENSOR.ENGRAM_KV: "blk.{bid}.engram_kv", MODEL_TENSOR.ENGRAM_GATE: "blk.{bid}.engram_gate", + MODEL_TENSOR.ENGRAM_KV_B: "blk.{bid}.engram_kv_b", + MODEL_TENSOR.ENGRAM_LOOKUP_NORM: "blk.{bid}.engram_lookup_norm", + MODEL_TENSOR.ENGRAM_PRE_GAMMA: "blk.{bid}.engram_pre_gamma", + MODEL_TENSOR.ENGRAM_HC_SCALE: "blk.{bid}.engram_hc_scale", MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base", MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj", MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base", @@ -5553,6 +5567,67 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_SHEXP, MODEL_TENSOR.FFN_UP_SHEXP, ], + MODEL_ARCH.HY_V4_FLASH: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.HC_HEAD_FN, + MODEL_TENSOR.HC_HEAD_BASE, + MODEL_TENSOR.HC_HEAD_SCALE, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_KV_A_MQA, + MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.ATTN_K_B, + MODEL_TENSOR.ATTN_V_B, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.INDEXER_K_NORM, + MODEL_TENSOR.INDEXER_PROJ, + MODEL_TENSOR.INDEXER_ATTN_K, + MODEL_TENSOR.INDEXER_ATTN_Q_B, + MODEL_TENSOR.HC_ATTN_FN, + MODEL_TENSOR.HC_ATTN_BASE, + MODEL_TENSOR.HC_ATTN_SCALE, + MODEL_TENSOR.HC_FFN_FN, + MODEL_TENSOR.HC_FFN_BASE, + MODEL_TENSOR.HC_FFN_SCALE, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.FFN_ROUTED_DOWN, + MODEL_TENSOR.FFN_ROUTED_UP, + MODEL_TENSOR.PER_LAYER_TOKEN_EMBD, + MODEL_TENSOR.ENGRAM_KV, + MODEL_TENSOR.ENGRAM_GATE, + MODEL_TENSOR.ENGRAM_KV_B, + MODEL_TENSOR.ENGRAM_LOOKUP_NORM, + MODEL_TENSOR.ENGRAM_PRE_GAMMA, + MODEL_TENSOR.ENGRAM_HC_SCALE, + MODEL_TENSOR.INDEXER_KPOOL_GATE, + MODEL_TENSOR.INDEXER_KPOOL_APE, + MODEL_TENSOR.INDEXER_KPOOL_VALUE, + MODEL_TENSOR.INDEXER_KPOOL_NORM, + MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_ENORM, + MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, + ], MODEL_ARCH.SMOLLM3: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -6468,6 +6543,10 @@ MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, ], + MODEL_ARCH.HY_V4_FLASH: [ + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_ROT_EMBD, + ], MODEL_ARCH.CHATGLM: [ MODEL_TENSOR.ROPE_FREQS, ], @@ -6734,6 +6813,7 @@ class VisionProjectorType: POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder SOPRANO = "soprano" # audio generation: Vocos decoder HUNYUANVL = "hunyuanvl" + HY_V4_FLASH = "hy_v4_flash" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" MINICPMV4_6 = "minicpmv4_6" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d06dc4ec5fdcd3c1f7b5aa75025a095ac0a1db07..a39f961dd75531051f24c168eecd756c68ad16b0 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -420,6 +420,7 @@ class TensorNameMap: "model.layers.{bid}.post_attn_norm", # kolibri1 "model.layers.layers.{bid}.post_mixer_norm.weight", # plamo2 "model.layers.{bid}.post_operator_norm", # lfm3 + "model.layers.{bid}.self_attn.post_proj_norm", # hy-v4-flash ), # Rotary embeddings @@ -504,6 +505,8 @@ class TensorNameMap: "model.layers.{bid}.post_mlp_layernorm", # glm-4-0414 "model.layers.{bid}.post_ffn_norm", # kolibri1 "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2 + "model.layers.{bid}.mlp.combine_output_norm", # hy-v4-flash (MoE layers) + "model.layers.{bid}.mlp.post_fc2_norm", # hy-v4-flash (dense layers) "model.layers.{bid}.feed_forward.up_proj", "model.layers.{bid}.post_moe_norm", # grok-2 "model.layers.{bid}.post_ffn_norm", # lfm3 @@ -1026,10 +1029,12 @@ class TensorNameMap: # Kimi K3 latent MoE: routed experts operate in a down-projected space MODEL_TENSOR.FFN_ROUTED_DOWN: ( "model.layers.{bid}.block_sparse_moe.routed_expert_down_proj", + "model.layers.{bid}.mlp.routed_expert_down_proj", # hy-v4-flash ), MODEL_TENSOR.FFN_ROUTED_UP: ( "model.layers.{bid}.block_sparse_moe.routed_expert_up_proj", + "model.layers.{bid}.mlp.routed_expert_up_proj", # hy-v4-flash ), MODEL_TENSOR.FFN_ROUTED_NORM: ( @@ -2406,6 +2411,7 @@ class TensorNameMap: MODEL_TENSOR.V_MM_PRE_NORM: ( "vit.perceive.before_rms", # HunyuanVL + "vit.perceive.norm", # hy-v4-flash ), MODEL_TENSOR.V_TOK_IMG_BEGIN: ( diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97250c2e885645386702f516d1f00ff2c9b121f2..d15912560f4743ac1fd509c82b11c12f6b1b1b2f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,6 +32,7 @@ set(LLAMA_CORE_SOURCES llama-memory-hybrid.cpp llama-memory-hybrid-iswa.cpp llama-memory-hybrid-idx.cpp + llama-memory-idx-iswa.cpp llama-memory-recurrent.cpp llama-mmap.cpp llama-model-loader.cpp diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 40f608371c548107f2dcf24217a4d8c60be16a02..b4df866e83b1fa4aeaa5d7f4e05abc010c9b074c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -132,6 +132,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_HUNYUAN_VL, "hunyuan_vl" }, { LLM_ARCH_HY_V3, "hy_v3" }, { LLM_ARCH_HY_V4, "hy_v4" }, + { LLM_ARCH_HY_V4_FLASH, "hy_v4_flash" }, { LLM_ARCH_SMOLLM3, "smollm3" }, { LLM_ARCH_OPENAI_MOE, "gpt-oss" }, { LLM_ARCH_LFM2, "lfm2" }, @@ -808,6 +809,8 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "blk.%d.indexer_compressor_norm" }, { LLM_TENSOR_INDEXER_KPOOL_GATE, "blk.%d.indexer_compressor_gate" }, { LLM_TENSOR_INDEXER_KPOOL_APE, "blk.%d.indexer_compressor_ape" }, + { LLM_TENSOR_INDEXER_KPOOL_VALUE, "blk.%d.indexer_compressor_value" }, + { LLM_TENSOR_INDEXER_KPOOL_NORM, "blk.%d.indexer_compressor_norm" }, { LLM_TENSOR_FFN_GATE_TID2EID, "blk.%d.ffn_gate_tid2eid" }, { LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" }, { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, @@ -820,6 +823,10 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_DSPARK_MASK_EMBD, "mask_embd" }, { LLM_TENSOR_ENGRAM_KV, "blk.%d.engram_kv" }, { LLM_TENSOR_ENGRAM_GATE, "blk.%d.engram_gate" }, + { LLM_TENSOR_ENGRAM_KV_B, "blk.%d.engram_kv_b" }, + { LLM_TENSOR_ENGRAM_LOOKUP_NORM, "blk.%d.engram_lookup_norm" }, + { LLM_TENSOR_ENGRAM_PRE_GAMMA, "blk.%d.engram_pre_gamma" }, + { LLM_TENSOR_ENGRAM_HC_SCALE, "blk.%d.engram_hc_scale" }, { LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" }, { LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" }, { LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" }, @@ -1139,6 +1146,8 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_INDEXER_COMPRESSOR_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_INDEXER_KPOOL_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_INDEXER_KPOOL_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_INDEXER_KPOOL_VALUE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_INDEXER_KPOOL_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_GATE_TID2EID, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, @@ -1175,6 +1184,10 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_DSPARK_MASK_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_ENGRAM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ENGRAM_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_KV_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_ENGRAM_LOOKUP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_PRE_GAMMA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_HC_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1372,6 +1385,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_HY_V4: + case LLM_ARCH_HY_V4_FLASH: case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_GLM_DSA: case LLM_ARCH_BITNET: diff --git a/src/llama-arch.h b/src/llama-arch.h index b98be40cf29bb7c6609a9531671cc73cb93b9b2c..afff4ca7e70965deccc1cff94966754583f13d6a 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -136,6 +136,7 @@ enum llm_arch { LLM_ARCH_HUNYUAN_VL, LLM_ARCH_HY_V3, LLM_ARCH_HY_V4, + LLM_ARCH_HY_V4_FLASH, LLM_ARCH_SMOLLM3, LLM_ARCH_OPENAI_MOE, LLM_ARCH_LFM2, @@ -801,6 +802,8 @@ enum llm_tensor { LLM_TENSOR_INDEXER_COMPRESSOR_NORM, LLM_TENSOR_INDEXER_KPOOL_GATE, // glm5-next: k-pool gate scores LLM_TENSOR_INDEXER_KPOOL_APE, // glm5-next: k-pool position bias + LLM_TENSOR_INDEXER_KPOOL_VALUE, // hy_v4_flash: what the k-pool actually pools + LLM_TENSOR_INDEXER_KPOOL_NORM, // hy_v4_flash: norm of the pooled key LLM_TENSOR_FFN_GATE_TID2EID, LLM_TENSOR_NEXTN_PROJ_PRE, LLM_TENSOR_NEXTN_PROJ_POST, @@ -825,6 +828,10 @@ enum llm_tensor { LLM_TENSOR_DSPARK_MASK_EMBD, LLM_TENSOR_ENGRAM_KV, LLM_TENSOR_ENGRAM_GATE, + LLM_TENSOR_ENGRAM_KV_B, // hy_v4_flash: bias on the fused key/value projections + LLM_TENSOR_ENGRAM_LOOKUP_NORM, // hy_v4_flash: per-head RMSNorm of the looked-up rows + LLM_TENSOR_ENGRAM_PRE_GAMMA, // hy_v4_flash: scales the normed value before the gate + LLM_TENSOR_ENGRAM_HC_SCALE, // hy_v4_flash: per-hc per-channel additive gate offset LLM_TENSOR_DFLASH_ATTN_CONV_BASE, LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, LLM_TENSOR_DFLASH_FFN_CONV_BASE, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0cd795a5690b31b55834f0e7ab3b4e7e3bf3fd7c..0095311c6be80a3a86139dfc9ab43c0551873e49 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2715,7 +2715,8 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_01 || model.arch == LLM_ARCH_MINIMAX_M3 || - model.arch == LLM_ARCH_HY_V4) { + model.arch == LLM_ARCH_HY_V4 || + model.arch == LLM_ARCH_HY_V4_FLASH) { res = std::max(n_tokens * 40, 32u * model.n_tensors()); } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) { // DFlash2's convolutions and selector are shape work rather than matmuls, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe8bdebe5e2c7452375dfa01c364e0d48ebc198c..5fab1d08425927a8aee488a1904d9e0e1467ca78 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -834,8 +834,12 @@ void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) { mctx->get_swa()->set_input_attn_mask(self_attn_mask_swa, ubatch, cparams.causal_attn); } +const llama_kv_cache_iswa_context * llm_graph_input_attn_k_iswa::mctx_of(const llm_graph_params & params) const { + return static_cast(params.mctx); +} + bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) { - const auto * mctx = static_cast(params.mctx); + const auto * mctx = mctx_of(params); this->mctx = mctx; @@ -2452,7 +2456,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( const float limit = hparams.swiglu_clamp_exp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { + if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4 || arch == LLM_ARCH_HY_V4_FLASH) { cur = ggml_swiglu_clamp(ctx0, cur, up, limit); } else { up = ggml_clamp(ctx0, up, -limit, limit); @@ -3868,7 +3872,13 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { const auto * mctx_cur = static_cast(mctx); - auto inp = std::make_unique(hparams, cparams, mctx_cur); + return build_attn_inp_k_iswa( + std::make_unique(hparams, cparams, mctx_cur), mctx_cur); +} + +llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa( + std::unique_ptr inp, + const llama_kv_cache_iswa_context * mctx_cur) const { { inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); diff --git a/src/llama-graph.h b/src/llama-graph.h index 37b47acbb30aca36007d90d87e8412adfd22f6b1..688fda7a0d1173369a8b017f29d1db9bb0ee9f95 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -562,12 +562,17 @@ public: cparams(cparams), mctx(mctx) { } - ~llm_graph_input_attn_k_iswa() = default; + virtual ~llm_graph_input_attn_k_iswa() = default; void set_input(const llama_ubatch * ubatch) override; bool can_reuse(const llm_graph_params & params) override; + // Where this input finds its context in the next batch's params. The default is that the + // memory is the iSWA cache itself; an arch whose memory merely composes one overrides this, + // because params.mctx is then that outer context and the cast would be wrong. + virtual const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const; + ggml_tensor * get_k_idxs() const { return self_k_idxs; } ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; } @@ -1370,6 +1375,12 @@ struct llm_graph_context { llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const; + // same, for a memory that composes an iSWA pair rather than being one: the arch supplies the + // input, so it can override mctx_of(), and the attention context out of its own + llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa( + std::unique_ptr inp, + const llama_kv_cache_iswa_context * mctx_cur) const; + // note: if k_cur is not provided, it will not be stored in the memory // note: the K cache is used as V (MLA-style attention) ggml_tensor * build_attn( diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4d2cb89314d24fab9ca845f2469847c5912ef807..19748815b7f947a3ad4b02452792d02db762f809 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -10,7 +10,10 @@ // bump if necessary #define LLAMA_MAX_LAYERS 512 #define LLAMA_MAX_EXPERTS 1024 // Kimi K3 -#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp, deepseek41 (all hash modules) +// multipliers are stored flat as [module][ngram] and indexed at stride ple_ngram_size +// (deepseek41.cpp:247), so this bounds ple_n_modules*ple_ngram_size, not ngram_size alone. +// hy_v4_flash needs 4 modules x 3 = 12. +#define LLAMA_MAX_PLE_NGRAM 16 // qwen4exp, deepseek41, hy_v4_flash (all hash modules) #define LLAMA_MAX_PLE_HEADS 64 // qwen4exp, deepseek41 (all hash modules) enum llama_expert_gating_func_type { diff --git a/src/llama-memory-idx-iswa.cpp b/src/llama-memory-idx-iswa.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f6298b4d4385b535eaa80fc3a3c0fe65ca0f9d2 --- /dev/null +++ b/src/llama-memory-idx-iswa.cpp @@ -0,0 +1,984 @@ +#include "llama-memory-idx-iswa.h" + +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-io.h" +#include "llama-model.h" + +#include +#include +#include +#include + +// +// llama_memory_idx_iswa +// + +llama_memory_idx_iswa::llama_memory_idx_iswa( + const llama_model & model, + bool no_alloc, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + const layer_filter_cb & filter, + const layer_filter_cb & filter_idx) : + hparams_idx(model.hparams), + mem_attn(new llama_kv_cache_iswa( + model, no_alloc, type_k, type_v, v_trans, offload, swa_full, unified, + kv_size, n_seq_max, n_ubatch, + nullptr, filter, nullptr, nullptr)), + mem_idx([&] { + // a model without an indexer uses llama_kv_cache_iswa directly, so there is no null case here + GGML_ASSERT(filter_idx != nullptr); + + // the indexer mirrors the base child alone, so a SWA layer has no cells of its own to index + for (uint32_t il = 0; il < model.hparams.n_layer_all; ++il) { + GGML_ASSERT(!(filter_idx(il) && model.hparams.is_swa(il)) && "an indexer layer cannot be a SWA layer"); + } + + // MQA with a single key head of indexer_head_size, as llama_kv_cache_dsa shapes its own + std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1); + // a k-pool indexer caches its per-token rows and the pooled key side by side + // (glm5-next: key | gate | pooled, qwen4exp: key | pooled) + hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size * (model.hparams.indexer_kpool > 0 ? model.hparams.indexer_kpool_row : 1); + + // the cached indexer keys are raw, rotation happens after pooling at read time, so a + // K-shift must not rotate them while the stream copies in the same update still apply + hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE; + + // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors + hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; + hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; + + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); + + // the indexer mirrors the base child, so it takes its size and its (absent) SWA as well + return new llama_kv_cache( + model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified, + kv_size, n_seq_max, 0, LLAMA_SWA_TYPE_NONE, + nullptr, filter_idx, nullptr, nullptr, "idx_"); + }()) {} + +llama_memory_context_ptr llama_memory_idx_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { + // note: repeats llama_kv_cache_iswa::init_batch, as the indexer needs the base slot infos that its context hides + do { + balloc.split_reset(); + + std::vector ubatches; + + while (true) { + llama_ubatch ubatch; + + if (embd_all) { + // if all tokens are output, split by sequence + ubatch = balloc.split_seq(n_ubatch); + } else { + // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) + const bool unified = (mem_attn->get_base()->get_n_stream() == 1); + + ubatch = balloc.split_equal(n_ubatch, !unified, 0); + } + + if (ubatch.n_tokens == 0) { + break; + } + + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + // failed to find a suitable split + break; + } + + auto sinfos_base = mem_attn->get_base()->prepare(ubatches); + if (sinfos_base.empty()) { + LLAMA_LOG_ERROR("%s: failed to prepare non-SWA attention ubatches\n", __func__); + break; + } + + auto sinfos_swa = mem_attn->get_swa()->prepare(ubatches); + if (sinfos_swa.empty()) { + LLAMA_LOG_ERROR("%s: failed to prepare SWA attention ubatches\n", __func__); + break; + } + + assert(sinfos_base.size() == sinfos_swa.size()); + + // the indexer uses the base cache's slot layout; a separate one can drift from it + llama_kv_cache::slot_info_vec_t sinfos_idx = sinfos_base; + + return std::make_unique( + this, std::move(sinfos_base), std::move(sinfos_swa), std::move(sinfos_idx), std::move(ubatches)); + } while(false); + + return std::make_unique(LLAMA_MEMORY_STATUS_FAILED_PREPARE); +} + +llama_memory_context_ptr llama_memory_idx_iswa::init_full() { + return std::make_unique(this); +} + +llama_memory_context_ptr llama_memory_idx_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); +} + +bool llama_memory_idx_iswa::get_can_shift() const { + // the indexer holds raw keys, so a shift never has to rotate them: only attention can refuse one + return mem_attn->get_can_shift(); +} + +void llama_memory_idx_iswa::clear(bool data) { + mem_attn->clear(data); + mem_idx ->clear(data); + + mem_idx_stale_set(-1, 0); +} + +// A pooled key is only valid while the grouping that produced it holds. Grouping is sequence relative, +// so an edit at p0 leaves every pool that ends before p0 alone. +void llama_memory_idx_iswa::mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0) { + p0 = std::max(p0, 0); + + if (seq_id < 0) { + for (auto & p : mem_idx_stale) { + p = std::min(p, p0); + } + + return; + } + + GGML_ASSERT(seq_id < (llama_seq_id) LLAMA_MAX_SEQ); + + mem_idx_stale[seq_id] = std::min(mem_idx_stale[seq_id], p0); +} + +// An edit at or below the first position moves pos_min, which regroups the whole sequence. +llama_pos llama_memory_idx_iswa::mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const { + if (seq_id < 0 || p0 <= mem_idx->seq_pos_min(seq_id)) { + return 0; + } + + return p0; +} + +bool llama_memory_idx_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + // the stale position is relative to the first position the indexer still holds, so read it before the removal + const llama_pos stale = mem_idx_stale_pos(seq_id, p0); + + bool res = true; + + res = res & mem_attn->seq_rm(seq_id, p0, p1); + res = res & mem_idx ->seq_rm(seq_id, p0, p1); + + mem_idx_stale_set(seq_id, stale); + + return res; +} + +void llama_memory_idx_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + // only whole sequences are copied: a shared cell holds a single pool grouping + GGML_ASSERT(p0 <= 0 && p1 < 0 && "partial seq_cp is not supported"); + + mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1); + mem_idx ->seq_cp(seq_id_src, seq_id_dst, p0, p1); + + // a whole sequence copy gives the destination the source's pools, rep rows included: the source keeps its + // pooled keys, the destination rebuilds its layout and re-pools into the same rows + mem_idx_stale_set(seq_id_dst, 0); +} + +void llama_memory_idx_iswa::seq_keep(llama_seq_id seq_id) { + mem_attn->seq_keep(seq_id); + mem_idx ->seq_keep(seq_id); + + // every other sequence loses its cells, so their layouts must rebuild + mem_idx_stale_set(-1, 0); +} + +void llama_memory_idx_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + // a negative shift moves the cells below p0, so they regroup as well + const llama_pos stale = mem_idx_stale_pos(seq_id, shift < 0 ? p0 + shift : p0); + + mem_attn->seq_add(seq_id, p0, p1, shift); + mem_idx ->seq_add(seq_id, p0, p1, shift); + + mem_idx_stale_set(seq_id, stale); +} + +void llama_memory_idx_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + mem_attn->seq_div(seq_id, p0, p1, d); + mem_idx ->seq_div(seq_id, p0, p1, d); + + mem_idx_stale_set(seq_id, 0); +} + +llama_pos llama_memory_idx_iswa::seq_pos_min(llama_seq_id seq_id) const { + return mem_attn->seq_pos_min(seq_id); +} + +llama_pos llama_memory_idx_iswa::seq_pos_max(llama_seq_id seq_id) const { + return mem_attn->seq_pos_max(seq_id); +} + +std::map llama_memory_idx_iswa::memory_breakdown() const { + std::map mb = mem_attn->memory_breakdown(); + + for (const auto & buft_size : mem_idx->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + + return mb; +} + +void llama_memory_idx_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + mem_attn->state_write(io, seq_id, flags); + + // the indexer section goes last, so it is a pure suffix: an iSWA-only reader stops early instead of misparsing it + mem_idx->state_write(io, seq_id, flags); +} + +void llama_memory_idx_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + // note: repeats llama_kv_cache_iswa::state_read + // the indexer needs the base cache's cells, and a half-failed restore must leave both caches alike + + // the indexer restore adopts the base cache's layout instead of searching for cells of its own + // two find_slot calls agree only while both caches see the same occupancy, which a restore cannot promise + llama_kv_cache::slot_info_vec_t sinfos_base; + + try { + mem_attn->get_base()->state_read_sinfo(io, seq_id, flags, &sinfos_base, nullptr); + mem_attn->get_swa ()->state_read(io, seq_id, flags); + + // must mirror the write order in state_write + // the indexer has the SWA type of the base child, so both read the same partial state + mem_idx->state_read_sinfo(io, seq_id, flags, nullptr, &sinfos_base); + + // a restore that rewrites cells does it behind the pool layout's back + if (!sinfos_base.empty()) { + mem_idx_stale_set(seq_id, 0); + } + } catch (...) { + // a half-restored context is the one state the indexer cannot fix by itself: attention holds new cells, the indexer old ones + // drop what was being restored from both of them, which is a state they do agree on. + state_drop(seq_id); + + throw; + } +} + +void llama_memory_idx_iswa::state_drop(llama_seq_id seq_id) { + if (seq_id < 0) { + clear(true); + + return; + } + + mem_attn->get_base()->state_clear(seq_id); + mem_attn->get_swa ()->state_clear(seq_id); + + mem_idx->state_clear(seq_id); + mem_idx_stale_set(seq_id, 0); +} + +llama_kv_cache_iswa * llama_memory_idx_iswa::get_mem_attn() const { + return mem_attn.get(); +} + +llama_kv_cache * llama_memory_idx_iswa::get_mem_idx() const { + return mem_idx.get(); +} + +// +// llama_memory_idx_iswa_context +// + +// streams in each ubatch's slot info, matching get_k/get_v's `ns` +static std::vector llama_memory_idx_iswa_ns(const llama_kv_cache::slot_info_vec_t & sinfos) { + std::vector res; + res.reserve(sinfos.size()); + + for (const auto & sinfo : sinfos) { + res.push_back(sinfo.s1 - sinfo.s0 + 1); + } + + return res; +} + +// Which cells of a sequence make up which pool, for the whole cache. +struct llama_memory_idx_iswa::kpool_layout { + struct seq { + llama_pos pos_min = 0; + uint32_t strm = 0; // Stream holding this sequence's cells + std::vector> cells; // Position and stream local cell pairs, sorted by position. + std::vector pools; + + // Where the pool scan stopped, so an append resumes instead of starting over. + size_t j_next = 0; + }; + + std::array seqs; + + uint32_t n_pool_real = 0; +}; + +// Which pools of the layout the current ubatch must re-pool, in the layout's pool order. +struct llama_memory_idx_iswa_context::kpool_state { + std::vector is_new; + std::vector rep_gen; // per global cell, the generation that last marked a pool with that rep + uint32_t generation = 0; + + uint32_t n_pool_real = 0; + uint32_t n_new = 0; + uint32_t n_new_g = 1; // graph size of the new pool list, stable across decode steps +}; + +namespace { + +// The last padded pool is always unused. +uint32_t kpool_pad(uint32_t n_pool) { + return std::max(64u, GGML_PAD(n_pool + 1, 64u)); +} + +// Rank of (pos, cell) in a sequence's cells sorted by position then cell, or -1 when absent. +// In order mode the rank alone places a token: cells sharing a position (M-RoPE images) have distinct ranks. +int64_t kpool_rank(const std::vector> & cells, llama_pos pos, uint32_t cell) { + auto it = std::lower_bound(cells.begin(), cells.end(), std::make_pair(pos, cell)); + return it != cells.end() && it->second == cell && it->first == pos ? it - cells.begin() : -1; +} + +} + +llama_memory_idx_iswa::~llama_memory_idx_iswa() = default; + +const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_get() const { + GGML_ASSERT(kpool_lay != nullptr); + + return *kpool_lay; +} + +// Pools are fixed by the positions relative to the sequence's first one, so the layout survives a plain +// append. A sequence edit can regroup them, and mem_idx_stale tells us it happened. +const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_update() { + if (!kpool_lay) { + kpool_lay = std::make_unique(); + } + + auto & lay = *kpool_lay; + + const uint32_t kpool = get_kpool(); + const uint32_t n_stream_kv = mem_idx->get_n_stream(); + const bool unified = n_stream_kv == 1; + + lay.n_pool_real = 0; + + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + auto & sq = lay.seqs[s]; + + // a non unified cache gives each sequence its own stream, with stream local cell indices + if (!unified && s >= (llama_seq_id) n_stream_kv) { + sq = kpool_layout::seq(); + continue; + } + + const auto & cells = mem_idx->get_cells(unified ? 0 : s); + const auto & sp = cells.seq_pos_get(s); + + sq.strm = unified ? 0 : mem_idx->get_stream(s); + + if (mem_idx_stale[s] == POS_CLEAN && !sq.cells.empty() && !sp.empty() && + sq.pos_min == sp.begin()->first) { + for (auto it = sp.upper_bound(sq.cells.back()); it != sp.end(); ++it) { + sq.cells.push_back(*it); + } + } + + // the appended tail accounts for every cell only if nothing before it was dropped, but an edit can + // regroup a sequence without changing its cell count, so a stale sequence must rebuild regardless + if (sq.cells.size() != sp.size() || mem_idx_stale[s] != POS_CLEAN) { + sq.cells.assign(sp.begin(), sp.end()); + sq.pools.clear(); + sq.j_next = 0; + sq.pos_min = sp.empty() ? 0 : sp.begin()->first; + } + + // Pools start at the first valid token + size_t j = sq.j_next; + if (hparams_idx.indexer_kpool_by_order) { + // consecutive cells in sequence order, whatever their positions + for (; j + kpool <= sq.cells.size(); j += kpool) { + sq.pools.push_back((uint32_t) j); + } + } else { + while (j + kpool <= sq.cells.size()) { + const llama_pos p0 = sq.cells[j].first; + if ((p0 - sq.pos_min) % (llama_pos) kpool != 0) { + ++j; + continue; + } + bool ok = true; + for (uint32_t k = 1; k < kpool; ++k) { + if (sq.cells[j + k].first != p0 + (llama_pos) k) { + ok = false; + break; + } + } + if (ok) { + sq.pools.push_back((uint32_t) j); + j += kpool; + } else { + ++j; + } + } + } + sq.j_next = j; + + lay.n_pool_real += (uint32_t) sq.pools.size(); + } + + return lay; +} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_status status) : status(status) {} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem) : + mem(mem), + // graph reservation walks a full context, and the sparse attention is built only when this is set + // without it the reserved worst case is the dense graph, so ggml-alloc must grow the buffer on the first decode + ns_ubatch(std::vector{ mem->get_mem_idx()->get_n_stream() }), + ctx_attn(mem->get_mem_attn()->init_full()), + ctx_idx (mem->get_mem_idx ()->init_full()), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) { + if (kpool_track()) { + mem->kpool_layout_update(); + auto st = kpool_build_sizes(); + const auto * idx = mem->get_mem_idx(); + const uint64_t n_pool_max = uint64_t(idx->get_size() / mem->get_kpool()) * idx->get_n_seq_max(); + GGML_ASSERT(n_pool_max <= UINT32_MAX - 64); + st.n_pool_real = std::max(st.n_pool_real, uint32_t(n_pool_max)); + st.n_new = st.n_pool_real; + st.n_new_g = std::max(st.n_new, 1u); + kpool_st = std::make_unique(std::move(st)); + i_kpool = 0; + } +} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + llama_context * lctx, + bool optimize) : + mem(mem), + ctx_attn(mem->get_mem_attn()->init_update(lctx, optimize)), + // update() applies a pending cross-stream seq_cp, else the copy keeps stale indexer keys + ctx_idx (mem->get_mem_idx ()->init_update(lctx, optimize)), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) {} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_swa, + slot_info_vec_t sinfos_idx, + std::vector ubatches) : + mem(mem), + ubatches(std::move(ubatches)), + ns_ubatch(llama_memory_idx_iswa_ns(sinfos_idx)), + sinfos_kpool(mem->get_kpool() > 0 && mem->get_kpool_by_order() ? sinfos_idx : slot_info_vec_t()), + // note: the children copy the ubatches; ctx_idx gets a copy of its own + ctx_attn(new llama_kv_cache_iswa_context(mem->get_mem_attn(), std::move(sinfos_base), std::move(sinfos_swa), this->ubatches)), + ctx_idx (new llama_kv_cache_context(mem->get_mem_idx(), std::move(sinfos_idx), this->ubatches)), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) { + // Sequence edits force the touched positions to re-pool. + mem_idx_stale_batch = mem->mem_idx_stale_get(); +} + +llama_memory_idx_iswa_context::~llama_memory_idx_iswa_context() = default; + +bool llama_memory_idx_iswa_context::next() { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + // Clear only after a successful ubatch. + if (i_next == 0 && mem != nullptr) { + mem->mem_idx_stale_clear(); + } + + ctx_attn->next(); + ctx_idx ->next(); + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_memory_idx_iswa_context::apply() { + assert(!llama_memory_status_is_fail(status)); + + bool res = true; + + res = res & ctx_attn->apply(); + res = res & ctx_idx ->apply(); + + // Extend the pool layout with this ubatch's cells, then pick what it must re-pool. + if (res && kpool_track()) { + mem->kpool_layout_update(); + if (!kpool_st) { + kpool_st = std::make_unique(); + } + kpool_build_state(get_ubatch()); + i_kpool = i_next; + } + + return res; +} + +llama_memory_status llama_memory_idx_iswa_context::get_status() const { + return status; +} + +const llama_ubatch & llama_memory_idx_iswa_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +bool llama_memory_idx_iswa_context::kpool_track() const { + // Derived from mem instead of being cached. + return mem != nullptr && mem->get_kpool() > 0 && !ns_ubatch.empty(); +} + +const llama_kv_cache_iswa_context * llama_memory_idx_iswa_context::get_attn() const { + return static_cast(ctx_attn.get()); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_base() const { + return get_attn()->get_base(); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_swa() const { + return get_attn()->get_swa(); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_idx() const { + return static_cast(ctx_idx.get()); +} + +uint32_t llama_memory_idx_iswa_context::get_n_stream() const { + GGML_ASSERT(i_next < ns_ubatch.size()); + + return ns_ubatch[i_next]; +} + +llama_memory_idx_iswa_context::kpool_access::kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd) : ctx(ctx) { + // rows are the per-token part (glm5-next: key | gate, qwen4exp: key), then the pooled key + const int64_t n_tok = k->ne[0] - n_embd; + GGML_ASSERT(n_tok > 0 && n_tok % n_embd == 0); + + const int64_t n_cells = k->ne[1]*k->ne[2]; + + // Pool indices can refer to other streams. Revisit these full-storage views if that changes: + // https://github.com/ggml-org/llama.cpp/pull/27773#discussion_r4130905603 + key_gate = ggml_view_2d(ctx, k, n_tok, n_cells, k->nb[1], 0); + pooled = ggml_view_2d(ctx, k, n_embd, n_cells, k->nb[1], ggml_row_size(k->type, n_tok)); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_key_gate(ggml_tensor * idxs) const { + return ggml_get_rows(ctx, key_gate, idxs); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const { + return ggml_set_rows(ctx, pooled, values, idxs); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_pooled(ggml_tensor * idxs) const { + return ggml_get_rows(ctx, pooled, idxs); +} + +llama_memory_idx_iswa_context::kpool_access llama_memory_idx_iswa_context::get_kpool_access( + ggml_context * ctx, int32_t il, int64_t n_embd) const { + GGML_ASSERT(mem != nullptr); + + return kpool_access(ctx, mem->get_mem_idx()->get_k_storage(il), n_embd); +} + +// k-pool DSA indexer (hy_v4_flash, glm5-next, qwen4exp QSA) + +// Sizes only, used by the full cache context so get_n_kpool() works during graph reserve. +llama_memory_idx_iswa_context::kpool_state llama_memory_idx_iswa_context::kpool_build_sizes() const { + const auto & lay = mem->kpool_layout_get(); + + kpool_state st; + st.n_pool_real = lay.n_pool_real; + + return st; +} + +// Which pools this ubatch must re-pool. +// Pool cache lifecycle: +// 1. cpy_k writes each token's key | gate into its idx cache row, pooled slot are zeroed. +// 2. This marks the pools the ubatch touches or completes as new, during decode that's one pool every kpool tokens, zero elsewise. +// 3. The graph pools only the new pools and set_rows each result into the pooled slot of the pool's last member row. +// 4. All pools are gathered in one get_rows via pool_cells, fresh ones just written, older ones from whatever batch last wrote them. +// A seq_* edit regroups the pools from the edited position on, so it stales them and the first ubatch of the next batch +// rebuilds them from the still-valid key | gate rows, rewriting the (possibly different) rep rows. +// Orphaned pooled slots are never cleared, a slot is only ever read through pool_cells, which follows the current grouping. +void llama_memory_idx_iswa_context::kpool_build_state(const llama_ubatch & ubatch) { + const auto & lay = mem->kpool_layout_get(); + auto & st = *kpool_st; + + const auto * idx = mem->get_mem_idx(); + const uint32_t kv_size = idx->get_size(); + const uint32_t kpool = mem->get_kpool(); + + st.n_pool_real = lay.n_pool_real; + st.n_new = 0; + if (++st.generation == 0) { + std::fill(st.is_new.begin(), st.is_new.end(), 0); + std::fill(st.rep_gen.begin(), st.rep_gen.end(), 0); + st.generation = 1; + } + st.is_new.resize(lay.n_pool_real, 0); + st.rep_gen.resize((size_t) kv_size*idx->get_n_stream(), 0); + + std::array pool_start; + + // a pool is marked once per rep: sequences sharing cells (a seq_cp, or tokens decoded for several sequences) + // share their pools, whose single pooled row they all read through pool_cells, so the scatter rows stay unique + auto mark = [&](llama_seq_id s, size_t k) { + const auto & sq = lay.seqs[s]; + const size_t rep = (size_t) sq.strm*kv_size + sq.cells[sq.pools[k] + kpool - 1].second; + if (st.rep_gen[rep] != st.generation) { + st.rep_gen[rep] = st.generation; + st.is_new[pool_start[s] + k] = st.generation; + ++st.n_new; + } + }; + + uint32_t ip = 0; + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + const auto & sq = lay.seqs[s]; + pool_start[s] = ip; + ip += (uint32_t) sq.pools.size(); + + // A sequence edit invalidates only pools ending after the edited position. + const llama_pos stale_from = i_next == 0 ? + mem_idx_stale_batch[s] : llama_memory_idx_iswa::POS_CLEAN; + if (stale_from == llama_memory_idx_iswa::POS_CLEAN) { + continue; + } + + auto first = std::lower_bound(sq.pools.begin(), sq.pools.end(), stale_from, + [&](uint32_t j, llama_pos p) { return sq.cells[j + kpool - 1].first < p; }); + for (auto it = first; it != sq.pools.end(); ++it) { + mark(s, it - sq.pools.begin()); + } + } + GGML_ASSERT(ip == st.is_new.size()); + + // in order mode a token's cell gives its rank, and the rank its pool: positions cannot, as an image shares one + const bool by_order = mem->get_kpool_by_order(); + const auto * sinfo = by_order ? &sinfos_kpool[i_next] : nullptr; + const uint32_t n_tps = by_order ? (uint32_t) sinfo->size() : 0; + + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + const llama_pos p = ubatch.pos[i]; + for (int32_t k = 0; k < ubatch.n_seq_id[i]; ++k) { + const llama_seq_id s = ubatch.seq_id[i][k]; + const auto & sq = lay.seqs[s]; + if (by_order) { + const int64_t r = kpool_rank(sq.cells, p, sinfo->idxs[i / n_tps][i % n_tps]); + GGML_ASSERT(r >= 0); + if ((size_t) r / kpool < sq.pools.size()) { + mark(s, (size_t) r / kpool); + } + continue; + } + auto it = std::upper_bound(sq.pools.begin(), sq.pools.end(), p, + [&](llama_pos pos, uint32_t j) { return pos < sq.cells[j].first; }); + if (it == sq.pools.begin()) { + continue; + } + --it; + if (p <= sq.cells[*it + kpool - 1].first) { + mark(s, it - sq.pools.begin()); + } + } + } + + // a ubatch touches at most t_s/kpool + 1 pools per sequence, pad to that bound so the graph keeps its shape + // as the count moves; reserve sizes the list for every pool the cache can hold, so never pad past n_pool_max + const uint32_t n_pool_max = kv_size / kpool * idx->get_n_seq_max(); + const uint32_t bound = ubatch.n_tokens/kpool + ubatch.n_seqs_unq; + st.n_new_g = std::max({st.n_new, 1u, std::min({bound, kpool_pad(st.n_pool_real) - 1, n_pool_max})}); +} + +const llama_memory_idx_iswa_context::kpool_state & llama_memory_idx_iswa_context::kpool_cur() const { + GGML_ASSERT(kpool_st != nullptr && i_kpool == i_next && "k-pool state read before apply()"); + + return *kpool_st; +} + +uint32_t llama_memory_idx_iswa_context::get_n_kpool() const { + return kpool_pad(kpool_cur().n_pool_real); +} + +uint32_t llama_memory_idx_iswa_context::get_n_kpool_new() const { + return kpool_cur().n_new_g; +} + +void llama_memory_idx_iswa_context::set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs, + ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep, + const llama_ubatch * ubatch, ggml_tensor * new_pool_pos) const { + GGML_ASSERT(mem != nullptr); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_cells->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_idxs->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_mask->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(tail_idxs->buffer)); + + const uint32_t kpool = mem->get_kpool(); + const uint32_t n_kv = get_idx()->get_n_kv(); + + const auto & st = kpool_cur(); + const auto & lay = mem->kpool_layout_get(); + + const uint32_t n_tokens = ubatch->n_tokens; + const uint32_t n_pool = (uint32_t) pool_cells->ne[0]; + const uint32_t n_new = st.n_new; + // the graph always pools at least one entry, padded to a stable bound, see kpool_build_state + const uint32_t n_new_g = st.n_new_g; + + const bool by_order = mem->get_kpool_by_order(); + + const uint32_t n_tail_slots = (uint32_t) tail_idxs->ne[0]; + const bool tail_full = n_tail_slots == kpool; + + GGML_ASSERT(n_pool == kpool_pad(st.n_pool_real)); + GGML_ASSERT(st.is_new.size() == st.n_pool_real); + GGML_ASSERT(pool_mask->ne[0] == (int64_t) n_pool && pool_mask->ne[1] == (int64_t) n_tokens); + GGML_ASSERT((n_tail_slots == kpool - 1 || tail_full) && tail_idxs->ne[1] == (int64_t) n_tokens); + GGML_ASSERT(pool_idxs->ne[0] == (int64_t) kpool && pool_idxs->ne[1] == (int64_t) n_pool); + GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_idxs->buffer)); + GGML_ASSERT(ggml_nelements(new_pool_idxs) == (int64_t) kpool*n_new_g); + // the graph always scatters the fresh pooled keys back into the cache + GGML_ASSERT(new_pool_rep != nullptr && ggml_backend_buffer_is_host(new_pool_rep->buffer)); + GGML_ASSERT(new_pool_rep->ne[0] == (int64_t) n_new_g); + if (new_pool_pos != nullptr) { + GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_pos->buffer)); + GGML_ASSERT(new_pool_pos->ne[0] == 4*(int64_t) n_new_g); + } + + const uint32_t kv_size = mem->get_mem_idx()->get_size(); + const uint32_t n_stream_kv = mem->get_mem_idx()->get_n_stream(); + + auto gcell = [&](const llama_memory_idx_iswa::kpool_layout::seq & sq, uint32_t cell) { + return (int64_t) sq.strm*kv_size + cell; + }; + + // Sequences present in this ubatch, pools of absent sequences must fall on the scatter sentinel row. + std::vector seq_in_ub(LLAMA_MAX_SEQ, 0); + for (uint32_t i = 0; i < n_tokens; ++i) { + for (int32_t k = 0; k < ubatch->n_seq_id[i]; ++k) { + seq_in_ub[ubatch->seq_id[i][k]] = 1; + } + } + + // a cell of this ubatch, written before any read, so the padded pools read a finite K row + int64_t dummy_cell = 0; + { + const llama_seq_id s = ubatch->seq_id[0][0]; + const auto & sq = lay.seqs[s]; + auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(ubatch->pos[0], 0u)); + GGML_ASSERT(it != sq.cells.end() && it->first == ubatch->pos[0]); + dummy_cell = gcell(sq, it->second); + } + + // in order mode a token sees the pools and the tail up to its own rank in the sequence, which its cell pins down + std::vector rank; + if (by_order) { + const auto & sinfo = sinfos_kpool[i_next]; + const uint32_t n_tps = (uint32_t) sinfo.size(); + + rank.resize(n_tokens); + for (uint32_t i = 0; i < n_tokens; ++i) { + rank[i] = kpool_rank(lay.seqs[ubatch->seq_id[i][0]].cells, ubatch->pos[i], sinfo.idxs[i / n_tps][i % n_tps]); + GGML_ASSERT(rank[i] >= 0); + } + } + + // padding and absent cells point at the n_kv sentinel row, one past the live cells + const int32_t sentinel = (int32_t) n_kv; + + float * gm = nullptr; + uint32_t n_sel = 0; + uint32_t n_top = 0; // Pools per token in the selection. + uint32_t n_tail_sel = 0; + if (sel_mask != nullptr) { + GGML_ASSERT(ggml_backend_buffer_is_host(sel_mask->buffer)); + GGML_ASSERT(sel_mask->type == GGML_TYPE_F32); + GGML_ASSERT(sel_mask->ne[3] == (int64_t) n_tokens && sel_mask->ne[1] == 1 && sel_mask->ne[2] == 1); + n_sel = (uint32_t) sel_mask->ne[0]; + n_tail_sel = tail_full ? kpool : n_sel % kpool; + GGML_ASSERT(n_sel >= n_tail_sel && (n_sel - n_tail_sel) % kpool == 0); + GGML_ASSERT(n_tail_sel == 0 || n_tail_sel == n_tail_slots); + n_top = (n_sel - n_tail_sel) / kpool; + gm = (float *) sel_mask->data; + } + + // pools are laid out per sequence + std::vector seq_pool_start(LLAMA_MAX_SEQ, 0); + std::vector pool_end; + pool_end.reserve(n_pool); + + int32_t * pcell = (int32_t *) pool_cells->data; + int32_t * pidx = (int32_t *) pool_idxs->data; + int32_t * nidx = (int32_t *) new_pool_idxs->data; + int64_t * nrep = (int64_t *) new_pool_rep->data; + int32_t * npos = new_pool_pos != nullptr ? (int32_t *) new_pool_pos->data : nullptr; + + if (npos != nullptr) { + std::fill(npos, npos + 4*n_new_g, 0); + } + + uint32_t i_new = 0; + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + const auto & sq = lay.seqs[s]; + seq_pool_start[s] = (uint32_t) pool_end.size(); + + const bool inert = n_stream_kv > 1 && !seq_in_ub[s]; + + for (size_t pi = 0; pi < sq.pools.size(); ++pi) { + const uint32_t j = sq.pools[pi]; + const uint32_t ip = (uint32_t) pool_end.size(); + GGML_ASSERT(ip + 1 < n_pool); + + // The pooled key lives in the last member's row. + const uint32_t rep = sq.cells[j + kpool - 1].second; + pcell[ip] = (int32_t) gcell(sq, rep); + + for (uint32_t k = 0; k < kpool; ++k) { + pidx[(size_t) ip*kpool + k] = inert ? sentinel : (int32_t) sq.cells[j + k].second; + } + + if (st.is_new[ip] == st.generation) { + GGML_ASSERT(i_new < n_new); + for (uint32_t k = 0; k < kpool; ++k) { + nidx[(size_t) i_new*kpool + k] = (int32_t) gcell(sq, sq.cells[j + k].second); + } + nrep[i_new] = gcell(sq, rep); + if (npos != nullptr) { + // a pooled key is rotated to the M-RoPE position of its first member + const uint32_t c = sq.cells[j].second; + const auto & e = mem->get_mem_idx()->get_cells(s).ext_get(c); + npos[0*n_new_g + i_new] = sq.cells[j].first; + npos[1*n_new_g + i_new] = e.y; + npos[2*n_new_g + i_new] = e.x; + npos[3*n_new_g + i_new] = sq.cells[j].first; + } + ++i_new; + } + + pool_end.push_back(sq.cells[j + kpool - 1].first); + } + } + GGML_ASSERT(i_new == n_new); + + // Padded entries re-pool cells whose pooled slot is never read: only the reps of complete pools are read. + // Each entry takes its own cell, entries sharing one would write it from several threads in the scatter. + if (n_new_g > n_new) { + std::vector reps(pcell, pcell + pool_end.size()); + std::sort(reps.begin(), reps.end()); + + int64_t pad_cell = 0; + for (uint32_t i = n_new; i < n_new_g; ++i, ++pad_cell) { + while (std::binary_search(reps.begin(), reps.end(), pad_cell)) { + ++pad_cell; + } + GGML_ASSERT(pad_cell < (int64_t) kv_size*n_stream_kv); + for (uint32_t k = 0; k < kpool; ++k) { + nidx[(size_t) i*kpool + k] = (int32_t) pad_cell; + } + nrep[i] = pad_cell; + } + } + + const uint32_t n_pool_real = (uint32_t) pool_end.size(); + for (uint32_t ip = n_pool_real; ip < n_pool; ++ip) { + pcell[ip] = (int32_t) dummy_cell; // pool_cells always addresses the K storage + for (uint32_t k = 0; k < kpool; ++k) { + pidx[(size_t) ip*kpool + k] = sentinel; + } + } + + // a pool is visible when it belongs to the token's sequence and ends at or before it, unless reserved by a full-width tail + auto fill_mask = [&](auto * data) { + using T = std::remove_pointer_t; + const T keep = llama_cast(0.0f); + const T drop = llama_cast(-INFINITY); + + for (uint32_t i = 0; i < n_tokens; ++i) { + const llama_seq_id s = ubatch->seq_id[i][0]; + const llama_pos p = ubatch->pos[i]; + + T * row = data + (size_t) i*n_pool; + std::fill(row, row + n_pool, drop); + + const uint32_t p0 = seq_pool_start[s]; + const uint32_t p1 = p0 + (uint32_t) lay.seqs[s].pools.size(); + const uint32_t nv = by_order ? std::min(p1 - p0, (uint32_t) ((rank[i] + !tail_full)/kpool)) : + (uint32_t) ((tail_full ? std::lower_bound(pool_end.begin() + p0, pool_end.begin() + p1, p) : + std::upper_bound(pool_end.begin() + p0, pool_end.begin() + p1, p)) - (pool_end.begin() + p0)); + std::fill(row + p0, row + p0 + nv, keep); + + // Finite visible pools occupy the first min(nv, n_top) ranked slots. + if (gm != nullptr) { + const uint32_t nvc = std::min(nv, n_top); + float * grow = gm + (size_t) i*n_sel; + std::fill(grow, grow + (size_t) nvc*kpool, 0.0f); + std::fill(grow + (size_t) nvc*kpool, grow + (size_t) n_top*kpool, -INFINITY); + } + } + }; + if (pool_mask->type == GGML_TYPE_F16) { + fill_mask((ggml_fp16_t *) pool_mask->data); + } else { + fill_mask((float *) pool_mask->data); + } + + int32_t * tidx = (int32_t *) tail_idxs->data; + for (uint32_t i = 0; i < n_tokens; ++i) { + const llama_seq_id s = ubatch->seq_id[i][0]; + const llama_pos p = ubatch->pos[i]; + const auto & sq = lay.seqs[s]; + + const int64_t pos = by_order ? rank[i] : p - sq.pos_min; + const uint32_t n_tail = tail_full ? (uint32_t) (pos % kpool) + 1 : (uint32_t) ((pos + 1) % kpool); + + for (uint32_t k = 0; k < n_tail_slots; ++k) { + int32_t cell = sentinel; + bool real = false; + if (k < n_tail && by_order) { + const uint32_t c = sq.cells[rank[i] - k].second; + cell = (int32_t) c; + real = true; + } else if (k < n_tail) { + const llama_pos pt = p - (llama_pos) k; + auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(pt, 0u)); + if (it != sq.cells.end() && it->first == pt) { + cell = (int32_t) it->second; + real = true; + } + } + tidx[(size_t) i*n_tail_slots + k] = cell; + + if (gm != nullptr && n_tail_sel != 0) { + gm[(size_t) i*n_sel + (size_t) n_top*kpool + k] = real ? 0.0f : -INFINITY; + } + } + } +} diff --git a/src/llama-memory-idx-iswa.h b/src/llama-memory-idx-iswa.h new file mode 100644 index 0000000000000000000000000000000000000000..3997de8b3344b06f4604a6d8e4bf2c145d3a18a5 --- /dev/null +++ b/src/llama-memory-idx-iswa.h @@ -0,0 +1,249 @@ +#pragma once + +#include "llama-kv-cache-iswa.h" + +#include +#include +#include +#include + +// +// llama_memory_idx_iswa +// + +// llama_kv_cache_iswa plus a second cache with one indexer key per token, for block-sparse attention (hy_v4_flash) +// only the non-SWA layers own an indexer, so it mirrors the base child alone: same size, padding, streams and +// slots, so cell j is one token in both + +class llama_memory_idx_iswa : public llama_memory_i { +public: + llama_memory_idx_iswa( + const llama_model & model, + bool no_alloc, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + /* layer filters */ + const layer_filter_cb & filter, + /* the SWA layers never own an indexer, so this must select non-SWA layers only */ + const layer_filter_cb & filter_idx); + + // Defined out of line because kpool_layout is incomplete here. + ~llama_memory_idx_iswa(); + + // + // llama_memory_i + // + + llama_memory_context_ptr init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) override; + + llama_memory_context_ptr init_full() override; + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; + + void clear(bool data) override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + + std::map memory_breakdown() const override; + + // state write/load + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; + + // + // llama_memory_idx_iswa specific API + // + + llama_kv_cache_iswa * get_mem_attn() const; + llama_kv_cache * get_mem_idx () const; + + // The model's indexer pool size. + uint32_t get_kpool() const { return hparams_idx.indexer_kpool; } + + // Whether pools are kpool consecutive cells in sequence order (qwen4exp) instead of kpool consecutive positions. + bool get_kpool_by_order() const { return hparams_idx.indexer_kpool_by_order; } + + // Which cells of a sequence make up which pool of kpool consecutive positions (or cells, in order mode). + // It is kept here because it outlives the batch: pools are fixed by the positions relative to the + // sequence's first one, so a ubatch only ever appends to it. Sequence edits drop it, see mem_idx_stale. + struct kpool_layout; + + const kpool_layout & kpool_layout_update(); + const kpool_layout & kpool_layout_get() const; + + // The pooled keys persist in the idx cache across batches. A sequence edit can regroup the pools + // from some position on, which stales every pooled key at or after it. POS_CLEAN means none. + using stale_pos_t = std::array; + + static constexpr llama_pos POS_CLEAN = std::numeric_limits::max(); + + static stale_pos_t stale_pos_clean() { + stale_pos_t res; + res.fill(POS_CLEAN); + return res; + } + + const stale_pos_t & mem_idx_stale_get() const { return mem_idx_stale; } + void mem_idx_stale_clear() { mem_idx_stale.fill(POS_CLEAN); } + +private: + // forget seq_id (all of it if seq_id < 0) in both caches at once, so a failed restore cannot leave them out of step + // seq_id < 0 drops the whole context, as the caches themselves do on a failed restore + void state_drop(llama_seq_id seq_id); + + // the indexer cache holds one key head per layer, so it needs its own hparams: + // llama_kv_cache keeps a reference to what it is given + llama_hparams hparams_idx; + + const std::unique_ptr mem_attn; + const std::unique_ptr mem_idx; + + // unique_ptr because kpool_layout is incomplete here + std::unique_ptr kpool_lay; + + // seq_id < 0 stales every sequence, p0 < 0 stales the sequence from its first position + void mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0); + + // the position an edit at p0 stales the sequence from + llama_pos mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const; + + stale_pos_t mem_idx_stale = stale_pos_clean(); +}; + +class llama_memory_idx_iswa_context : public llama_memory_context_i { +public: + class kpool_access { + public: + ggml_tensor * gather_key_gate(ggml_tensor * idxs) const; + ggml_tensor * scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const; + ggml_tensor * gather_pooled(ggml_tensor * idxs) const; + + private: + friend class llama_memory_idx_iswa_context; + + kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd); + + ggml_context * ctx; + ggml_tensor * key_gate; + ggml_tensor * pooled; + }; + + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + // used for errors + explicit llama_memory_idx_iswa_context(llama_memory_status status); + + // used to create a full-cache context + explicit llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem); + + // used to create an update context + llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + llama_context * lctx, + bool optimize); + + // used to create a batch processing context from a batch + llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_swa, + slot_info_vec_t sinfos_idx, + std::vector ubatches); + + ~llama_memory_idx_iswa_context(); // Defined out of line because kpool_state is incomplete here. + + // + // llama_memory_context_i + // + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + // + // llama_memory_idx_iswa_context specific API + // + + const llama_kv_cache_iswa_context * get_attn() const; + + // the per-layer attention context: base for the non-SWA layers, swa for the rest + const llama_kv_cache_context * get_base() const; + const llama_kv_cache_context * get_swa () const; + + const llama_kv_cache_context * get_idx () const; + + // streams in the current slot info, the `ns` of get_k/get_v; 1 if unified + uint32_t get_n_stream() const; + + // hy_v4_flash, complete pools of kpool cells per sequence, scored as whole pools. + uint32_t get_n_kpool () const; // Padded pool count, where the last pool is always unused. + uint32_t get_n_kpool_new() const; // Pools to re-pool this ubatch, padded to a stable bound, never below 1. + kpool_access get_kpool_access(ggml_context * ctx, int32_t il, int64_t n_embd) const; + // sel_mask (F32 [n_sel, 1, 1, n_tokens], can be null): 0 for the live selection slots, -inf for the dead ones + // new_pool_pos (I32 [4*n_new]): M-RoPE position of each new pool's first member, for pooled keys rotated at pooling time + // tail_idxs width: kpool reserves the current group; kpool - 1 keeps only the incomplete tail. + void set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs, + ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep, + const llama_ubatch * ubatch, ggml_tensor * new_pool_pos = nullptr) const; + +private: + llama_memory_idx_iswa * mem = nullptr; + + // the index of the next ubatch to process + size_t i_next = 0; + + std::vector ubatches; + + // streams per ubatch, read from the slot infos before ctx_idx takes them + // declared before sinfos_kpool and ctx_idx, so it is initialised while sinfos_idx is still intact + const std::vector ns_ubatch; + + // the indexer cells of each ubatch, kept for pools in cache order (qwen4exp): token s*n + i of ubatch u + // sits in cell idxs[s][i] of stream strm[s] of sinfos_kpool[u], and several cells can share a position + const slot_info_vec_t sinfos_kpool; + + const llama_memory_context_ptr ctx_attn; + const llama_memory_context_ptr ctx_idx; + + const llama_memory_status status; + + // Which pools of the layout this ubatch must re-pool. The layout itself belongs to the memory. + struct kpool_state; + kpool_state kpool_build_sizes() const; + void kpool_build_state(const llama_ubatch & ubatch); + const kpool_state & kpool_cur() const; + + // unique_ptr because kpool_state is incomplete here. + std::unique_ptr kpool_st; + + // The ubatch kpool_st was built for, guards against reads before apply. + size_t i_kpool = SIZE_MAX; + + // Whether this context tracks k-pool states. + bool kpool_track() const; + + // Positions each sequence must re-pool from, cleared only after the first ubatch succeeds + llama_memory_idx_iswa::stale_pos_t mem_idx_stale_batch = llama_memory_idx_iswa::stale_pos_clean(); +}; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 36f3850d518019e4e9612b2fb1bbe1e41145c74f..1287e80519ddcf57149cbada2edce014b2cc2a8d 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -357,10 +357,14 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks); add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size); } - if (hparams.ple_n_heads > 0) { - add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); - add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map); - } + } + + // any arch with an Engram module needs these, not just deepseek41. The token map is what says + // the arch has one at all: qwen4exp sets ple_n_heads for a plain per-layer embedding and has no + // hashed n-gram table, so it must not pick up a stray pad id. + if (hparams.ple_n_heads > 0 && !model->engram_token_map.empty()) { + add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); + add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map); } // the PLE group only means anything whole: write all of it or none diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7529f55c2025f259d10f18f1c59c34a62cb3a40d..a07ab12269b6d91e60cfdd062cfb320c897861bd 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -17,6 +17,7 @@ #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" #include "llama-memory-hybrid-idx.h" +#include "llama-memory-idx-iswa.h" #include "llama-memory-recurrent.h" #include "llama.h" @@ -306,6 +307,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_hy_v3(params); case LLM_ARCH_HY_V4: return new llama_model_hy_v4(params); + case LLM_ARCH_HY_V4_FLASH: + return new llama_model_hy_v4_flash(params); case LLM_ARCH_SMOLLM3: return new llama_model_smollm3(params); case LLM_ARCH_OPENAI_MOE: @@ -2638,7 +2641,8 @@ void llama_model::print_info() const { if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4 || - arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4) { + arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4 || + arch == LLM_ARCH_HY_V4_FLASH) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q); LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv); @@ -3029,6 +3033,75 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr); } } break; + case LLM_ARCH_HY_V4_FLASH: + { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + if (hparams.indexer_top_k == 0) { + // no indexer, so the sparse layers attend their whole context: a plain iSWA pair is enough + llama_kv_cache_iswa::layer_filter_cb filter = [&](uint32_t il) { + return il < hparams.n_layer(); + }; + + // the draft context holds the MTP block alone + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + if (hparams.n_layer_nextn == 0) { + throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp"); + } + filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; + } + + res = new llama_kv_cache_iswa( + *this, + params.no_alloc, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + nullptr, + filter, + nullptr, + nullptr); + } else { + // the indexer lives on the sparse layers, which are the non-SWA ones + llama_memory_idx_iswa::layer_filter_cb filter = [&](uint32_t il) { + return il < hparams.n_layer(); + }; + llama_memory_idx_iswa::layer_filter_cb filter_idx = [&](uint32_t il) { + return il < hparams.n_layer() && !hparams.is_swa(il) && hparams.is_indexer_full(il); + }; + + // the draft context holds the MTP block alone, which is a sparse layer and so + // owns an indexer of its own + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + if (hparams.n_layer_nextn == 0) { + throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp"); + } + filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; + filter_idx = [&](uint32_t il) { return il >= hparams.n_layer(); }; + } + + res = new llama_memory_idx_iswa( + *this, + params.no_alloc, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + std::move(filter), + std::move(filter_idx)); + } + } break; case LLM_ARCH_DOTS3NOTE: { GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); @@ -3616,6 +3689,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM; // HY_V4 rotates consecutive pairs, matching the reference implementation case LLM_ARCH_HY_V4: + case LLM_ARCH_HY_V4_FLASH: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 diff --git a/src/llama-model.h b/src/llama-model.h index da4964c98f7aee145ad132e641d082f1fc4eed49..9d11ce5177192335cc2e1f76fc1531002492cf71 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -612,10 +612,20 @@ struct llama_layer { // glm5-next k-pool indexer struct ggml_tensor * indexer_kpool_gate = nullptr; struct ggml_tensor * indexer_kpool_ape = nullptr; + // hy_v4_flash: the pool runs over a projection of the normed key, and the pooled result is + // normed again before it is scored + struct ggml_tensor * indexer_kpool_value = nullptr; + struct ggml_tensor * indexer_kpool_norm = nullptr; // deepseek41 struct ggml_tensor * engram_kv = nullptr; struct ggml_tensor * engram_gate = nullptr; // q_weight*k_weight folded + // hy_v4_flash only: biases on engram_kv, the per-head norm of the looked-up rows, and the + // scale/offset of the delta the module writes back + struct ggml_tensor * engram_kv_b = nullptr; + struct ggml_tensor * engram_lookup_norm = nullptr; + struct ggml_tensor * engram_pre_gamma = nullptr; + struct ggml_tensor * engram_hc_scale = nullptr; // MSA struct ggml_tensor * index_q_proj = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c24785818819bb5be9ef04939ce8b24be9d55d..3f1b58e008880915027e3fa7ed3719e914d685d8 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -326,6 +326,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // Engram gate is an elementwise weight (ggml_mul), not a matmul operand quantize &= name.find("engram_gate.weight") == std::string::npos; + // same for these: they are the second operand of a ggml_add / ggml_mul, which has no + // quantized kernel, so quantizing them aborts at run time rather than degrading quality + quantize &= name.find("engram_kv_b.weight") == std::string::npos; + quantize &= name.find("engram_hc_scale.weight") == std::string::npos; + quantize &= name.find("indexer_compressor_ape.weight") == std::string::npos; + // these are very small (e.g. 4x4) quantize &= name.find("altup") == std::string::npos; quantize &= name.find("laurel") == std::string::npos; diff --git a/src/models/hy-v4-flash.cpp b/src/models/hy-v4-flash.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0f7b9417b805064071fd5c96a2473bcd1e828ccf --- /dev/null +++ b/src/models/hy-v4-flash.cpp @@ -0,0 +1,1319 @@ +#include "models.h" + +#include "llama-kv-cache.h" +#include "llama-kv-cache-iswa.h" +#include "llama-memory-idx-iswa.h" + +#include + +// HY_V4_FLASH reuses hy_v4's iHC, gated MLA and learnable sink. What it adds: +// - a hybrid stack: "sparse" full-attention layers and sliding-window layers, with their own +// head count and rope base, over one shared MLA latent geometry +// - sandwich norms after the attention output and after the FFN +// - routed experts that run at n_expert_latent instead of n_embd + +// a token that cannot be hashed: outside the map, or an embedding (image) position +static constexpr int64_t HY_V4_FLASH_ENGRAM_DEAD = -1; + +// The reference builds every Engram norm as an RMSNorm with eps left unset, which resolves to the +// fp32 machine epsilon rather than to the model's rms_norm_eps. +static constexpr float HY_V4_FLASH_ENGRAM_EPS = 1.1920929e-07f; + +static size_t hy_v4_flash_elem_offset(const ggml_tensor * t, int64_t i) { + return ggml_row_size(t->type, i); +} + +static ggml_tensor * hy_v4_flash_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) { + return ggml_view_1d(ctx, t, ne0, hy_v4_flash_elem_offset(t, i0)); +} + +static ggml_tensor * hy_v4_flash_view_2d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t ne1, int64_t i0) { + return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], hy_v4_flash_elem_offset(t, i0)); +} + +void llama_model_hy_v4_flash::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false); + + // routed experts only; the shared and dense FFNs are not clamped + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false); + + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + ml.get_key(LLM_KV_HYPER_CONNECTION_MAGNITUDE, hparams.hc_magnitude); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + // Compressed indexer, optional: present on the sparse layers only. A file without it falls + // back to those layers attending their whole context, which is what phase 1 shipped, so the + // keys are read only when indexer.top_k says there is an indexer at all. + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k, false); + if (hparams.indexer_top_k > 0) { + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL, hparams.indexer_kpool); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail, false); + ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); + + // indexer.types is what says which layers own one. Without it nothing does, and a file that + // carries the other indexer keys anyway must not select a cache this graph cannot drive. + bool any = false; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + any = any || hparams.is_indexer_full(il); + } + + if (!any) { + hparams.indexer_top_k = 0; + } else { + if (hparams.indexer_kpool < 2 || hparams.indexer_top_k % hparams.indexer_kpool != 0) { + throw std::runtime_error(format( + "hy_v4_flash: indexer top_k %u must be a multiple of the pool size %u", + hparams.indexer_top_k, hparams.indexer_kpool)); + } + if (hparams.indexer_kpool_select_tail && hparams.indexer_top_k / hparams.indexer_kpool < 2) { + throw std::runtime_error("hy_v4_flash: indexer top_k must reserve at least one history group and one tail group"); + } + // the indexer cache mirrors the full-attention child, so a window layer cannot own one + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (!hparams.is_indexer_full(il)) { + continue; + } + if (hparams.is_swa(il)) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u is a sliding-window layer and cannot own an indexer", il)); + } + // the indexer splits each head into a nope front and a rope tail, so the tail must fit + if (hparams.n_rot(il) >= hparams.indexer_head_size) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u indexer key length %u must be larger than the rope dimension %u", + il, hparams.indexer_head_size, hparams.n_rot(il))); + } + } + } + } + + GGML_ASSERT(hparams.is_mla()); + + if (!hparams.is_swa_any()) { + throw std::runtime_error("hy_v4_flash: no sliding-window layer, expected a hybrid stack"); + } + + // One latent geometry for both layer types: only the head count differs, so the graph keeps a + // single latent cache and reads one set of MLA dimensions. dots3note is the arch that varies + // these per layer, and a file written for that shape would be read here as if it did not, so + // refuse it instead of building mis-sized wk_b / wv_b / wo views. + uint32_t n_embd_head_k_mla_swa = 0; + uint32_t n_embd_head_v_mla_swa = 0; + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, n_embd_head_k_mla_swa, false); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, n_embd_head_v_mla_swa, false); + + if ((n_embd_head_k_mla_swa && n_embd_head_k_mla_swa != hparams.n_embd_head_k_mla()) || + (n_embd_head_v_mla_swa && n_embd_head_v_mla_swa != hparams.n_embd_head_v_mla())) { + throw std::runtime_error(format( + "hy_v4_flash: needs one MLA geometry for both layer types, but the " + "sliding-window layers declare key/value %u/%u against %u/%u", + n_embd_head_k_mla_swa, n_embd_head_v_mla_swa, + hparams.n_embd_head_k_mla(), hparams.n_embd_head_v_mla())); + } + + // MLA is driven as MQA here: one shared latent row per token, so a file claiming more + // key/value heads would silently disagree with the cache this graph builds + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.n_head_kv(il) != 1) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u has n_head_kv %u, expected 1 for MLA", + il, hparams.n_head_kv(il))); + } + } + + // Engram, optional: the same n-gram hash tables deepseek41 carries, with one module per Engram + // layer. Multipliers, primes and offsets are all stored flat as [module][...] and indexed at + // module stride, so their lengths pin down the module count. + std::vector e_layers; + ml.get_arr(LLM_KV_PLE_LAYERS, e_layers, false); + if (!e_layers.empty()) { + std::vector mult; + std::vector offsets; + std::vector primes; + ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size); + ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram); + ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim); + ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult); + ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets); + ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes); + ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); + ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map); + + const size_t n_e = e_layers.size(); + const size_t n_gram = hparams.ple_ngram_size; + const size_t n_cols = n_gram >= 2 ? (n_gram - 1)*hparams.ple_heads_per_ngram : 0; + if (n_cols == 0 || hparams.ple_head_dim == 0 || + mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols || + mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS || + hparams.engram_pad_id >= engram_token_map.size()) { + throw std::runtime_error("hy_v4_flash engram metadata is inconsistent"); + } + for (size_t e = 0; e < n_e; ++e) { + if (e_layers[e] >= hparams.n_layer() || (e > 0 && e_layers[e] <= e_layers[e - 1])) { + throw std::runtime_error(format("hy_v4_flash engram layer %u is out of range or out of order", e_layers[e])); + } + hparams.is_ple_impl.set(e_layers[e]); + } + for (size_t c = 0; c < n_e*n_cols; ++c) { + // a bucket count of 0 traps in the hash; row ids are int32 + if (primes[c] < 2 || offsets[c] + primes[c] > (uint64_t) INT32_MAX) { + throw std::runtime_error(format("hy_v4_flash engram head %zu: bucket range out of range", c)); + } + hparams.ple_head_offsets[c] = (uint32_t) offsets[c]; + hparams.ple_head_vocab_sizes[c] = (uint32_t) primes[c]; + } + std::copy(mult.begin(), mult.end(), hparams.ple_layer_multipliers.begin()); + hparams.ple_n_heads = (uint32_t) n_cols; + hparams.ple_n_modules = (uint32_t) n_e; + } + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_hy_v4_flash::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + GGML_ASSERT(n_embd_head_qk_nope >= 1); + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_ff_exp = hparams.n_ff_exp(); + const int64_t n_expert_shared = hparams.n_expert_shared; + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t n_embd_latent = hparams.n_expert_latent ? hparams.n_expert_latent : n_embd; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc * n_embd, hc}, 0); + hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc}, 0); + hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + + const int64_t ple_n_heads = hparams.ple_n_heads; + const int64_t ple_head_dim = hparams.ple_head_dim; + if (ple_n_heads > 0) { + // every module's buckets live in one table, with the converter rebasing each module's head + // offsets into it, so the row count is the end of the last bucket + int64_t n_rows = 0; + for (size_t c = 0; c < (size_t) hparams.ple_n_modules*ple_n_heads; ++c) { + n_rows = std::max(n_rows, (int64_t) hparams.ple_head_offsets[c] + hparams.ple_head_vocab_sizes[c]); + } + // 118 GiB at the full size: read lazily, so only the rows a batch hashes to are faulted in + per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {ple_head_dim, n_rows}, TENSOR_READ_LAZY); + } + + // the MTP block sits after the trunk and is a sparse layer in its own right, minus the + // hyper-connections: it is not part of the hc stack the trunk runs + for (int i = 0; i < (int) hparams.n_layer_all; ++i) { + auto & layer = layers[i]; + const bool is_nextn = i >= n_layer; + + // the sliding-window layers carry more heads over the same latent geometry + const int64_t n_head_l = hparams.n_head(i); + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head_l}, 0); + + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, 0); + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0); + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, 0); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, 0); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l * n_embd_head_v_mla}, 0); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + + if (!is_nextn) { + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {2 * hc}, 0); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {2}, 0); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {2 * hc}, 0); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {2}, 0); + } else { + // the draft block folds the next token's embedding into the trunk's final hidden state + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, 0); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, 0); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, 0); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, 0); + } + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + + // is_indexer_full() only covers the trunk, and the draft block always owns an indexer + if (hparams.indexer_top_k > 0 && (is_nextn || hparams.is_indexer_full(i))) { + const int64_t n_idx = hparams.indexer_head_size; + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t kpool = hparams.indexer_kpool; + + layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd, n_idx}, 0); + layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx}, 0); + // the pooled quantity is a projection of the normed key, not the key itself, and the + // gate that weights the pooling is another one: both square at the indexer width + layer.indexer_kpool_value = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_VALUE, "weight", i), {n_idx, n_idx}, 0); + layer.indexer_kpool_gate = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_GATE, "weight", i), {n_idx, n_idx}, 0); + layer.indexer_kpool_ape = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_APE, "weight", i), {n_idx, kpool}, 0); + layer.indexer_kpool_norm = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_NORM, "weight", i), {n_idx}, 0); + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx*n_idx_head}, 0); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0); + } + + if (!is_nextn && engram_index(i) >= 0) { + // the hc key projections and the single value projection are fused into one matmul, as + // deepseek41 does; unlike deepseek41 they carry a bias + layer.engram_kv = create_tensor(tn(LLM_TENSOR_ENGRAM_KV, "weight", i), {ple_n_heads * ple_head_dim, n_embd * (hc + 1)}, 0); + layer.engram_kv_b = create_tensor(tn(LLM_TENSOR_ENGRAM_KV_B, "weight", i), {n_embd, hc + 1}, 0); + layer.engram_gate = create_tensor(tn(LLM_TENSOR_ENGRAM_GATE, "weight", i), {n_embd, hc}, 0); + layer.engram_lookup_norm = create_tensor(tn(LLM_TENSOR_ENGRAM_LOOKUP_NORM, "weight", i), {ple_head_dim, ple_n_heads}, 0); + layer.engram_pre_gamma = create_tensor(tn(LLM_TENSOR_ENGRAM_PRE_GAMMA, "weight", i), {n_embd}, 0); + layer.engram_hc_scale = create_tensor(tn(LLM_TENSOR_ENGRAM_HC_SCALE, "weight", i), {n_embd, hc}, 0); + } + + if (!is_nextn && i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + if (n_expert == 0 || n_expert_used == 0) { + throw std::runtime_error("hy_v4_flash: n_expert and n_expert_used must be > 0"); + } + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + + if (hparams.n_expert_latent) { + layer.ffn_routed_down = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_DOWN, "weight", i), {n_embd, n_embd_latent}, 0); + layer.ffn_routed_up = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_UP, "weight", i), {n_embd_latent, n_embd}, 0); + } + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_latent, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + } + } +} + +int llama_model_hy_v4_flash::engram_index(int il) const { + if (hparams.ple_n_heads == 0 || !hparams.is_ple(il)) { + return -1; + } + int e = 0; + for (int j = 0; j < il; ++j) { + e += hparams.is_ple(j); + } + return e; +} + +void llama_model_hy_v4_flash::engram_hash(int e, const llama_ubatch & ubatch, + const std::vector & prev, + std::vector & ids, std::vector & valid) const { + const int64_t n_gram = hparams.ple_ngram_size; + const int64_t n_prev = n_gram - 1; + const int64_t n_head = hparams.ple_heads_per_ngram; + const int64_t n_cols = hparams.ple_n_heads; + + const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram; + const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols; + const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols; + const int64_t pad = engram_token_map[hparams.engram_pad_id]; + + const auto cid = [&](llama_token t) -> int64_t { + return t < 0 || t >= (llama_token) engram_token_map.size() ? HY_V4_FLASH_ENGRAM_DEAD : engram_token_map[t]; + }; + + ids.resize(ubatch.n_tokens*n_cols); + valid.resize(ubatch.n_tokens*n_cols); + + std::vector toks(n_gram); + + for (int64_t i = 0; i < ubatch.n_tokens; ++i) { + // A history shorter than the n-gram is padded, not dropped: the reference shifts the token + // window and fills the overhang with the pad id, so a real row is looked up even at the + // start of a sequence. Only an embedding (image) position has no n-gram at all, and there + // the reference marks every column invalid and masks the whole contribution. + const bool is_embd = !ubatch.token || (ubatch.is_mixed() && ubatch.type[i]); + int64_t c = is_embd ? HY_V4_FLASH_ENGRAM_DEAD : cid(ubatch.token[i]); + const bool dead = c == HY_V4_FLASH_ENGRAM_DEAD; + toks[0] = dead ? pad : c; + for (int64_t s = 1; s < n_gram; ++s) { + c = ubatch.pos[i] - s < 0 ? HY_V4_FLASH_ENGRAM_DEAD : cid(prev[i*n_prev + (n_prev - s)]); + toks[s] = c == HY_V4_FLASH_ENGRAM_DEAD ? pad : c; + } + + uint64_t rolling = toks[0]*mult[0]; + for (int64_t o = 1; o < n_gram; ++o) { + rolling ^= toks[o]*mult[o]; + for (int64_t h = 0; h < n_head; ++h) { + const int64_t col = (o - 1)*n_head + h; + ids [i*n_cols + col] = (int32_t) (rolling % primes[col] + offs[col]); + valid[i*n_cols + col] = dead ? 0.0f : 1.0f; + } + } + } +} + +// the row ids and their validity are host-side, so they are graph inputs rather than computed +class llm_graph_input_hy_v4_flash : public llm_graph_input_i { +public: + llm_graph_input_hy_v4_flash(const llama_model_hy_v4_flash & model, + const llama_kv_cache_context * mctx) : model(model), mctx(mctx) {} + + void set_input(const llama_ubatch * ubatch) override; + bool can_reuse(const llm_graph_params & params) override; + + std::vector engram_ids; + std::vector engram_valid; + + const llama_model_hy_v4_flash & model; + const llama_kv_cache_context * mctx; +}; + +void llm_graph_input_hy_v4_flash::set_input(const llama_ubatch * ubatch) { + if (engram_ids.empty()) { + return; + } + + // a token shared by several sequences has the same history in each, so its first one resolves it + std::vector prev; + mctx->get_prev_tokens(*ubatch, model.hparams.ple_ngram_size - 1, prev); + + // hash every module before the first gather, so a lazily read table gets one batched readahead + std::vector> ids(engram_ids.size()); + std::vector> valid(engram_ids.size()); + std::vector all; + for (size_t e = 0; e < engram_ids.size(); ++e) { + model.engram_hash((int) e, *ubatch, prev, ids[e], valid[e]); + all.insert(all.end(), ids[e].begin(), ids[e].end()); + } + if (model.can_prefetch.count(model.per_layer_tok_embd)) { + llama_prefetch_rows(model.per_layer_tok_embd, all.data(), all.size()); + } + for (size_t e = 0; e < engram_ids.size(); ++e) { + if (engram_ids[e]->buffer) { + ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t)); + } + if (engram_valid[e]->buffer) { + ggml_backend_tensor_set(engram_valid[e], valid[e].data(), 0, valid[e].size()*sizeof(float)); + } + } +} + +bool llm_graph_input_hy_v4_flash::can_reuse(const llm_graph_params & params) { + // The memory context is rebuilt for every batch, so the one captured at graph build time is + // dangling by now and set_input would read freed memory. Re-fetch it, exactly as deepseek41 + // does for its own Engram input. + const auto * iswa = model.hparams.indexer_top_k > 0 + ? static_cast(params.mctx)->get_attn() + : static_cast(params.mctx); + mctx = iswa->get_base(); + + bool res = true; + for (auto * t : engram_ids) { + res &= t->ne[0] == (int64_t) (model.hparams.ple_n_heads*params.ubatch.n_tokens); + } + return res; +} + +std::unique_ptr llama_model_hy_v4_flash::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } + return std::make_unique(*this, params); +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_engram( + const llama_model & model, + ggml_tensor * x, + ggml_tensor * rows, + ggml_tensor * valid, + int il) const { + const auto & layer = model.layers[il]; + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + const int64_t n_cols = hparams.ple_n_heads; + const int64_t head_dim = hparams.ple_head_dim; + + GGML_ASSERT(rows->ne[0] == head_dim && rows->ne[1] == n_cols && rows->ne[2] == nt); + + // delta 1: each looked-up row is RMS-normed on its own and scaled by its head's weight. + // The per-column scale by valid is belt and braces: valid is uniform across a token's columns + // (a short history is padded, not invalidated), and the whole-token mask below is what actually + // suppresses an embedding position. + ggml_tensor * r = ggml_rms_norm(ctx0, rows, HY_V4_FLASH_ENGRAM_EPS); + r = ggml_mul(ctx0, r, layer.engram_lookup_norm); + r = ggml_mul(ctx0, r, ggml_reshape_3d(ctx0, valid, 1, n_cols, nt)); + r = ggml_reshape_2d(ctx0, r, head_dim*n_cols, nt); + cb(r, "engram_rows_normed", il); + + // delta 2: the fused key/value projection carries a bias + ggml_tensor * kv = build_lora_mm(layer.engram_kv, r); + kv = ggml_add(ctx0, ggml_reshape_3d(ctx0, kv, n_embd, hc + 1, nt), layer.engram_kv_b); + cb(kv, "engram_kv", il); + + ggml_tensor * key = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, hc, nt, kv->nb[1], kv->nb[2], 0)); + ggml_tensor * value = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, 1, nt, kv->nb[1], kv->nb[2], hc*kv->nb[1])); + + // the gate is deepseek41's: engram_gate folds norm_key.w * norm_query.w + ggml_tensor * egate = ggml_is_quantized(layer.engram_gate->type) ? ggml_cast(ctx0, layer.engram_gate, GGML_TYPE_F32) : layer.engram_gate; + ggml_tensor * dot = ggml_mul(ctx0, ggml_mul(ctx0, ggml_rms_norm(ctx0, x, HY_V4_FLASH_ENGRAM_EPS), ggml_rms_norm(ctx0, key, HY_V4_FLASH_ENGRAM_EPS)), egate); + dot = ggml_scale(ctx0, ggml_sum_rows(ctx0, dot), 1.0f/sqrtf((float) n_embd)); + cb(dot, "engram_dot", il); + + // sigmoid(copysign(sqrt(max(|dot|, 1e-6)), dot)); sign(0) = +1 as in copysign + ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY)); + ggml_tensor * sign = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_neg(ctx0, dot)), -2.0f, 1.0f); + ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, sign)); + cb(gate, "engram_gate", il); + + // delta 3: the delta is the normed value scaled by a learned gamma, not the raw value + ggml_tensor * signal = ggml_mul(ctx0, ggml_rms_norm(ctx0, value, HY_V4_FLASH_ENGRAM_EPS), layer.engram_pre_gamma); + // delta 4: the per-hc offset is additive and per channel, so it broadcasts over tokens only + ggml_tensor * scale = ggml_add(ctx0, ggml_repeat_4d(ctx0, gate, n_embd, hc, nt, 1), layer.engram_hc_scale); + cb(scale, "engram_scale", il); + + ggml_tensor * delta = ggml_mul(ctx0, ggml_repeat_4d(ctx0, signal, n_embd, hc, nt, 1), scale); + + // A token with no valid n-gram contributes nothing. Zeroing its columns is not enough: the + // projection bias would still put a constant through the value path. This is the first token + // of every sequence, and any embedding (image) position. + ggml_tensor * any = ggml_sum_rows(ctx0, ggml_reshape_2d(ctx0, valid, n_cols, nt)); + any = ggml_step(ctx0, any); + delta = ggml_mul(ctx0, delta, ggml_reshape_3d(ctx0, any, 1, 1, nt)); + cb(delta, "engram_delta", il); + + return ggml_add(ctx0, x, delta); +} + +// reduce hc streams x[:,i,:] weighted by w[i,:] -> [n_embd, n_tokens] +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_reduce( + ggml_tensor * x, + ggml_tensor * weights, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(x->ne[1] == hc); + + // the fused node is registered per layer, so the head reduce (il < 0) keeps the loop + if (cparams.fused_dsv4_hc_pre && il >= 0) { + ggml_tensor * result = ggml_dsv4_hc_pre(ctx0, x, weights); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, result, il}); + return result; + } + + ggml_tensor * result = nullptr; + for (int64_t ih = 0; ih < hc; ++ih) { + ggml_tensor * xh = ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih * x->nb[1]); + ggml_tensor * wh = ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih * weights->nb[0]); + ggml_tensor * cur = ggml_mul(ctx0, xh, wh); + result = result ? ggml_add(ctx0, result, cur) : cur; + } + return result; +} + +llama_model_hy_v4_flash::graph::hc_pre +llama_model_hy_v4_flash::graph::build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + GGML_ASSERT(x->ne[0] == n_embd && x->ne[1] == hc); + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_mixes", il); + + ggml_tensor * scale_pre = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 0); + ggml_tensor * scale_post = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 1); + ggml_tensor * base_pre = hy_v4_flash_view_1d(ctx0, hc_base, hc, 0); + ggml_tensor * base_post = hy_v4_flash_view_1d(ctx0, hc_base, hc, hc); + + ggml_tensor * pre = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, 0); + pre = ggml_mul(ctx0, pre, scale_pre); + pre = ggml_add(ctx0, pre, base_pre); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_pre", il); + + ggml_tensor * post = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, hc); + post = ggml_mul(ctx0, post, scale_post); + post = ggml_add(ctx0, post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, hparams.hc_magnitude); + post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps); + cb(post, "hc_post_gate", il); + + return { build_hc_reduce(x, pre, il), post }; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[1]; + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(residual->ne[1] == hc); + + // comb is the identity here: each stream keeps its own residual, hy_v4 has no cross-stream mix + if (cparams.fused_dsv4_hc_post) { + ggml_tensor * result = ggml_dsv4_hc_post(ctx0, x, residual, post, nullptr); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, result, il}); + return result; + } + + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < hc; ++i) { + ggml_tensor * res_i = ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], i * residual->nb[1]); + ggml_tensor * post_i = ggml_view_2d(ctx0, post, 1, nt, post->nb[1], i * post->nb[0]); + ggml_tensor * cur = ggml_add(ctx0, res_i, ggml_mul(ctx0, x, post_i)); + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, nt); + out = out ? ggml_concat(ctx0, out, cur, 1) : cur; + } + + return out; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_head_mixes", -1); + + ggml_tensor * pre = ggml_mul(ctx0, mixes, hc_scale); + pre = ggml_add(ctx0, pre, hc_base); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_head_pre", -1); + + return build_hc_reduce(x, pre, -1); +} + +// The memory composes an iSWA pair rather than being one, so the standard input cannot recover its +// context from params.mctx on its own. +class llama_model_hy_v4_flash::llm_graph_input_attn_idx : public llm_graph_input_attn_k_iswa { +public: + using llm_graph_input_attn_k_iswa::llm_graph_input_attn_k_iswa; + + const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const override { + return static_cast(params.mctx)->get_attn(); + } +}; + +class llama_model_hy_v4_flash::llm_graph_input_kpool : public llm_graph_input_i { +public: + llm_graph_input_kpool(const llama_memory_idx_iswa_context * mctx, uint32_t kpool) : mctx(mctx), kpool(kpool) {} + virtual ~llm_graph_input_kpool() = default; + + void set_input(const llama_ubatch * ubatch) override { + mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch); + mctx->set_input_kpool(pool_cells, pool_idxs, pool_mask, tail_idxs, sel_mask, + new_pool_idxs, new_pool_rep, ubatch, new_pool_pos); + } + + bool can_reuse(const llm_graph_params & params) override { + mctx = static_cast(params.mctx); + + bool res = true; + + res &= k_idxs->ne[0] == params.ubatch.n_tokens; + res &= pool_cells->ne[0] == mctx->get_n_kpool(); + res &= pool_mask->ne[1] == params.ubatch.n_tokens; + res &= tail_idxs->ne[1] == params.ubatch.n_tokens; + res &= n_kv == mctx->get_idx()->get_n_kv(); + res &= n_new == std::max(mctx->get_n_kpool_new(), 1u); + + return res; + } + + ggml_tensor * k_idxs = nullptr; // I64 [n_tokens] + ggml_tensor * pool_cells = nullptr; // I32 [n_pool] cell caching each pool's pooled key + ggml_tensor * pool_idxs = nullptr; // I32 [kpool, n_pool] member cells per pool + ggml_tensor * pool_mask = nullptr; // F32/F16 [n_pool, n_tokens] + ggml_tensor * tail_idxs = nullptr; // I32 [kpool, n_tokens] when the current group is forced + ggml_tensor * sel_mask = nullptr; // F32 [n_sel, 1, 1, n_tokens] + ggml_tensor * new_pool_idxs = nullptr; // I32 [kpool*n_new] + ggml_tensor * new_pool_rep = nullptr; // I64 [n_new] + ggml_tensor * new_pool_pos = nullptr; // I32 [4*n_new] first member's position per new pool + + const llama_memory_idx_iswa_context * mctx; + const uint32_t kpool; + uint32_t n_new = 0; + uint32_t n_sel = 0; + uint32_t n_kv = 0; +}; + +llama_model_hy_v4_flash::llm_graph_input_kpool * llama_model_hy_v4_flash::graph::build_inp_kpool( + const llama_memory_idx_iswa_context * mctx_idx) { + const auto * mctx_lid = mctx_idx->get_idx(); + + const uint32_t kpool = hparams.indexer_kpool; + const uint32_t n_pool = mctx_idx->get_n_kpool(); + // a ubatch that completes no pool still builds one dummy entry, so the graph does not change + // shape every kpool tokens + const uint32_t n_new = std::max(mctx_idx->get_n_kpool_new(), 1u); + + const auto type_mask = cparams.fused_lid ? GGML_TYPE_F16 : GGML_TYPE_F32; + + auto inp = std::make_unique(mctx_idx, kpool); + + inp->k_idxs = mctx_lid->build_input_k_idxs(ctx0, ubatch); + inp->pool_cells = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pool); + inp->pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_pool); + inp->pool_mask = ggml_new_tensor_2d(ctx0, type_mask, n_pool, n_tokens); + inp->tail_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, hparams.indexer_kpool_select_tail ? kpool : kpool - 1, n_tokens); + ggml_set_input(inp->pool_cells); + ggml_set_input(inp->pool_idxs); + ggml_set_input(inp->pool_mask); + ggml_set_input(inp->tail_idxs); + + ggml_build_forward_expand(gf, inp->pool_cells); + ggml_build_forward_expand(gf, inp->pool_idxs); + ggml_build_forward_expand(gf, inp->pool_mask); + ggml_build_forward_expand(gf, inp->tail_idxs); + + inp->n_kv = mctx_lid->get_n_kv(); + + { + // The reference keeps group_topk - 1 scored groups and forces the one the current token + // sits in, which the tail covers even when that group is complete. + const int64_t n_top_pool = std::min(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail); + const int64_t n_sel = kpool*n_top_pool + (hparams.indexer_kpool_select_tail ? kpool : 0); + inp->n_sel = (uint32_t) n_sel; + + if (!cparams.fused_top_k_mask) { + inp->sel_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_sel, 1, 1, n_tokens); + ggml_set_input(inp->sel_mask); + ggml_build_forward_expand(gf, inp->sel_mask); + } + } + + inp->n_new = n_new; + + inp->new_pool_idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kpool*n_new); + ggml_set_input(inp->new_pool_idxs); + inp->new_pool_rep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_new); + ggml_set_input(inp->new_pool_rep); + // 4 planes so the vision case can rotate pooled keys with mrope; text uses plane 0 + inp->new_pool_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 4*n_new); + ggml_set_input(inp->new_pool_pos); + ggml_build_forward_expand(gf, inp->new_pool_pos); + + return (llm_graph_input_kpool *) res->add_input(std::move(inp)); +} + +// rotate only the trailing n_rot of each indexer head, leaving the head's front half untouched +ggml_tensor * llama_model_hy_v4_flash::graph::build_indexer_rope( + ggml_tensor * x, ggml_tensor * pos, int64_t n_idx, float freq_base, int il) const { + const int64_t n_nope = n_idx - n_rot; + GGML_ASSERT(n_nope > 0); + + ggml_tensor * front = ggml_view_3d(ctx0, x, n_nope, x->ne[1], x->ne[2], + x->nb[1], x->nb[2], 0); + ggml_tensor * back = ggml_view_3d(ctx0, x, n_rot, x->ne[1], x->ne[2], + x->nb[1], x->nb[2], ggml_row_size(x->type, n_nope)); + + // the same base the layer's own attention uses, so a cparams override reaches the indexer too + back = ggml_rope_ext(ctx0, ggml_cont(ctx0, back), pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + cb(back, "indexer_rope", il); + + return ggml_concat(ctx0, ggml_cont(ctx0, front), back, 0); +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_kpool_select( + ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * inp_pos, float freq_base, + const llama_layer & layer, + const llama_memory_idx_iswa_context * mctx_idx, llm_graph_input_kpool * inp_kpool, int il) { + const auto * mctx_lid = mctx_idx->get_idx(); + + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t n_idx = hparams.indexer_head_size; + const int64_t kpool = hparams.indexer_kpool; + const int64_t n_pool = inp_kpool->pool_cells->ne[0]; + const int64_t n_new = inp_kpool->n_new; + const int64_t n_kv = mctx_lid->get_n_kv(); + + // the query comes off the same lora residual the main attention uses, and is rotated at the + // token's own position + ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr); + iq = ggml_reshape_3d(ctx0, iq, n_idx, n_idx_head, n_tokens); + iq = build_indexer_rope(iq, inp_pos, n_idx, freq_base, il); + cb(iq, "indexer_q", il); + + // key side: weight-only RMSNorm, unlike glm5-next's LayerNorm with a bias this model lacks + ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur); + ik = build_norm(ik, layer.indexer_k_norm, nullptr, LLM_NORM_RMS, il); + cb(ik, "indexer_k", il); + + // what the pool averages is a projection of the normed key, and the weights come from another + // projection of it: glm5-next pools the key itself and gates off the hidden state + ggml_tensor * iv = ggml_mul_mat(ctx0, layer.indexer_kpool_value, ik); + ggml_tensor * ig = ggml_mul_mat(ctx0, layer.indexer_kpool_gate, ik); + cb(ig, "indexer_gate", il); + + // cache rows hold value | gate | pooled + ggml_tensor * pzero = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_idx, n_tokens), 0.0f); + ggml_tensor * packed = ggml_concat(ctx0, ggml_concat(ctx0, iv, ig, 0), pzero, 0); + packed = ggml_reshape_3d(ctx0, packed, 3*n_idx, 1, n_tokens); + ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, packed, inp_kpool->k_idxs, il)); + + auto kpool_cache = mctx_idx->get_kpool_access(ctx0, il, n_idx); + + const ggml_type type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32; + + // pool the groups this ubatch completed; the last entry is a dummy when it completed none + ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_idxs); + rows = ggml_reshape_3d(ctx0, rows, 2*n_idx, kpool, n_new); + + ggml_tensor * pk = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2], 0); + ggml_tensor * pg = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2], + ggml_row_size(rows->type, n_idx)); + + ggml_tensor * logits = ggml_add(ctx0, pg, layer.indexer_kpool_ape); + logits = ggml_cont(ctx0, ggml_permute(ctx0, logits, 1, 0, 2, 3)); // [kpool, n_idx, n_new] + // fold the pool axis into rows: soft_max maps ne[2] to a CUDA gridDim.y that caps at 65535 + ggml_tensor * probs = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, logits, kpool, n_idx*n_new)); + probs = ggml_reshape_3d(ctx0, probs, kpool, n_idx, n_new); + + pk = ggml_cont(ctx0, ggml_permute(ctx0, pk, 1, 0, 2, 3)); + ggml_tensor * pooled_new = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, pk)); // [1, n_idx, n_new] + pooled_new = ggml_reshape_2d(ctx0, pooled_new, n_idx, n_new); + + // the pooled key is normed and then rotated at its group's first position, so the score keeps + // the same relative geometry as the main attention + pooled_new = build_norm(pooled_new, layer.indexer_kpool_norm, nullptr, LLM_NORM_RMS, il); + { + ggml_tensor * pos = ggml_view_1d(ctx0, inp_kpool->new_pool_pos, n_new, 0); + ggml_tensor * p3 = ggml_reshape_3d(ctx0, pooled_new, n_idx, 1, n_new); + pooled_new = ggml_reshape_2d(ctx0, build_indexer_rope(p3, pos, n_idx, freq_base, il), n_idx, n_new); + } + cb(pooled_new, "indexer_pool_k_new", il); + + // write the fresh pooled keys into their rep rows, then read every pool back by cell: the + // older ones come from the rows earlier ubatches wrote + ggml_build_forward_expand(gf, kpool_cache.scatter_pooled(pooled_new, inp_kpool->new_pool_rep)); + ggml_tensor * pooled = kpool_cache.gather_pooled(inp_kpool->pool_cells); + pooled = ggml_reshape_3d(ctx0, pooled, n_idx, 1, n_pool); + cb(pooled, "indexer_pool_k", il); + + ggml_tensor * weights = ggml_mul_mat(ctx0, layer.indexer_proj, cur); + weights = ggml_scale(ctx0, weights, 1.0f / sqrtf(float(n_idx * n_idx_head))); + cb(weights, "indexer_weights", il); + + ggml_tensor * score = nullptr; + if (cparams.fused_lid) { + score = ggml_lightning_indexer(ctx0, iq, pooled, weights, inp_kpool->pool_mask); + res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); + } else { + ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [n_idx, n_tokens, n_head] + ggml_tensor * k_p = ggml_permute(ctx0, pooled, 0, 2, 1, 3); // [n_idx, n_pool, 1] + + ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_pool, n_tokens, n_head] + kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_head, n_tokens, n_pool] + score = ggml_relu(ctx0, kq); + score = ggml_mul(ctx0, score, weights); + score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_pool] + score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_pool, n_tokens, 1] + score = ggml_add(ctx0, score, inp_kpool->pool_mask); + } + cb(score, "indexer_score", il); + + const int64_t n_top_pool = std::min(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail); + ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered + + // keep the cells of the picked pools and of the current group, -inf elsewhere; every + // retained cell is in the token's own sequence, so the kq_mask is not needed on top + ggml_tensor * tail = hparams.indexer_kpool_select_tail ? inp_kpool->tail_idxs : nullptr; + GGML_ASSERT(kpool*n_top_pool + (tail ? tail->ne[0] : 0) == (int64_t) inp_kpool->n_sel); + + if (cparams.fused_top_k_mask) { + ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, type_mask, n_kv), 0.0f); + ggml_tensor * sel = ggml_top_k_mask(ctx0, keep, top_k, score, inp_kpool->pool_idxs, tail); + res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, sel, il}); + cb(sel, "indexer_sel", il); + + return sel; + } + + // Backends without a fused top-k mask scatter the same selection by hand. Everything this arch + // changes about the indexer is upstream of score, so from here on it is glm5-next's path. + ggml_tensor * sel_idx = nullptr; + { + // sel_mask marks the first min(nv, n_top_pool) slots as the live pools, so put the picked + // set in descending score order to match it (ggml_top_k leaves it unordered) + ggml_tensor * sel_score = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, score, 1, n_pool, n_tokens), top_k); // [1, n_top_pool, n_tokens] + ggml_tensor * sel_order = ggml_argsort(ctx0, + ggml_reshape_2d(ctx0, sel_score, n_top_pool, n_tokens), GGML_SORT_ORDER_DESC); + + ggml_tensor * ordered = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, ggml_cast(ctx0, top_k, GGML_TYPE_F32), 1, n_top_pool, n_tokens), sel_order); + ordered = ggml_cast(ctx0, + ggml_cont(ctx0, ggml_reshape_2d(ctx0, ordered, n_top_pool, n_tokens)), GGML_TYPE_I32); + cb(ordered, "indexer_top_k", il); + + sel_idx = ggml_get_rows(ctx0, inp_kpool->pool_idxs, + ggml_reshape_1d(ctx0, ordered, n_top_pool*n_tokens)); // [kpool, n_top_pool*n_tokens] + sel_idx = ggml_reshape_2d(ctx0, sel_idx, kpool*n_top_pool, n_tokens); + + if (tail) { + // the tail, with n_kv standing in for the cells it does not have yet + sel_idx = ggml_concat(ctx0, sel_idx, tail, 0); + } + } + + const int64_t n_sel = sel_idx->ne[0]; + GGML_ASSERT(n_sel == (int64_t) inp_kpool->n_sel); + GGML_ASSERT(inp_kpool->sel_mask && inp_kpool->sel_mask->ne[0] == n_sel && + inp_kpool->sel_mask->ne[3] == n_tokens); + + ggml_build_forward_expand(gf, sel_idx); + + ggml_tensor * mask_all = ggml_new_tensor_4d(ctx0, type_mask, n_kv + n_sel, 1, 1, 1); + mask_all = ggml_fill(ctx0, mask_all, -INFINITY); + mask_all = ggml_repeat_4d(ctx0, mask_all, n_kv + n_sel, n_tokens, 1, 1); + mask_all = ggml_reshape_3d(ctx0, mask_all, 1, n_kv + n_sel, n_tokens); + + ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, type_mask, n_sel, 1, 1, 1); + zeros = ggml_fill(ctx0, zeros, 0.0f); + zeros = ggml_repeat_4d(ctx0, zeros, n_sel, n_tokens, 1, 1); + zeros = ggml_reshape_3d(ctx0, zeros, 1, n_sel, n_tokens); + + // Live slots (visible pools, real tail cells) address disjoint cells, and each dead slot writes + // its own dump row n_kv + slot, so a token's scatter indices stay unique: + // idx = dump + live*(idx - dump), live = exp(mask). + ggml_tensor * live = ggml_reshape_2d(ctx0, ggml_exp(ctx0, inp_kpool->sel_mask), n_sel, n_tokens); + ggml_tensor * dump = ggml_arange(ctx0, (float) n_kv, (float) (n_kv + n_sel), 1.0f); + ggml_tensor * idx_f = ggml_cast(ctx0, sel_idx, GGML_TYPE_F32); + idx_f = ggml_add(ctx0, ggml_mul(ctx0, ggml_sub(ctx0, idx_f, dump), live), dump); + sel_idx = ggml_cast(ctx0, idx_f, GGML_TYPE_I32); + + ggml_tensor * sel = ggml_set_rows(ctx0, mask_all, zeros, + ggml_reshape_3d(ctx0, sel_idx, n_sel, n_tokens, 1)); + + sel = ggml_view_2d(ctx0, sel, n_kv, n_tokens, sel->nb[2], 0); + sel = ggml_cont(ctx0, sel); + cb(sel, "indexer_sel", il); + + return sel; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_attn, + ggml_tensor * cur, + ggml_tensor * inp_pos, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il) { + const auto & layer = model.layers[il]; + + const int64_t n_head_l = hparams.n_head(il); + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k)); + const float freq_base_l = model.get_rope_freq_base(cparams, il); + + ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur); + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "q_resid", il); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr); + + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens, + ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l, 0); + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens, + ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l, + ggml_row_size(q->type, n_embd_head_qk_nope)); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + + // rope goes last so build_rope_shift() can shift the cache in place + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + ggml_tensor * Vcur = kv_cmpr; + + ggml_tensor * sel = nullptr; + // the draft block is a sparse layer and always owns an indexer, and is_indexer_full() only + // covers the trunk + if (inp_kpool != nullptr && + (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il))) { + sel = build_kpool_select(cur, qr, inp_pos, freq_base_l, layer, mctx_idx, inp_kpool, il); + } + + // wo is applied below so the gate can sit in front of it + ggml_tensor * attn; + if (sel != nullptr) { + const auto * mctx_base = mctx_idx->get_base(); + const int64_t n_sel = inp_kpool->n_sel; + // The indexer's selection replaces the kq_mask, so this layer cannot go through build_attn: + // write the latent row ourselves, then read the whole cache back and let the mask decide. + ggml_tensor * k = mctx_base->get_k(ctx0, il); + ggml_tensor * v = ggml_view_4d(ctx0, k, kv_lora_rank, k->ne[1], k->ne[2], k->ne[3], + k->nb[1], k->nb[2], k->nb[3], 0); + + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Kcur); + ggml_build_forward_expand(gf, mctx_base->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il)); + + // split the selection into the streams the K view carries + const int64_t n_stream = k->ne[3]; + GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0); + sel = ggml_reshape_4d(ctx0, sel, sel->ne[0], sel->ne[1]/n_stream, 1, n_stream); + + attn = build_attn_mha(Qcur, k, v, nullptr, sel, nullptr, layer.attn_sinks, layer.wv_b, + n_sel, kq_scale, il); + } else { + attn = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, kq_scale, il); + } + cb(attn, "attn_kqv", il); + + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur); + gate = ggml_sigmoid(ctx0, gate); + attn = ggml_mul(ctx0, attn, gate); + cb(attn, "attn_gated", il); + + ggml_tensor * out = build_lora_mm(layer.wo, attn); + out = build_norm(out, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(out, "attn_out", il); + + return out; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_latent_moe( + const llama_model & model, + ggml_tensor * cur, + int il) const { + const auto & layer = model.layers[il]; + + // the router scores the full-width input, the experts take the latent one + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, cur); + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * routed_in = layer.ffn_routed_down + ? ggml_mul_mat(ctx0, layer.ffn_routed_down, cur) + : cur; + + ggml_tensor * moe_out = build_moe_ffn(routed_in, + nullptr, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + logits, + nullptr, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + if (layer.ffn_routed_up) { + moe_out = ggml_mul_mat(ctx0, layer.ffn_routed_up, moe_out); + cb(moe_out, "ffn_moe_up", il); + } + + ggml_tensor * shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(shexp, "ffn_shexp", il); + + return ggml_add(ctx0, moe_out, shexp); +} + +llama_model_hy_v4_flash::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t hc = hparams.dsv4_hc_mult; + + ggml_tensor * cur; + + ggml_tensor * inp = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + // With an indexer the memory is a composition, so the attention context has to be fetched out + // of it and the input told how to find it again on reuse. + const auto * mctx_idx = hparams.indexer_top_k > 0 + ? static_cast(mctx) : nullptr; + + llm_graph_input_attn_k_iswa * inp_attn = mctx_idx + ? build_attn_inp_k_iswa( + std::make_unique(hparams, cparams, mctx_idx->get_attn()), + mctx_idx->get_attn()) + : build_attn_inp_k_iswa(); + + llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr; + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // Engram gathers go right after the token embeddings so they share the first graph split and + // the lazily read table stays on the host. An embedding-only ubatch has no token ids to hash, + // which matches the reference skipping Engram at its masked (image) positions. + auto inp_eng = std::make_unique( + static_cast(model), inp_attn->mctx->get_base()); + std::vector engram_rows; + std::vector engram_valid; + for (int il = 0; ubatch.token && hparams.ple_n_heads > 0 && il < n_layer; ++il) { + if (!hparams.is_ple(il)) { + continue; + } + const int64_t n_cols = hparams.ple_n_heads; + + ggml_tensor * ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cols*n_tokens); + ggml_set_input(ids); + ggml_set_name(ids, ("engram_ids-" + std::to_string(il)).c_str()); + inp_eng->engram_ids.push_back(ids); + + ggml_tensor * valid = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cols*n_tokens); + ggml_set_input(valid); + ggml_set_name(valid, ("engram_valid-" + std::to_string(il)).c_str()); + inp_eng->engram_valid.push_back(valid); + engram_valid.push_back(valid); + + ggml_tensor * rows = ggml_get_rows(ctx0, model.per_layer_tok_embd, ids); + rows = ggml_reshape_3d(ctx0, rows, hparams.ple_head_dim, n_cols, n_tokens); + cb(rows, "engram_rows", il); + ggml_build_forward_expand(gf, rows); + engram_rows.push_back(rows); + } + res->add_input(std::move(inp_eng)); + + // one embedding feeds hc parallel residual streams + ggml_tensor * inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + // Engram writes into the streams before the attention pre-mix reads them + const int e = static_cast(model).engram_index(il); + if (e >= 0 && !engram_rows.empty()) { + inpL = build_engram(model, inpL, engram_rows[e], engram_valid[e], il); + cb(inpL, "engram_out", il); + } + + ggml_tensor * residual = inpL; + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); + cur = build_norm(attn_hc.cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il); + + inpL = build_hc_post(cur, residual, attn_hc.post, il); + cb(inpL, "hc_attn_out", il); + + residual = inpL; + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + cur = build_norm(ffn_hc.cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + layer.ffn_up, NULL, NULL, + layer.ffn_gate, NULL, NULL, + layer.ffn_down, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + } else { + cur = build_latent_moe(model, cur, il); + } + + cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, ffn_hc.post, il); + cb(inpL, "l_out", il); + } + + // prune to the requested rows once, after every stream is done. Unmasked nextn embeddings want + // one row per token, so in that mode the narrowing waits until the state has been captured. + if (crop_before_nextn(inp_out_ids)) { + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd * hc, n_tokens); + flat = ggml_get_rows(ctx0, flat, inp_out_ids); + inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs); + } + + cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); + cb(cur, "hc_head", -1); + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + + // the draft block consumes this exact state through its hnorm, so expose it for the caller to + // hand back via llama_batch_ext_set_embd_state() + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (crop_after_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; +} + +llama_model_hy_v4_flash::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + graph(params, no_build{}) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "hy_v4_flash MTP needs the draft block, convert without --no-mtp"); + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && + "hy_v4_flash MTP block is missing"); + + const auto * mctx_idx = hparams.indexer_top_k > 0 + ? static_cast(mctx) : nullptr; + + llm_graph_input_attn_k_iswa * inp_attn = mctx_idx + ? build_attn_inp_k_iswa( + std::make_unique(hparams, cparams, mctx_idx->get_attn()), + mctx_idx->get_attn()) + : build_attn_inp_k_iswa(); + + llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr; + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // the draft takes the next token's embedding alongside the trunk's final hidden state, which + // is the same tensor the trunk's own output head reads: already collapsed over the hc streams + // and already passed through output_norm + auto inp = std::make_unique(hparams.n_embd_inp(), n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * tok_embd = ubatch.token + ? ggml_get_rows(ctx0, model.tok_embd, inp->tokens) + : inp->embd; + ggml_tensor * prev_h = inp->h; + res->add_input(std::move(inp)); + + ggml_tensor * cur = ggml_concat(ctx0, + build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il), + build_norm(prev_h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il), 0); + cur = ggml_mul_mat(ctx0, layer.nextn.eh_proj, cur); + cb(cur, "mtp_eh_proj", il); + + // from here it is an ordinary sparse layer, except that the residual is the plain hidden state + // rather than one of the trunk's hc streams + ggml_tensor * residual = cur; + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il); + cur = ggml_add(ctx0, cur, residual); + + if (crop_before_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + residual = cur; + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cur = build_latent_moe(model, cur, il); + cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il); + cur = ggml_add(ctx0, cur, residual); + + cur = build_norm(cur, layer.nextn.shared_head_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_result_norm", -1); + + // the draft's own post-norm state, so a second draft step can be fed from this one + res->t_h_nextn = cur; + + if (crop_after_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_lora_mm(model.output, cur); + cb(cur, "mtp_result_output", -1); + + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 1fc7cae1dff0c2e7ffea4e75bd7e94bd699cdf3c..f8cdc66fcf478825a46bc8daa6d12f31da045ea5 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -9,6 +9,7 @@ #include class llama_memory_hybrid_idx_context; +class llama_memory_idx_iswa_context; // ref: https://github.com/ggml-org/llama.cpp/pull/28068 static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) { @@ -2370,6 +2371,122 @@ struct llama_model_hy_v4 : public llama_model_base { }; +// HY_V4_FLASH: hy_v4's iHC + gated MLA + sinks, with a hybrid attention stack (sparse full +// layers alternating with sliding-window layers, each with its own head count and rope base), +// sandwich norms and routed experts that run in a latent space. +struct llama_model_hy_v4_flash : public llama_model_base { + llama_model_hy_v4_flash(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + // -1 when the layer has no Engram module, else its index among the Engram layers + int engram_index(int il) const; + + // n-gram hash of module e. Writes one row id per (token, column) into ids, and alongside it the + // per-column validity: a short history is padded, as in deepseek41, so only a position with no + // n-gram at all (an embedding position) is marked invalid. + void engram_hash(int e, const llama_ubatch & ubatch, const std::vector & prev, + std::vector & ids, std::vector & valid) const; + + class llm_graph_input_kpool; + + // the memory only composes an iSWA pair, so the standard input cannot find its context in + // params.mctx by itself + class llm_graph_input_attn_idx; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + // the MTP graph reuses the helpers without running the trunk + struct no_build {}; + graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {} + + // iHC, same layout as hy_v4: pre reduces the hc streams to one and returns the per-stream + // post gates, post writes the sublayer output back, head collapses before the final norm. + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + }; + + // reduce the hc streams to one, weighted by the pre gates + ggml_tensor * build_hc_reduce( + ggml_tensor * x, + ggml_tensor * weights, + int il) const; + + hc_pre build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + int il) const; + + ggml_tensor * build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + int il) const; + + ggml_tensor * build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const; + + // mctx_idx and inp_kpool are null when the file carries no indexer, and the sparse layers + // then attend their whole context + ggml_tensor * build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_attn, + ggml_tensor * cur, + ggml_tensor * inp_pos, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il); + + llm_graph_input_kpool * build_inp_kpool(const llama_memory_idx_iswa_context * mctx_idx); + + // rotate only the trailing n_rot of each indexer head + ggml_tensor * build_indexer_rope(ggml_tensor * x, ggml_tensor * pos, int64_t n_idx, + float freq_base, int il) const; + + // score the pooled keys against the query and keep the best groups, plus the group the + // current token sits in + ggml_tensor * build_kpool_select( + ggml_tensor * cur, + ggml_tensor * qr, + ggml_tensor * inp_pos, + float freq_base, + const llama_layer & layer, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il); + + // routed experts run at n_expert_latent, the router and the shared expert at n_embd + ggml_tensor * build_latent_moe( + const llama_model & model, + ggml_tensor * cur, + int il) const; + + // Engram writes its delta into the hc residual streams before the layer's attention pre-mix + ggml_tensor * build_engram( + const llama_model & model, + ggml_tensor * x, + ggml_tensor * rows, + ggml_tensor * valid, + int il) const; + }; + + // the draft block: one sparse layer fed the next token's embedding and the trunk's final + // hidden state, with no hyper-connections of its own + struct graph_mtp : public graph { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_hunyuan_vl : public llama_model_base { llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index eec0423cbef583b689a916a00fe49d3d67bb868a..367ab540be35e13191d3b2c949f93cd69f020d7b 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4535,6 +4535,62 @@ struct test_snake_fuse : public test_case { }; +struct test_dsv4_hc_gate : public test_case { + const int64_t n_tokens; + const bool post; + const bool live_mul; + const bool view_bias; + + test_dsv4_hc_gate(int64_t n_tokens, bool post, bool live_mul = false, bool view_bias = false) + : n_tokens(n_tokens), post(post), live_mul(live_mul), view_bias(view_bias) {} + + std::string vars() override { return VARS_TO_STR4(n_tokens, post, live_mul, view_bias); } + std::string op_desc(ggml_tensor *) override { return "DSV4_HC_GATE"; } + bool run_whole_graph() override { return true; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * mixes = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 8, n_tokens); + ggml_tensor * factor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, view_bias ? 8 : 4); + if (view_bias) { + bias = ggml_view_1d(ctx, bias, 4, post ? 4*sizeof(float) : 0); + } + ggml_tensor * x = ggml_view_2d(ctx, mixes, 4, n_tokens, mixes->nb[1], post ? 4*sizeof(float) : 0); + ggml_tensor * mul = ggml_mul(ctx, x, factor); + ggml_tensor * out = ggml_add(ctx, mul, bias); + out = ggml_sigmoid(ctx, out); + if (post) { + out = ggml_scale(ctx, out, 2.0f); + } + out = ggml_scale_bias(ctx, out, 1.0f, 1e-6f); + return live_mul ? ggml_concat(ctx, out, mul, 0) : out; + } +}; + +struct test_dsv4_hc_gate_consumer : public test_dsv4_hc_gate { + const int64_t width; + const bool live_gate; + + test_dsv4_hc_gate_consumer(int64_t width, int64_t nt, bool post, bool live_gate) + : test_dsv4_hc_gate(nt, post, false, true), width(width), live_gate(live_gate) {} + + std::string vars() override { return VARS_TO_STR5(width, n_tokens, post, live_gate, view_bias); } + std::string op_desc(ggml_tensor *) override { return "DSV4_HC_GATE_CONSUMER"; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * gate = test_dsv4_hc_gate::build_graph(ctx); + ggml_tensor * residual = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, width, 4, n_tokens); + ggml_tensor * out; + if (post) { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, n_tokens); + out = ggml_dsv4_hc_post(ctx, x, residual, gate, nullptr); + } else { + out = ggml_dsv4_hc_pre(ctx, residual, gate); + } + return live_gate ? ggml_add(ctx, out, ggml_sum(ctx, gate)) : out; + } +}; + struct test_dsv4_hc : public test_case { static constexpr int64_t hc = 4; @@ -7808,6 +7864,48 @@ struct test_mul_mat_add : public test_case { } }; +struct test_mmvq_q8_reuse : public test_case { + const ggml_type type; + const int64_t k; + const int64_t n; + const int mode; + + test_mmvq_q8_reuse(ggml_type type, int64_t k, int mode, int64_t n = 1) + : type(type), k(k), n(n), mode(mode) {} + + std::string vars() override { return VARS_TO_STR4(type, k, n, mode); } + std::string op_desc(ggml_tensor *) override { return "MMVQ_Q8_REUSE"; } + bool run_whole_graph() override { return true; } + bool use_weight_context() override { return true; } + double max_nmse_err() override { return 5e-4; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * weights) override { + ggml_tensor * w0 = ggml_new_tensor_2d(weights, type, k, 128); + ggml_tensor * w1 = ggml_new_tensor_2d(weights, GGML_TYPE_Q8_0, k, 128); + ggml_tensor * w2 = ggml_new_tensor_2d(weights, type, k, 128); + ggml_tensor * base = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k + 32, n); + ggml_tensor * x = ggml_view_2d(ctx, base, k, n, base->nb[1], 32*sizeof(float)); + ggml_tensor * a = ggml_mul_mat(ctx, w0, x); + ggml_tensor * middle; + if (mode == 1) { + // A write through another view must prevent reuse of the old quantization. + ggml_tensor * alias = ggml_view_tensor(ctx, x); + middle = ggml_mul_mat(ctx, w1, ggml_scale_inplace(ctx, alias, 0.5f)); + } else if (mode == 2) { + ggml_tensor * other = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + middle = ggml_mul_mat(ctx, w1, other); + } else { + middle = ggml_mul_mat(ctx, w1, x); + } + ggml_tensor * b = ggml_mul_mat(ctx, w2, mode == 3 ? ggml_view_tensor(ctx, x) : x); + return ggml_concat(ctx, ggml_concat(ctx, a, middle, 0), b, 0); + } +}; + struct test_mul_mat_vec_fusion : public test_case { const ggml_type type; const ggml_glu_op glu_op; @@ -7995,6 +8093,63 @@ struct test_mul_mat_vec_fusion : public test_case { } }; + +struct test_spark_decode_layout : public test_mul_mat_vec_fusion { + test_spark_decode_layout(int64_t rows, int64_t tokens, bool broadcast, bool scales, bool gate) + : test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU_CLAMP, tokens, rows, 1536, + true, 32, 16, broadcast, false, gate, scales) {} + + std::string op_desc(ggml_tensor *) override { return "SPARK_DECODE_LAYOUT"; } + bool use_weight_context() override { return true; } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * weights) override { + ggml_tensor * up_w = ggml_new_tensor_3d(weights, type, k, n, n_mats); + ggml_tensor * gate_w = ggml_new_tensor_3d(weights, type, k, n, n_mats); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, m); + ids = ggml_view_2d(ctx, ids, n_used, m, ids->nb[1], 0); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, b ? 1 : n_used, m); + ggml_tensor * up = ggml_mul_mat_id(ctx, up_w, input, ids); + ggml_tensor * gate = with_gate ? ggml_mul_mat_id(ctx, gate_w, input, ids) : nullptr; + if (with_lane_scale) { + up = build_lane_scale_id(ctx, weights, up, ids); + if (gate) { + gate = build_lane_scale_id(ctx, weights, gate, ids); + } + } + return gate ? build_gate(ctx, gate, up) : up; + } +}; + +struct test_spark_mmq_fusion : public test_mul_mat_vec_fusion { + bool skewed = false; + test_spark_mmq_fusion(int64_t tokens, int64_t rows, int64_t k, bool broadcast, ggml_glu_op op, bool bias = false, bool scale = true) + : test_mul_mat_vec_fusion(GGML_TYPE_NVFP4, op, tokens, rows, k, true, 32, 4, broadcast, bias, true, scale) {} + + std::string op_desc(ggml_tensor *) override { return "SPARK_MMQ_FUSION"; } + + void initialize_tensors(ggml_context * ctx) override { + test_mul_mat_vec_fusion::initialize_tensors(ctx); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) { + if (skewed && t->type == GGML_TYPE_I32 && !t->view_src) { + std::vector ids(ggml_nelements(t)); + for (size_t i = 0; i < ids.size(); ++i) { + ids[i] = i % n_mats; + } + ggml_backend_tensor_set(t, ids.data(), 0, ids.size()*sizeof(int32_t)); + } + if (strcmp(t->name, "cur") != 0) { + continue; + } + const float values[] = {-6, -4, -3, -2, -1.5f, -1, -0.5f, 0, 0.5f, 1, 1.5f, 2, 3, 4, 6, 0}; + std::vector data(ggml_nelements(t)); + for (size_t i = 0; i < data.size(); ++i) { + data[i] = values[i % 16] * (1 << ((i / t->ne[0]) % 3)); + } + ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(float)); + } + } +}; + // GGML_OP_SUM struct test_sum : public test_case { const ggml_type type; @@ -10204,6 +10359,43 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_snake_fuse(type, { 64, 32, 2, 3})); // ne[2] > 1 and ne[3] > 1 } + for (ggml_type type : { GGML_TYPE_Q8_0, GGML_TYPE_NVFP4 }) { + for (int mode : { 0, 1, 2, 3 }) { + test_cases.emplace_back(new test_mmvq_q8_reuse(type, 1536, mode)); + test_cases.emplace_back(new test_mmvq_q8_reuse(type, 3072, mode)); + } + } + test_cases.emplace_back(new test_mmvq_q8_reuse(GGML_TYPE_Q8_0, 512, 0, 3)); + + for (int64_t rows : { 512, 1536 }) { + for (int64_t tokens : { 1, 3 }) { + for (bool broadcast : { false, true }) { + for (bool scales : { false, true }) { + for (bool gate : { false, true }) { + test_cases.emplace_back(new test_spark_decode_layout(rows, tokens, broadcast, scales, gate)); + } + } + } + } + } + + for (int64_t nt : { 1, 17, 512 }) { + for (bool post : { false, true }) { + test_cases.emplace_back(new test_dsv4_hc_gate(nt, post)); + test_cases.emplace_back(new test_dsv4_hc_gate(nt, post, false, true)); + } + } + test_cases.emplace_back(new test_dsv4_hc_gate(17, false, true, true)); + test_cases.emplace_back(new test_dsv4_hc_gate(17, true, true, true)); + + for (int64_t width : { 1, 257, 3072 }) { + for (bool post : { false, true }) { + test_cases.emplace_back(new test_dsv4_hc_gate_consumer(width, 1, post, false)); + test_cases.emplace_back(new test_dsv4_hc_gate_consumer(width, 1, post, true)); + test_cases.emplace_back(new test_dsv4_hc_gate_consumer(width, 17, post, false)); + } + } + test_cases.emplace_back(new test_dsv4_hc_comb(1, 1)); test_cases.emplace_back(new test_dsv4_hc_comb(17, 4)); test_cases.emplace_back(new test_dsv4_hc_comb(257, 8)); @@ -10850,6 +11042,10 @@ static std::vector> make_test_cases_eval() { // quant block count not a multiple of the kernel block size test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {96, 1, 1, 1})); test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {96, 1, 1, 1})); + for (int64_t width : {128, 132, 129}) { + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {width, 67, 5, 3}, + {width*67, 15, 1, 1}, {0, 2, 3, 1})); + } test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4})); @@ -12584,6 +12780,21 @@ static std::vector> make_test_cases_eval() { } // Fused gate/up + GLU on the MMQ path, only used if its tiling is efficient: ~1280 tiles per case, one case per type and GLU op. + for (const bool broadcast : {false, true}) { + for (const ggml_glu_op op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_SWIGLU_CLAMP}) { + test_cases.emplace_back(new test_spark_mmq_fusion(33, 128, 512, broadcast, op)); + test_cases.emplace_back(new test_spark_mmq_fusion(129, 256, 1536, broadcast, op)); + test_cases.emplace_back(new test_spark_mmq_fusion(129, 256, 1536, broadcast, op, false, false)); + } + } + for (bool broadcast : {false, true}) { + auto * skewed = new test_spark_mmq_fusion(385, 128, 512, broadcast, GGML_GLU_OP_SWIGLU_CLAMP); + skewed->skewed = true; + test_cases.emplace_back(skewed); + } + test_cases.emplace_back(new test_spark_mmq_fusion(33, 192, 512, true, GGML_GLU_OP_SWIGLU_CLAMP)); + test_cases.emplace_back(new test_spark_mmq_fusion(33, 128, 512, true, GGML_GLU_OP_SWIGLU_CLAMP, true)); + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 17, 81920, 256, false, 1, 1, false, false, true, false, { 1, 1 })); test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q5_K, GGML_GLU_OP_GEGLU, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU_OAI, 150, 40960, 256, false, 1, 1, false, false, true, false, { 1, 1 })); @@ -12706,7 +12917,7 @@ static std::vector> make_test_cases_eval() { // lightning_indexer for (int kv : { 256 }) { for (int bs : { 1, 512 }) { - for (int nh : { 4, 32, 64 }) { + for (int nh : { 4, 16, 32, 64 }) { for (auto [ns, nm] : { std::pair{1, 1}, std::pair{4, 4}, std::pair{4, 1} }) { for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, nm, type_K)); @@ -12716,6 +12927,12 @@ static std::vector> make_test_cases_eval() { } } + for (int kv : { 1, 7, 31, 32, 33, 63, 64, 65 }) { + for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_lightning_indexer(128, 16, kv, 3, 4, 1, type_K)); + } + } + for (int kv : { 1, 7, 8, 63, 64, 65 }) { for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0}) { test_cases.emplace_back(new test_lightning_indexer(128, 64, kv, 32, 4, 1, type_K)); @@ -13243,7 +13460,7 @@ static std::vector> make_test_cases_perf() { // lightning_indexer for (int kv : { 256, 4096, 65536 }) { for (int bs : { 1, 512, 2048 }) { - for (int nh : { 4, 32, 64 }) { + for (int nh : { 4, 16, 32, 64 }) { for (int ns : { 1, 4 }) { for (ggml_type type_K : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0, GGML_TYPE_IQ4_NL}) { test_cases.emplace_back(new test_lightning_indexer(128, nh, kv, bs, ns, ns, type_K)); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0e61ebab5f4f5195510fe7ef3833ca1535b45769..82d8121515791beb86ca00e2c367320af3e76bba 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -282,13 +282,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { || arch == LLM_ARCH_GLM5_NEXT || arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MISTRAL_LARGE4 - || arch == LLM_ARCH_HY_V4) { + || arch == LLM_ARCH_HY_V4 + || arch == LLM_ARCH_HY_V4_FLASH) { n_embd = 128; n_head = 1; n_ff = 192; if (arch == LLM_ARCH_MISTRAL_LARGE4) { n_layer = 4; // cover NoPE, shared indexer and KV sharing layers } + if (arch == LLM_ARCH_HY_V4_FLASH) { + n_layer = 4; // dense + MoE across both sparse and sliding attention layers + } } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; } else if (arch == LLM_ARCH_CHAMELEON) { @@ -346,13 +350,18 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE || arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 || - arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT + arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT || + arch == LLM_ARCH_HY_V4_FLASH ) { GGML_ASSERT(n_layer >= 2); std::vector n_head_per_layer; n_head_per_layer.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { - n_head_per_layer.push_back(il == 1 ? 0 : n_head); + // hy_v4_flash has no recurrent layer: the odd layers are sliding-window and + // carry twice the heads over the same latent geometry + n_head_per_layer.push_back(arch == LLM_ARCH_HY_V4_FLASH + ? (il % 2 ? 2*n_head : n_head) + : (il == 1 ? 0 : n_head)); } // GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer. if (arch == LLM_ARCH_GLM5_NEXT) { @@ -360,7 +369,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer); } - ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); + if (arch == LLM_ARCH_HY_V4_FLASH) { + // MLA-as-MQA: one latent KV head on every layer, whatever the query head count + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, uint32_t(1)); + } else { + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); + } } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv); @@ -380,6 +394,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT || arch == LLM_ARCH_HY_V4 + || arch == LLM_ARCH_HY_V4_FLASH || arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MISTRAL_LARGE4) { // GLM5 next MLA is nope only, the cache row is the compressed latent alone. @@ -438,7 +453,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM || + arch == LLM_ARCH_HY_V4_FLASH) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { @@ -542,6 +558,58 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types); } + if (arch == LLM_ARCH_HY_V4_FLASH) { + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f); + ms.add_kv(LLM_KV_HYPER_CONNECTION_MAGNITUDE, 2.0f); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); + // the sliding window and its il%2 pattern come from the shared blocks above, + // which is what the per-layer head count is matched to + // the indexer lives on the non-SWA layers, which here are the even ones. The shared + // INDEXER_TOP_K above is deliberately huge, so every pool is selected and the result stays + // comparable with the full-attention path. + { + std::vector indexer_types; + indexer_types.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + indexer_types.push_back(il % 2 ? 0 : 1); + } + ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types); + } + ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, uint32_t(96)); + + // one Engram module, on layer 1. The bucket counts are tiny but must stay distinct and + // above 1, and their running sum is what the head offsets rebase into. + { + // the loader reads the bucket ranges as u64 arrays + const std::vector primes = {7, 11, 13, 17}; + std::vector offsets; + uint64_t base = 0; + for (uint64_t p : primes) { + offsets.push_back(base); + base += p; + } + ms.add_kv(LLM_KV_PLE_LAYERS, std::vector({1})); + ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, uint32_t(3)); + ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, uint32_t(2)); + ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, uint32_t(32)); + // n_modules * ngram_size multipliers, odd as the reference derives them + ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector({3, 5, 7})); + ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, offsets); + ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes); + ms.add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, uint32_t(1)); + std::vector token_map(n_vocab); + for (uint32_t t = 0; t < n_vocab; t++) { + token_map[t] = (int32_t) t; + } + ms.add_kv(LLM_KV_ENGRAM_TOKEN_MAP, token_map); + } + } + if (arch == LLM_ARCH_MISTRAL_LARGE4) { // layer 3 reads the KV cache of layer 2, both are NoPE ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, std::vector({1, 1, 0, 0})); @@ -636,7 +704,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (moe) { ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff); ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload - ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff); + // keep the latent width distinct from n_ff_exp, or a transposed routed_down / routed_up + // would still load and the orientation would go untested + ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, arch == LLM_ARCH_HY_V4_FLASH ? uint32_t(64) : n_ff); ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2)); // with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : arch == LLM_ARCH_CLOVER_MOE ? 4 : 2)); @@ -942,6 +1012,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_HUNYUAN_MOE: case LLM_ARCH_HY_V3: case LLM_ARCH_HY_V4: + case LLM_ARCH_HY_V4_FLASH: case LLM_ARCH_OPENAI_MOE: case LLM_ARCH_LFM2MOE: case LLM_ARCH_SMALLTHINKER: @@ -1024,7 +1095,7 @@ static bool arch_supported(const llm_arch arch) { // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI. #ifdef GGML_USE_WEBGPU if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP || - arch == LLM_ARCH_HY_V4) { + arch == LLM_ARCH_HY_V4 || arch == LLM_ARCH_HY_V4_FLASH) { return false; } #endif // GGML_USE_WEBGPU diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index f1ec3c456645c34aecbf418e1f847a14291968f6..6ed391ce4833d29f61c2f768145a5444d0512f46 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -42,6 +42,7 @@ add_library(mtmd models/granite-speech.cpp models/granite4-vision.cpp models/hunyuanvl.cpp + models/hy-v4-flash.cpp models/internvl.cpp models/kimivl.cpp models/kimik25.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 0504e8fdcc4c3ad980037dcdef6d3d627d483b18..5da9e7358c4e974cd5daf71b053d058982a2d908 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -561,6 +561,7 @@ enum projector_type { PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_NEMOTRON_OMNI, PROJECTOR_TYPE_HUNYUANVL, + PROJECTOR_TYPE_HY_V4_FLASH, PROJECTOR_TYPE_PARAKEET, PROJECTOR_TYPE_EXAONE4_5, PROJECTOR_TYPE_MINICPMV4_6, @@ -643,6 +644,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, + { PROJECTOR_TYPE_HY_V4_FLASH, "hy_v4_flash"}, { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, { PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"}, { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9a043d7f9ca32d5df2a78ab4e65026ae15399646..ce67f196128ee0468823d08bd6fcf208820be43c 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1127,6 +1127,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_HY_V4_FLASH: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_MLP: case PROJECTOR_TYPE_MLP_NORM: case PROJECTOR_TYPE_LDP: @@ -2281,6 +2285,22 @@ struct clip_model_loader { get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); hparams.set_warmup_n_tokens(32*32); } break; + case PROJECTOR_TYPE_HY_V4_FLASH: + { + hparams.n_merge = 2; + hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + hparams.image_resize_pad = PAD_NONE; + // the encoder ffn fuses the bias with the tanh gelu, see bias_activation_fusion + hparams.ffn_op = FFN_GELU; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + // n_merge divides the patch grid in clip_n_output_tokens, reject 0 + GGML_ASSERT(hparams.n_merge > 0); + // HF processor defaults, the gguf keys below carry the real values + hparams.set_limit_image_tokens(4, 4096); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); + hparams.set_warmup_n_tokens(16*16); // vision_config.min_image_size + } break; case PROJECTOR_TYPE_LFM2A: case PROJECTOR_TYPE_D1OMNI_A: { @@ -3793,6 +3813,16 @@ struct clip_model_loader { model.image_newline = get_tensor(TN_IMAGE_NEWLINE); model.view_seperator = get_tensor(TN_IMAGE_SEPERATOR, false); } break; + case PROJECTOR_TYPE_HY_V4_FLASH: + { + // perceive.proj.0 -> mm.0 (conv1), perceive.proj.2 -> mm.2 (conv2) + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + model.mm_pre_norm_w = get_tensor(string_format(TN_MM_PRE_NORM, "weight")); + model.image_newline = get_tensor(TN_IMAGE_NEWLINE); + } break; case PROJECTOR_TYPE_JANUS_PRO: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); @@ -4704,6 +4734,9 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_LFM3VL: return (img->nx() / params.patch_size) / 2; + case PROJECTOR_TYPE_HY_V4_FLASH: + // the projector appends one newline embedding per token row + return (img->nx() / params.patch_size) / params.n_merge + 1; case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: @@ -4737,6 +4770,8 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_MUSE_GLIMMER: case PROJECTOR_TYPE_LFM3VL: return (img->ny() / params.patch_size) / 2; + case PROJECTOR_TYPE_HY_V4_FLASH: + return (img->ny() / params.patch_size) / params.n_merge; case PROJECTOR_TYPE_STEP3VL: case PROJECTOR_TYPE_MINICPMV4_6: case PROJECTOR_TYPE_MINICPMV4_7: @@ -4988,6 +5023,14 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { int oh = (img->ny() / patch_size) / merge; n_patches = (ow + 1) * oh + 2; } break; + case PROJECTOR_TYPE_HY_V4_FLASH: + { + // no begin/end embeddings, the prompt carries the image start/end tokens + int merge = ctx->model.hparams.n_merge; + int ow = (img->nx() / patch_size) / merge; + int oh = (img->ny() / patch_size) / merge; + n_patches = (ow + 1) * oh; + } break; case PROJECTOR_TYPE_DEEPSEEKOCR2: { // 1024 global view -> 256 query tokens + 1 view separator = 257; @@ -6354,6 +6397,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } } break; case PROJECTOR_TYPE_HUNYUANVL: + case PROJECTOR_TYPE_HY_V4_FLASH: { // Compute the HunyuanVL 2D position embedding on CPU (with the // custom sf=(target+0.1)/n_grid bilinear sampling that the @@ -7073,6 +7117,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: return ctx->model.mm_model_proj->ne[1]; + case PROJECTOR_TYPE_HY_V4_FLASH: + return ctx->model.mm_1_w->ne[3]; // conv2d kernel is [1, 1, IC, OC] case PROJECTOR_TYPE_COGVLM: return ctx->model.mm_4h_to_h_w->ne[1]; case PROJECTOR_TYPE_DEEPSEEKOCR: diff --git a/tools/mtmd/models/hy-v4-flash.cpp b/tools/mtmd/models/hy-v4-flash.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a8753f7d5644ff2677767406b188bcecccdc6e9d --- /dev/null +++ b/tools/mtmd/models/hy-v4-flash.cpp @@ -0,0 +1,47 @@ +#include "models.h" + +ggml_cgraph * clip_graph_hy_v4_flash::build() { + const int merge = hparams.n_merge; + const int pw = n_patches_x; + const int ph = n_patches_y; + + // position embedding: a graph input, interpolated on CPU in clip_encode + // (same name and interpolation as hunyuanvl, see the branch there) + ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw); + ggml_set_name(pos_embd, "hunyuanvl_pos_embd"); + ggml_set_input(pos_embd); + + ggml_tensor * inp = build_inp(); + ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, pos_embd, nullptr); + + // rmsnorm mlp adapter + cur = build_norm(cur, model.mm_pre_norm_w, nullptr, NORM_TYPE_RMS, eps, -1); + + // [C, W*H] -> [W, H, C] for conv2d + cur = ggml_reshape_3d(ctx0, cur, n_embd, pw, ph); + cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); + cur = ggml_cont(ctx0, cur); + + // Conv2d(1152->4608, k=2, s=2) + GELU + Conv2d(4608->3072, k=1, s=1) + cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, merge, merge, 0, 0, 1, 1); + cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_0_b, 1, 1, model.mm_0_b->ne[0])); + cur = ggml_gelu_erf(ctx0, cur); // nn.GELU(), not the tanh approximation of the encoder ffn + cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 1, 1, 0, 0, 1, 1); + cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_1_b, 1, 1, model.mm_1_b->ne[0])); + + const int ow = pw / merge; + const int oh = ph / merge; + const int idim = (int)cur->ne[2]; // OC = n_mmproj_embd + + // append newline along W (dim 0) + ggml_tensor * nl = ggml_reshape_4d(ctx0, model.image_newline, 1, 1, idim, 1); + nl = ggml_repeat_4d(ctx0, nl, 1, oh, idim, 1); + cur = ggml_concat(ctx0, cur, nl, 0); + + // [OW+1, OH, OC] -> [OC, (OW+1)*OH] + cur = ggml_permute(ctx0, cur, 1, 2, 0, 3); + cur = ggml_cont_2d(ctx0, cur, idim, (ow + 1) * oh); + + ggml_build_forward_expand(gf, cur); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index c75004d4372545db179cb36a48a4d0be80c46a52..a87a0d0ef2b7f2298ff8b736a145926dcf17a789 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -226,6 +226,11 @@ struct clip_graph_hunyuanvl : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_hy_v4_flash : clip_graph { + clip_graph_hy_v4_flash(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_mobilenetv5 : clip_graph { clip_graph_mobilenetv5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d4ea2385258c79d575f11477bf1cd19781bee239..ace9bf4ec8d57a4a63c3eb0d67c080881d565052 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1048,6 +1048,14 @@ struct mtmd_context { img_end = "<|hy_place▁holder▁no▁101|>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_HY_V4_FLASH: + { + // <|hy_image▁start:HASH|> ... (image embeddings) ... <|hy_image▁end:HASH|> + // the trailing hash follows the tokenizer revision, so take it from the vocab + img_beg = lookup_token_text_prefix("<|hy_image▁start"); + img_end = lookup_token_text_prefix("<|hy_image▁end"); + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_EXAONE4_5: { // ... (image embeddings) ... @@ -1245,6 +1253,22 @@ private: return LLAMA_TOKEN_NULL; } + // returns the full text of the first token starting with prefix, for vocabularies + // where the marker tokens embed a per-revision tokenizer hash (hy-v4) + std::string lookup_token_text_prefix(const std::string & prefix) { + if (vocab == nullptr) { + return ""; + } + const int n_vocab = llama_vocab_n_tokens(vocab); + for (int i = 0; i < n_vocab; i++) { + std::string piece = token_to_piece(vocab, i, true); + if (piece.rfind(prefix, 0) == 0) { + return piece; + } + } + return ""; + } + std::string token_to_piece(const llama_vocab * vocab, llama_token token, bool special) { if (vocab == nullptr) { throw std::runtime_error("llama_vocab is not provided");