Instructions to use AngelSlim/Hy4-flash-GGUF with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use AngelSlim/Hy4-flash-GGUF with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M # Run inference directly in the terminal: llama cli -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M # Run inference directly in the terminal: llama cli -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M # Run inference directly in the terminal: ./build/bin/llama cli -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Use Docker
docker model run hf.co/AngelSlim/Hy4-flash-GGUF:Q4_K_M
- Llama new
- LM Studio
- Jan
- Ollama
How to use AngelSlim/Hy4-flash-GGUF with Ollama:
ollama run hf.co/AngelSlim/Hy4-flash-GGUF:Q4_K_M
- Unsloth Desktop
- Pi
How to use AngelSlim/Hy4-flash-GGUF with Pi:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "llama-cpp": { "baseUrl": "http://localhost:9931/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "AngelSlim/Hy4-flash-GGUF:Q4_K_M" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Docker Model Runner
How to use AngelSlim/Hy4-flash-GGUF with Docker Model Runner:
docker model run hf.co/AngelSlim/Hy4-flash-GGUF:Q4_K_M
- Lemonade
How to use AngelSlim/Hy4-flash-GGUF with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull AngelSlim/Hy4-flash-GGUF:Q4_K_M
Run and chat with the model
lemonade run user.Hy4-flash-GGUF-Q4_K_M
List all available models
lemonade list
- Hermes Agent
How to use AngelSlim/Hy4-flash-GGUF with Hermes Agent:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:9931/v1 hermes config set model.default AngelSlim/Hy4-flash-GGUF:Q4_K_M
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use AngelSlim/Hy4-flash-GGUF with OpenClaw:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf AngelSlim/Hy4-flash-GGUF:Q4_K_M
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:9931/v1 \ --custom-model-id "AngelSlim/Hy4-flash-GGUF:Q4_K_M" \ --custom-provider-id llama-cpp \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download patch/hy4-flash-mac-minimal.patch from AngelSlim/Hy4-flash-GGUF: direct link, hf CLI and curl.
- Browser
- Download file 291 kB
-
https://huggingface.co/AngelSlim/Hy4-flash-GGUF/resolve/main/patch/hy4-flash-mac-minimal.patch
- Command line
-
hf download hf://AngelSlim/Hy4-flash-GGUF/patch/hy4-flash-mac-minimal.patch
-
curl -L -o hy4-flash-mac-minimal.patch https://huggingface.co/AngelSlim/Hy4-flash-GGUF/resolve/main/patch/hy4-flash-mac-minimal.patch
291 kB
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 3ed418140..7f7b0904a 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -2711,6 +2711,16 @@ extern "C" {
// broadcast:
// ne3 % ne33 == 0
//
+ // same, over the K rows listed in rows (I32 [n_out]): k holds every row ([n_embd, 1, n_rows_k, n_stream]),
+ // the result and the mask have one entry per list element (the gather is fused into the scoring)
+ GGML_API struct ggml_tensor * ggml_lightning_indexer_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * weights,
+ struct ggml_tensor * mask,
+ struct ggml_tensor * rows);
+
GGML_API struct ggml_tensor * ggml_lightning_indexer(
struct ggml_context * ctx,
struct ggml_tensor * q,
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index dbdde6997..463f6801a 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -12457,6 +12457,7 @@ void ggml_compute_forward_lightning_indexer(
const ggml_tensor * k = dst->src[1];
const ggml_tensor * w = dst->src[2]; // weights
const ggml_tensor * m = dst->src[3]; // mask
+ const ggml_tensor * r = dst->src[4]; // optional list of the K rows to score
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT( q->type == GGML_TYPE_F32);
@@ -12484,7 +12485,8 @@ void ggml_compute_forward_lightning_indexer(
const int n_head = q->ne[1];
const int n_tokens = q->ne[2];
const int n_stream = q->ne[3];
- const int n_kv = k->ne[2];
+ const int n_kv = r ? (int) r->ne[0] : (int) k->ne[2];
+ const int32_t * rows = r ? (const int32_t *) r->data : nullptr;
ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float;
GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type");
@@ -12509,7 +12511,9 @@ void ggml_compute_forward_lightning_indexer(
const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3);
float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 );
for (int ik = ir0; ik < ir1; ++ik) {
- char * k_row = (char *) k->data + ik*nbk2 + s*nbk3;
+ const int64_t ir = rows ? rows[ik] : ik;
+ GGML_ASSERT(ir >= 0 && ir < k->ne[2]);
+ char * k_row = (char *) k->data + ir*nbk2 + s*nbk3;
if (k_to_float) {
k_to_float(k_row, k_row_f32, n_embd);
} else {
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 06956ce7a..bd2f5590b 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -6209,7 +6209,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_SOLVE_TRI:
return true;
case GGML_OP_LIGHTNING_INDEXER:
- return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
+ // the row-list form (src[4]) is Metal/CPU only
+ return op->src[4] == nullptr && ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
case GGML_OP_TOP_K_MASK:
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index 597c126ca..073872b90 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
@@ -12,6 +12,57 @@
#import <Metal/Metal.h>
+#include <mach/mach_time.h>
+#include <math.h>
+
+// hy4 profiling knobs, read once:
+// GGML_METAL_N_MAIN nodes encoded+committed by the calling thread first: <1 = fraction of the graph, >=1 = node count
+// GGML_METAL_CB_TIMING=1 log host encode and GPU start/end times of every graph command buffer
+static float hy4_metal_n_main(void) {
+ static float v = -1.0f;
+ if (v < 0.0f) {
+ v = 0.0f;
+ const char * env = getenv("GGML_METAL_N_MAIN");
+ if (env != NULL && *env != '\0') {
+ char * end = NULL;
+ const float f = strtof(env, &end);
+ if (*end == '\0' && isfinite(f) && f >= 0.0f && f <= 1e6f) {
+ v = f;
+ } else {
+ GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_MAIN='%s'\n", __func__, env);
+ }
+ }
+ }
+ return v;
+}
+
+static bool hy4_metal_cb_timing(void) {
+ static int v = -1;
+ if (v < 0) {
+ v = getenv("GGML_METAL_CB_TIMING") != NULL;
+ }
+ return v;
+}
+
+// GGML_METAL_OP_PROFILE=1 encode every op (or fused group) in its own command buffer and log its GPU time;
+// serializes the graph, so only relative costs are meaningful
+static bool hy4_metal_op_profile(void) {
+ static int v = -1;
+ if (v < 0) {
+ v = getenv("GGML_METAL_OP_PROFILE") != NULL;
+ }
+ return v;
+}
+
+// same timebase as MTLCommandBuffer GPUStartTime / GPUEndTime
+static double hy4_metal_host_time(void) {
+ static mach_timebase_info_data_t tb;
+ if (tb.denom == 0) {
+ mach_timebase_info(&tb);
+ }
+ return (double) mach_absolute_time() * tb.numer / tb.denom * 1e-9;
+}
+
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -59,6 +110,8 @@
struct ggml_cgraph * gf;
+ uint64_t hy4_graph_id; // GGML_METAL_CB_TIMING: graph counter
+
// the callback given to the thread pool
void (^encode_async)(size_t ith);
@@ -482,7 +535,9 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph *
}
// number of nodes encoded by the main thread (empirically determined)
- const int n_main = MAX(64, 0.1*gf->n_nodes);
+ const float n_main_env = hy4_metal_n_main();
+ const int n_main = n_main_env <= 0.0f ? MAX(64, 0.1*gf->n_nodes) :
+ n_main_env < 1.0f ? MAX(1, (int) (n_main_env*gf->n_nodes)) : (int) n_main_env;
// number of threads in addition to the main thread
const int n_cb = ctx->n_cb;
@@ -499,6 +554,7 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph *
@autoreleasepool {
ctx->gf = gf;
+ ctx->hy4_graph_id++;
if (ctx->n_cb == 0) {
// single-threaded encoding: the whole graph is encoded by one command buffer
@@ -724,7 +780,8 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) {
void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
// when fusion stats are collected the graph must be encoded by a single thread so the
// counters are race-free; override whatever the caller requested
- if (ggml_metal_fusion_info_stats(ctx->finfo)) {
+ // GGML_METAL_OP_PROFILE commits one buffer per op, which only keeps dependency order from one thread
+ if (ggml_metal_fusion_info_stats(ctx->finfo) || hy4_metal_op_profile()) {
n_cb = 0;
}
@@ -749,6 +806,8 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
const int n_nodes_per_cb = ctx->n_nodes_per_cb;
+ const double t_enc0 = hy4_metal_cb_timing() ? hy4_metal_host_time() : 0.0;
+
int idx_start = 0;
int idx_end = n_nodes_0;
@@ -759,6 +818,54 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
id<MTLCommandBuffer> cmd_buf = ctx->cmd_bufs[cb_idx].obj;
+ if (hy4_metal_op_profile()) {
+ // the assigned buffer stays empty; commit it first so the per-op buffers are not queued behind it
+ [cmd_buf commit];
+
+ id<MTLCommandQueue> queue = ggml_metal_device_get_queue(ctx->dev);
+ struct ggml_cgraph * gf = ctx->gf;
+ const unsigned long long gid = ctx->hy4_graph_id;
+
+ int * idxs = (int *) malloc(sizeof(int) * (size_t) MAX(1, idx_end - idx_start));
+ int n_idxs = 0;
+ for (int i = idx_start; i < idx_end; ++i) {
+ if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) {
+ idxs[n_idxs++] = i;
+ }
+ }
+
+ id<MTLCommandBuffer> last = nil;
+ for (int k = 0; k < n_idxs; ) {
+ id<MTLCommandBuffer> cb = [queue commandBufferWithUnretainedReferences];
+
+ ggml_metal_op_t ctx_op = ggml_metal_op_init(ctx->dev, cb, gf, ctx->finfo, idxs[k], idx_end, false, false, 0);
+ const int res = MAX(1, ggml_metal_op_encode(ctx_op, 0));
+ ggml_metal_op_free(ctx_op);
+
+ const struct ggml_tensor * node = gf->nodes[idxs[k]];
+ const struct ggml_tensor * last_node = gf->nodes[idxs[MIN(k + res, n_idxs) - 1]];
+ char label[256];
+ snprintf(label, sizeof(label), "%s|%s|%s|%s", ggml_op_desc(node), ggml_get_name(node),
+ node->src[0] ? ggml_get_name(node->src[0]) : "-", ggml_get_name(last_node));
+ NSString * lbl = [NSString stringWithUTF8String:label];
+ const int i0 = idxs[k];
+ [cb addCompletedHandler:^(id<MTLCommandBuffer> b) {
+ if (b.status == MTLCommandBufferStatusError) {
+ GGML_LOG_ERROR("hy4_op g=%llu i=%d failed: %s\n", gid, i0, [[b.error localizedDescription] UTF8String]);
+ }
+ GGML_LOG_INFO("hy4_op g=%llu i=%d n=%d gpu0=%.7f gpu1=%.7f %s\n", gid, i0, res, b.GPUStartTime, b.GPUEndTime, [lbl UTF8String]);
+ }];
+ [cb commit];
+ last = cb;
+ k += res;
+ }
+ free(idxs);
+ if (last) {
+ [last waitUntilCompleted];
+ }
+ return;
+ }
+
ggml_metal_op_t ctx_op = ggml_metal_op_init(
ctx->dev,
cmd_buf,
@@ -781,6 +888,17 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
ggml_metal_op_free(ctx_op);
+ if (hy4_metal_cb_timing()) {
+ const double t_enc1 = hy4_metal_host_time();
+ const unsigned long long gid = ctx->hy4_graph_id;
+ const int is_main = cb_idx == n_cb_l;
+ const int n_nodes = idx_end - idx_start;
+ [cmd_buf addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+ GGML_LOG_INFO("hy4_cbt g=%llu cb=%d main=%d nodes=%d enc0=%.6f enc1=%.6f gpu0=%.6f gpu1=%.6f\n",
+ gid, cb_idx, is_main, n_nodes, t_enc0, t_enc1, cb.GPUStartTime, cb.GPUEndTime);
+ }];
+ }
+
if (cb_idx < 2 || ctx->abort_callback == NULL) {
[cmd_buf commit];
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 463d57c7b..cfa020827 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -588,6 +588,60 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_met
return res;
}
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ggml_metal_library_t lib, const ggml_tensor * pre, bool with_mul, bool with_gate) {
+ char base[256];
+ char name[256];
+
+ const bool gated = ggml_get_op_params_i32(pre, 1) != 0;
+ if (with_gate) {
+ GGML_ASSERT(!gated && with_mul);
+ snprintf(base, 256, "kernel_hy4_hc_gate_pre_rms_mul_f32");
+ } else {
+ snprintf(base, 256, "kernel_hy4_hc_pre%s_rms%s_f32", gated ? "_gated" : "", with_mul ? "_mul" : "");
+ }
+ snprintf(name, 256, "%s_n_hc=%d", base, (int) pre->src[0]->ne[1]);
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ ggml_metal_cv_t cv = ggml_metal_cv_init();
+ ggml_metal_cv_set_int32(cv, (int32_t) pre->src[0]->ne[1], FC_DSV4_HC + 0);
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
+
+ ggml_metal_cv_free(cv);
+ }
+
+ res.smem = 32*sizeof(float);
+
+ return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_add_rms(ggml_metal_library_t lib) {
+ const char * name = "kernel_hy4_add_rms_norm_mul_f32";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+ }
+
+ res.smem = 32*sizeof(float);
+
+ return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_post_rms(ggml_metal_library_t lib) {
+ const char * name = "kernel_hy4_hc_post_rms_f32";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+ }
+
+ res.smem = 32*sizeof(float);
+
+ return res;
+}
+
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(
ggml_metal_library_t lib, const ggml_tensor * op, int32_t nc, bool use_silu) {
GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
@@ -1095,9 +1149,19 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
} break;
case GGML_TYPE_Q8_0:
{
- nsg = N_SG_Q8_0;
- nr0 = N_R0_Q8_0;
- smem = 32*sizeof(float)*N_R0_Q8_0;
+ if (ne00 <= 512 && tsrc1 == GGML_TYPE_F32) {
+ // short rows (e.g. per-head MLA projections): whole rows per simdgroup, no cross-simdgroup
+ // reduce; 4 rows per simdgroup up to 256 columns, 2 above
+ nsg = N_SG_Q8_0_SHORT;
+ const int r = ne00 <= 256 ? 4 : 2;
+ nr0 = r*nsg; // rows per threadgroup
+ smem = 0;
+ suffix = r == 4 ? "_short_r4" : "_short_r2";
+ } else {
+ nsg = N_SG_Q8_0;
+ nr0 = N_R0_Q8_0;
+ smem = 32*sizeof(float)*N_R0_Q8_0;
+ }
} break;
case GGML_TYPE_MXFP4:
{
@@ -2340,6 +2404,25 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale(ggml_
return res;
}
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_rope_concat(ggml_metal_library_t lib) {
+ const char * base = "kernel_hy4_rope_concat_f32";
+ const char * name = "kernel_hy4_rope_concat_f32_imrope=0_is_back=0";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ ggml_metal_cv_t cv = ggml_metal_cv_init();
+
+ ggml_metal_cv_set_bool(cv, false, FC_ROPE + 0);
+ ggml_metal_cv_set_bool(cv, false, FC_ROPE + 1);
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
+
+ ggml_metal_cv_free(cv);
+ }
+
+ return res;
+}
+
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_library_t lib, const ggml_tensor * op) {
assert(op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK);
@@ -2458,6 +2541,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d(ggml_m
return res;
}
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hc_gate(ggml_metal_library_t lib) {
+ const char * name = "kernel_hc_gate_f32";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+ }
+
+ return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sigmoid_mul(ggml_metal_library_t lib) {
+ const char * name = "kernel_sigmoid_mul_f32";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+ }
+
+ return res;
+}
+
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake(ggml_metal_library_t lib, enum ggml_type type) {
GGML_ASSERT(type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index e096d611b..2ba9e3fbc 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -59,6 +59,7 @@ struct ggml_metal_pipeline_with_params {
};
int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline);
+int ggml_metal_pipeline_simd_width (struct ggml_metal_pipeline_with_params pipeline);
//
// MTLCommandBuffer wrapper
@@ -168,6 +169,12 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_tran
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hc_gate (ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sigmoid_mul (ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_pre_rms (ggml_metal_library_t lib, const struct ggml_tensor * pre, bool with_mul, bool with_gate);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_post_rms (ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_add_rms (ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_rope_concat (ggml_metal_library_t lib);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 68a9addb4..9a3799de1 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -100,6 +100,10 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi
return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup;
}
+int ggml_metal_pipeline_simd_width(struct ggml_metal_pipeline_with_params pipeline) {
+ return pipeline.pipeline->obj.threadExecutionWidth;
+}
+
//
// MTLLibrary collection (one library per op-source, compiled separately)
//
@@ -1822,6 +1826,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
!ggml_is_contiguous_rows(op->src[3])) {
return false;
}
+ if (op->src[4] && (op->src[4]->type != GGML_TYPE_I32 || !ggml_is_contiguous(op->src[4]))) {
+ return false;
+ }
switch (op->src[1]->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp
index 020cad29d..8d022e417 100644
--- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp
@@ -7,6 +7,7 @@
#include <algorithm>
#include <cstddef>
+#include <cstdint>
#include <cstring>
#include <set>
#include <string>
@@ -288,6 +289,363 @@ static bool ggml_metal_fusion_check_gdn_cache(
return true;
}
+// UNARY (sigmoid) + MUL: dst = x * sigmoid(g) element-wise, x and g of the same contiguous F32 shape.
+// The attention output gate of hy_v4_flash.
+static bool ggml_metal_fusion_check_sigmoid_mul(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(fusion);
+ GGML_UNUSED(gf);
+ GGML_UNUSED(node_idxs);
+ GGML_UNUSED(idx);
+ GGML_UNUSED(mode);
+
+ const ggml_tensor * sig = nodes[0];
+ const ggml_tensor * mul = nodes[1];
+
+ if (ggml_get_unary_op(sig) != GGML_UNARY_OP_SIGMOID || mul->src[1] != sig || mul->src[0] == sig) {
+ return false;
+ }
+
+ const ggml_tensor * g = sig->src[0];
+ const ggml_tensor * x = mul->src[0];
+
+ if (g->type != GGML_TYPE_F32 || x->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 ||
+ !ggml_are_same_shape(g, x) || !ggml_are_same_shape(x, mul) ||
+ !ggml_is_contiguous(g) || !ggml_is_contiguous(x) || !ggml_is_contiguous(mul) ||
+ ggml_nelements(mul) >= INT32_MAX - 4) {
+ return false;
+ }
+
+ // the kernel moves float4s
+ for (const ggml_tensor * t : { g, x, mul }) {
+ if (t->data && ((uintptr_t) t->data) % 16 != 0) {
+ return false;
+ }
+ }
+
+ // each thread reads g[i], x[i] and writes dst[i]: storage may be shared element for element, but not shifted
+ // (leaf inputs are not covered by the generic range checks)
+ const ggml_tensor * ts[3] = { g, x, mul };
+ for (int a = 0; a < 3; ++a) {
+ for (int b = a + 1; b < 3; ++b) {
+ const char * pa = (const char *) ts[a]->data;
+ const char * pb = (const char *) ts[b]->data;
+ const size_t n = ggml_nbytes(mul);
+ if (pa && pb && pa != pb && pa < pb + n && pb < pa + n) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+// ADD + RMS_NORM + MUL: an elementwise sum of two same-shape F32 tensors normed and scaled by a weight row. The
+// sum has no other reader, so the fused kernel forms it in registers instead of writing it.
+static bool ggml_metal_fusion_check_add_rms(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(fusion);
+
+ const ggml_tensor * add = nodes[0];
+ const ggml_tensor * rms = nodes[1];
+ const ggml_tensor * mul = nodes[2];
+
+ const ggml_tensor * a = add->src[0];
+ const ggml_tensor * b = add->src[1];
+ const ggml_tensor * w = mul->src[1];
+
+ if (rms->src[0] != add || mul->src[0] != rms || w == rms || w == add ||
+ add->type != GGML_TYPE_F32 || a->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32 ||
+ rms->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ // the kernel moves float4s over one row per threadgroup, with the weight shared by every row
+ if (add->ne[0] % 4 != 0 || add->ne[2] != 1 || add->ne[3] != 1 ||
+ !ggml_are_same_shape(a, add) || !ggml_are_same_shape(b, add) || !ggml_are_same_shape(mul, add) ||
+ !ggml_is_contiguous_rows(a) || !ggml_is_contiguous_rows(b) || !ggml_is_contiguous_rows(mul) ||
+ w->ne[0] != add->ne[0] || ggml_nrows(w) != 1 || !ggml_is_contiguous(w)) {
+ return false;
+ }
+
+ // RMS_NORM + MUL + ADD is its own fusion: matching here first would take the ADD that ends it out of its reach
+ // and change that chain's arithmetic (it adds inside the norm kernel), so leave the sequence to it
+ for (int i = node_idxs[idx + 2] + 1; i < gf->n_nodes; ++i) {
+ const ggml_tensor * next = gf->nodes[i];
+ if (ggml_op_is_empty(next->op)) {
+ continue;
+ }
+ if (next->op == GGML_OP_ADD && (next->src[0] == mul || next->src[1] == mul)) {
+ return false;
+ }
+ break;
+ }
+
+ if (mode == GGML_METAL_FUSION_FULL) {
+ if (!a->data || !b->data || !w->data || !mul->data) {
+ return false;
+ }
+
+ // the kernel moves float4s
+ for (const ggml_tensor * t : { a, b, w, (const ggml_tensor *) mul }) {
+ if (((uintptr_t) t->data) % 16 != 0 || (ggml_nrows(t) > 1 && t->nb[1] % 16 != 0)) {
+ return false;
+ }
+ }
+ // the output is written while both addends and the weight are still read (leaf inputs are not covered by
+ // the generic range checks). An addend may share its storage element for element: a thread writes only
+ // the elements it read itself, after the reduction barriers, and a threadgroup only touches its own row,
+ // so the row stride must match too. The weight is read by every threadgroup, so it must not overlap.
+ for (const ggml_tensor * t : { a, b }) {
+ if (ggml_metal_fusion_overlap(mul, t) &&
+ (mul->data != t->data || (ggml_nrows(mul) > 1 && mul->nb[1] != t->nb[1]))) {
+ return false;
+ }
+ }
+ if (ggml_metal_fusion_overlap(mul, w)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// DSV4_HC_PRE + RMS_NORM (+ MUL by a weight row): the reduced residual streams normed for the next projection.
+// The fused kernel keeps the float4 thread mapping of the rms_norm kernel, so the row must be a multiple of 4.
+static bool ggml_metal_fusion_check_hc_pre_rms(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(gf);
+ GGML_UNUSED(node_idxs);
+ GGML_UNUSED(idx);
+ GGML_UNUSED(mode);
+
+ const ggml_tensor * pre = nodes[0];
+ const ggml_tensor * rms = nodes[1];
+ const ggml_tensor * mul = fusion->ops.size() > 2 ? nodes[2] : nullptr;
+
+ if (rms->src[0] != pre || pre->type != GGML_TYPE_F32 || rms->type != GGML_TYPE_F32 ||
+ pre->ne[0] % 4 != 0 || pre->ne[2] != 1 || pre->ne[3] != 1 || !ggml_is_contiguous(pre) || !ggml_is_contiguous(rms) ||
+ pre->nb[0] != sizeof(float)) {
+ return false;
+ }
+
+ if (mul) {
+ const ggml_tensor * w = mul->src[1];
+ if (mul->src[0] != rms || w == rms || w == pre || mul->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 ||
+ w->ne[0] != pre->ne[0] || ggml_nrows(w) != 1 || !ggml_is_contiguous(w) || !ggml_is_contiguous(mul)) {
+ return false;
+ }
+ }
+
+ // the output is written while the streams and the weights of every token are still read (leaf inputs are not
+ // covered by the generic range checks)
+ if (mode == GGML_METAL_FUSION_FULL) {
+ const ggml_tensor * dst = mul ? mul : rms;
+ if (ggml_metal_fusion_overlap(dst, pre->src[0]) || ggml_metal_fusion_overlap(dst, pre->src[1]) ||
+ (mul && ggml_metal_fusion_overlap(dst, mul->src[1]))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// DSV4_HC_POST (no comb) + RESHAPE + RMS_NORM: the streams a sublayer leaves and the norm of their concatenation
+// (the input of the next hc_mixes projection). Both are outputs; the norm must be the only reader of the reshape.
+static bool ggml_metal_fusion_check_hc_post_rms(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(nodes);
+
+ const std::vector<ggml_op> & ops_all = fusion->ops_all;
+
+ const int raw_start = node_idxs[idx];
+ const int raw_end = node_idxs[idx + (int) fusion->ops.size() - 1];
+ if (raw_end - raw_start + 1 != (int) ops_all.size()) {
+ return false;
+ }
+
+ int raw_idxs[GGML_METAL_FUSION_MAX];
+ for (int i = 0; i < (int) ops_all.size(); ++i) {
+ raw_idxs[i] = raw_start + i;
+ if (gf->nodes[raw_start + i]->op != ops_all[i]) {
+ return false;
+ }
+ }
+
+ const ggml_tensor * post = gf->nodes[raw_start + 0];
+ const ggml_tensor * flat = gf->nodes[raw_start + 1];
+ const ggml_tensor * rms = gf->nodes[raw_start + 2];
+ const ggml_tensor * x = post->src[0];
+ const ggml_tensor * res = post->src[1];
+ const ggml_tensor * pw = post->src[2];
+
+ if (post->src[3] || flat->src[0] != post || rms->src[0] != flat ||
+ post->type != GGML_TYPE_F32 || rms->type != GGML_TYPE_F32 || x->type != GGML_TYPE_F32 ||
+ res->type != GGML_TYPE_F32 || pw->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ const int64_t n_embd = post->ne[0];
+ const int64_t hc = post->ne[1];
+
+ if (n_embd % 4 != 0 || hc != 4 || post->ne[3] != 1 || !ggml_is_contiguous(post) || !ggml_is_contiguous(rms) ||
+ flat->ne[0] != n_embd*hc || flat->ne[1] != post->ne[2] || rms->ne[0] != flat->ne[0] || rms->ne[1] != flat->ne[1] ||
+ rms->ne[2] != 1 || rms->ne[3] != 1) {
+ return false;
+ }
+
+ const int outputs[2] = { raw_start, raw_end };
+ if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, (int) ops_all.size(), ops_all.data(), outputs, 2)) {
+ return false;
+ }
+
+ if (mode == GGML_METAL_FUSION_FULL) {
+ if (!post->data || !rms->data || !x->data || !res->data || !pw->data) {
+ return false;
+ }
+ // the norm is written while the streams are read back, and the streams while x/residual are read
+ if (ggml_metal_fusion_overlap(rms, post) || ggml_metal_fusion_overlap(rms, x) || ggml_metal_fusion_overlap(rms, res) ||
+ ggml_metal_fusion_overlap(rms, pw) || ggml_metal_fusion_overlap(post, x) || ggml_metal_fusion_overlap(post, pw) ||
+ (ggml_metal_fusion_overlap(post, res) && (post->data != res->data || post->nb[1] != res->nb[1] || post->nb[2] != res->nb[2]))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// ROPE (norm, F32) + CONCAT along dim 0 with the rotation as the second source: the kernel writes the rotation
+// straight into the concatenation. The rotation must have no other reader.
+static bool ggml_metal_fusion_check_rope_concat(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(nodes);
+
+ const std::vector<ggml_op> & ops_all = fusion->ops_all;
+
+ const int raw_start = node_idxs[idx];
+ const int raw_end = node_idxs[idx + 1];
+ if (raw_end != raw_start + 1) {
+ return false;
+ }
+
+ const ggml_tensor * rope = gf->nodes[raw_start];
+ const ggml_tensor * cat = gf->nodes[raw_end];
+ const ggml_tensor * a = cat->src[0];
+
+ const int mode_rope = ((const int32_t *) rope->op_params)[2];
+
+ if (mode_rope != 0 || rope->type != GGML_TYPE_F32 || rope->src[0]->type != GGML_TYPE_F32 ||
+ cat->type != GGML_TYPE_F32 || a->type != GGML_TYPE_F32 || cat->src[1] != rope || a == rope ||
+ ggml_get_op_params_i32(cat, 0) != 0 || rope->src[0]->ne[0] % 2 != 0 || rope->src[0]->ne[0] > 1024 ||
+ cat->ne[0] != a->ne[0] + rope->ne[0] ||
+ a->ne[1] != rope->ne[1] || a->ne[2] != rope->ne[2] || a->ne[3] != rope->ne[3] ||
+ cat->ne[1] != rope->ne[1] || cat->ne[2] != rope->ne[2] || cat->ne[3] != rope->ne[3] ||
+ a->nb[0] != sizeof(float) || cat->nb[0] != sizeof(float)) {
+ return false;
+ }
+
+ int raw_idxs[2] = { raw_start, raw_end };
+ const int outputs[1] = { raw_end };
+ if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, 2, ops_all.data(), outputs, 1)) {
+ return false;
+ }
+
+ if (mode == GGML_METAL_FUSION_FULL) {
+ if (!cat->data || !a->data || !rope->src[0]->data) {
+ return false;
+ }
+ // the concatenation is written while both sources, the positions and the frequency factors are read (leaf
+ // inputs are not covered by the generic range checks)
+ if (ggml_metal_fusion_overlap(cat, a) || ggml_metal_fusion_overlap(cat, rope->src[0]) ||
+ ggml_metal_fusion_overlap(cat, rope->src[1]) || (rope->src[2] && ggml_metal_fusion_overlap(cat, rope->src[2]))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// MUL + ADD + SIGMOID + SCALE (+ SCALE): s1*(s0*sigmoid(x*f + b) + c0) + c1 over a [n, rows] view, with a
+// single factor f and a per-column bias b. The hyper-connection pre/post gates of hy_v4_flash.
+static bool ggml_metal_fusion_check_hc_gate(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ GGML_UNUSED(mode);
+ GGML_UNUSED(gf);
+ GGML_UNUSED(node_idxs);
+ GGML_UNUSED(idx);
+
+ const ggml_tensor * mul = nodes[0];
+ const ggml_tensor * add = nodes[1];
+ const ggml_tensor * sig = nodes[2];
+
+ if (ggml_get_unary_op(sig) != GGML_UNARY_OP_SIGMOID) {
+ return false;
+ }
+
+ const ggml_tensor * x = mul->src[0];
+ const ggml_tensor * f = mul->src[1];
+ const ggml_tensor * b = add->src[0] == mul ? add->src[1] : add->src[0];
+ const ggml_tensor * dst = nodes[fusion->ops.size() - 1];
+
+ // the bias must be an outside operand, and the int32 indexing of the kernel must cover dst
+ if (b == mul || ggml_nelements(dst) >= INT32_MAX) {
+ return false;
+ }
+
+ // dst is written element-wise from x: overlapping storage is only safe with the same element mapping
+ if (x->data != nullptr && dst->data != nullptr) {
+ const char * x0 = (const char *) x->data;
+ const char * d0 = (const char *) dst->data;
+ const bool same = x0 == d0 && ggml_is_contiguous(x);
+ if (!same && x0 < d0 + ggml_nbytes(dst) && d0 < x0 + ggml_nbytes(x)) {
+ return false;
+ }
+ }
+
+ bool ok = x->type == GGML_TYPE_F32 && f->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32 &&
+ dst->type == GGML_TYPE_F32 && ggml_are_same_shape(x, mul) &&
+ ggml_nelements(f) == 1 &&
+ ggml_nelements(b) == x->ne[0] && b->ne[0] == x->ne[0] && ggml_is_contiguous(b) &&
+ x->ne[2] == 1 && x->ne[3] == 1 && x->nb[0] == sizeof(float) &&
+ ggml_is_contiguous(dst);
+
+ for (size_t i = 1; ok && i < fusion->ops.size(); ++i) {
+ ok = nodes[i]->type == GGML_TYPE_F32;
+ }
+
+ return ok;
+}
+
// MUL + SIN + SQR + MUL + ADD (snake activation)
static bool ggml_metal_fusion_check_snake(
const ggml_metal_fusion * fusion,
@@ -500,7 +858,8 @@ static bool ggml_metal_fusion_check_topk_moe(
return true;
}
-#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 8
+#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 16
+static_assert(2*GGML_METAL_MOE_REDUCE_MAX_EXPERTS <= GGML_METAL_FUSION_MAX, "the MUL + VIEWs + ADDs of a pattern must fit");
struct ggml_metal_moe_reduce_match {
const ggml_tensor * experts;
@@ -760,8 +1119,87 @@ static const std::vector<ggml_op> ops_moe_reduce_8 = {
GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD
};
+// MUL, n VIEWs, n - 1 ADDs (hy_v4_flash routes to 16 experts)
+static std::vector<ggml_op> ggml_metal_ops_moe_reduce(int n) {
+ std::vector<ggml_op> ops(1, GGML_OP_MUL);
+ ops.insert(ops.end(), n, GGML_OP_VIEW);
+ ops.insert(ops.end(), n - 1, GGML_OP_ADD);
+ return ops;
+}
+
static const std::vector<ggml_op> ops_mul_mat_add = { GGML_OP_MUL_MAT, GGML_OP_ADD };
+static const std::vector<ggml_op> ops_hc_gate_1 = { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE };
+static const std::vector<ggml_op> ops_hc_gate_2 = { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_SCALE };
+
+static const std::vector<ggml_op> ops_sigmoid_mul = { GGML_OP_UNARY, GGML_OP_MUL };
+
+static const std::vector<ggml_op> ops_add_rms = { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL };
+static const std::vector<ggml_op> ops_hc_pre_rms = { GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM };
+static const std::vector<ggml_op> ops_hc_post_rms = { GGML_OP_DSV4_HC_POST, GGML_OP_RESHAPE, GGML_OP_RMS_NORM };
+static const std::vector<ggml_op> ops_rope_concat = { GGML_OP_ROPE, GGML_OP_CONCAT };
+static const std::vector<ggml_op> ops_hc_pre_rms_mul = { GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM, GGML_OP_MUL };
+
+// the hc gate (MUL + ADD + SIGMOID + SCALE) computing the weights of DSV4_HC_PRE + RMS_NORM + MUL: the weights are
+// computed in the HC_PRE kernel (four per token, as kernel_hc_gate_f32 does) instead of being stored
+// (the raw sequence has the VIEW of the gate bias between MUL and ADD; it is listed so that the allocation
+// dependencies of ggml_metal_fusion_add_alloc_deps, which match raw nodes, apply)
+static const std::vector<ggml_op> ops_hc_gate_pre_rms = {
+ GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM, GGML_OP_MUL
+};
+
+static bool ggml_metal_fusion_check_hc_gate_pre_rms(
+ const ggml_metal_fusion * fusion,
+ const ggml_tensor * const * nodes,
+ const ggml_cgraph * gf,
+ const int * node_idxs,
+ int idx,
+ ggml_metal_fusion_mode mode) {
+ static const ggml_metal_fusion gate(GGML_METAL_FUSION_HC_GATE, ops_hc_gate_1, {}, false, nullptr);
+ static const ggml_metal_fusion prms(GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms_mul, {}, false, nullptr);
+
+ // the gate as its own fusion would be checked: a chain of same-shape nodes
+ for (int j = 1; j < 4; ++j) {
+ if ((nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) || !ggml_are_same_shape(nodes[j], nodes[j - 1])) {
+ return false;
+ }
+ }
+
+ // the gate nodes are not written: the norm weight must not be one of them
+ for (int j = 0; j < 4; ++j) {
+ if (nodes[6]->src[1] == nodes[j]) {
+ return false;
+ }
+ }
+
+ const ggml_tensor * pre = nodes[4];
+ if (pre->src[1] != nodes[3] || ggml_get_op_params_i32(pre, 1) != 0 || pre->src[0]->ne[1] != nodes[3]->ne[0] || pre->src[0]->ne[1] > 8 ||
+ nodes[3]->ne[1] != pre->ne[1]) {
+ return false;
+ }
+
+ if (!ggml_metal_fusion_check_hc_gate(&gate, nodes, gf, node_idxs, idx, mode) ||
+ !ggml_metal_fusion_check_hc_pre_rms(&prms, nodes + 4, gf, node_idxs, idx + 4, mode)) {
+ return false;
+ }
+
+ // the kernel also reads the mixes and the gate scale and bias
+ if (mode == GGML_METAL_FUSION_FULL) {
+ const ggml_tensor * dst = nodes[6];
+ const ggml_tensor * gmul = nodes[0];
+ const ggml_tensor * gadd = nodes[1];
+ const ggml_tensor * gb = gadd->src[0] == gmul ? gadd->src[1] : gadd->src[0];
+ if (ggml_metal_fusion_overlap(dst, gmul->src[0]) || ggml_metal_fusion_overlap(dst, gmul->src[1]) ||
+ ggml_metal_fusion_overlap(dst, gb)) {
+ return false;
+ }
+ }
+
+ const int n = (int) fusion->ops.size();
+ const int outputs[1] = { node_idxs[idx + n - 1] };
+ return ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, n, fusion->ops.data(), outputs, 1);
+}
+
static const std::vector<ggml_metal_fusion> ggml_metal_fusions = {
{ GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm },
{ GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm },
@@ -788,8 +1226,18 @@ static const std::vector<ggml_metal_fusion> ggml_metal_fusions = {
{ GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce },
{ GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce },
{ GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce },
+ { GGML_METAL_FUSION_MOE_REDUCE, ggml_metal_ops_moe_reduce(16), {}, true, ggml_metal_fusion_check_moe_reduce },
{ GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu },
{ GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add },
+ { GGML_METAL_FUSION_HC_GATE, ops_hc_gate_1, {}, false, ggml_metal_fusion_check_hc_gate },
+ { GGML_METAL_FUSION_HC_GATE, ops_hc_gate_2, {}, false, ggml_metal_fusion_check_hc_gate },
+ { GGML_METAL_FUSION_SIGMOID_MUL, ops_sigmoid_mul, {}, false, ggml_metal_fusion_check_sigmoid_mul },
+ { GGML_METAL_FUSION_ADD_RMS, ops_add_rms, {}, false, ggml_metal_fusion_check_add_rms },
+ { GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms, {}, false, ggml_metal_fusion_check_hc_pre_rms },
+ { GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms_mul, {}, false, ggml_metal_fusion_check_hc_pre_rms },
+ { GGML_METAL_FUSION_HC_GATE_PRE_RMS, ops_hc_gate_pre_rms, {}, true, ggml_metal_fusion_check_hc_gate_pre_rms },
+ { GGML_METAL_FUSION_HC_POST_RMS, ops_hc_post_rms, {0}, true, ggml_metal_fusion_check_hc_post_rms },
+ { GGML_METAL_FUSION_ROPE_CONCAT, ops_rope_concat, {}, true, ggml_metal_fusion_check_rope_concat },
};
// ---- alloc deps -----------------------------------------------------------
@@ -1200,6 +1648,10 @@ int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) {
if (!fusion || total + len > GGML_METAL_FUSION_MAX) {
break;
}
+ // a pattern after the first must fit whole in the raw span, or the reorder could split it
+ if (i_f > 0 && idxs[i_f + len - 1] - idx + 1 > GGML_METAL_FUSION_MAX) {
+ break;
+ }
total += len;
i_f += len;
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h
index 5d994c249..e7f59ce58 100644
--- a/ggml/src/ggml-metal/ggml-metal-fusion.h
+++ b/ggml/src/ggml-metal/ggml-metal-fusion.h
@@ -20,7 +20,7 @@ struct ggml_metal_device_props;
// the maximum number of nodes that can be fused in a single kernel
// (also the maximum length of a packed fusion group during graph optimization)
-#define GGML_METAL_FUSION_MAX 16
+#define GGML_METAL_FUSION_MAX 32
typedef enum ggml_metal_fusion_mode {
// structural checks for the graph optimizer, which runs before the graph tensors are allocated (weights
@@ -43,6 +43,13 @@ typedef enum ggml_metal_fusion_id {
GGML_METAL_FUSION_MOE_REDUCE, // MUL + expert VIEWs + ADD chain (MoE output reduction)
GGML_METAL_FUSION_SSM_CONV_SILU, // SSM_CONV + UNARY (silu)
GGML_METAL_FUSION_MUL_MAT_ADD, // MUL_MAT + ADD (residual added in the few-row MMA store)
+ GGML_METAL_FUSION_HC_GATE, // MUL + ADD + SIGMOID + SCALE (+ SCALE): hyper-connection gate (hy_v4_flash)
+ GGML_METAL_FUSION_ADD_RMS, // ADD + RMS_NORM + MUL: the residual sum formed inside the norm (hy_v4_flash)
+ GGML_METAL_FUSION_HC_PRE_RMS, // DSV4_HC_PRE + RMS_NORM (+ MUL): the sublayer input of hy_v4_flash
+ GGML_METAL_FUSION_HC_GATE_PRE_RMS, // MUL + ADD + SIGMOID + SCALE (hc gate) + DSV4_HC_PRE + RMS_NORM + MUL
+ GGML_METAL_FUSION_HC_POST_RMS, // DSV4_HC_POST + RESHAPE + RMS_NORM: new streams and their norm (hy_v4_flash)
+ GGML_METAL_FUSION_ROPE_CONCAT, // ROPE + CONCAT(x, rope, 0): the [nope | rope] q and k of hy_v4_flash
+ GGML_METAL_FUSION_SIGMOID_MUL, // UNARY (sigmoid) + MUL: x * sigmoid(g), the attention output gate (hy_v4_flash)
} ggml_metal_fusion_id;
struct ggml_metal_fusion; // defined in ggml-metal-fusion.cpp
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 11c15feff..de4a7ab17 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -45,6 +45,9 @@
#define N_R0_Q8_0 2
#define N_SG_Q8_0 4
+// short-row Q8_0 mat-vec (ne00 <= 512): simdgroups per threadgroup
+#define N_SG_Q8_0_SHORT 4
+
#define N_R0_MXFP4 2
#define N_SG_MXFP4 2
@@ -514,6 +517,31 @@ typedef struct {
int32_t nrows;
} ggml_metal_kargs_flash_attn_ext_vec_reduce;
+// hy4 MLA decode (kernel_hy4_mla_decode_f16): one query token, n_head heads over one latent KV head
+typedef struct {
+ int32_t n_rows; // length of the row list (padded with -1)
+ int32_t rows_per_wg; // rows of the list per workgroup (multiple of the tile)
+ int32_t n_head;
+ int32_t nwg;
+ uint64_t nb_q_head; // bytes between the heads of q (f32)
+ uint64_t nb_k_row; // bytes between the K rows (f16)
+ float scale;
+ int32_t has_sinks;
+ int32_t has_mask; // 0: no mask; 2: list from kernel_flash_attn_ext_vec_idx (padded with -1);
+ // 3: the kernel compacts the mask row (n_cells long) itself. The mask is per cache cell
+ int32_t n_cells;
+ // mode 4: the rows come from the indexer picks (TOP_K_MASK inputs), the mask is its keep row
+ int32_t n_top;
+ int32_t n_blk;
+ int32_t n_pick;
+ int32_t n_extra;
+ uint64_t nbk0; // top_k
+ uint64_t nbs0; // score
+ uint64_t nbb0; // blocks
+ uint64_t nbb1;
+ uint64_t nbe0; // extra
+} ggml_metal_kargs_hy4_mla;
+
typedef struct {
int32_t ne00;
int32_t ne02;
@@ -705,6 +733,16 @@ typedef struct {
int32_t C;
} ggml_metal_kargs_snake;
+typedef struct {
+ int32_t n; // columns per row
+ int32_t n_rows;
+ uint64_t nb_x1; // row stride of the input view
+ float s0;
+ float c0;
+ float s1;
+ float c1;
+} ggml_metal_kargs_hc_gate;
+
typedef struct {
int32_t IC;
int32_t IH;
@@ -1307,6 +1345,7 @@ typedef struct {
uint64_t nbw3;
uint64_t nbm1;
uint64_t nbm3;
+ int32_t has_rows; // the K rows to score come from a list (buffer 6)
} ggml_metal_kargs_lightning_indexer;
typedef struct {
@@ -1368,6 +1407,47 @@ typedef struct {
float scale;
} ggml_metal_kargs_dsv4_hc_pre;
+// DSV4_HC_PRE + RMS_NORM (+ MUL), see kernel_hy4_hc_pre_rms_f32
+typedef struct {
+ ggml_metal_kargs_dsv4_hc_pre hc;
+ int32_t ne00_t; // n_embd/4
+ float eps;
+ uint64_t nb_y1; // output row stride
+ // the weights from the mixes (kernel_hc_gate_f32): s1*(s0*sigmoid(mix*f + b) + c0) + c1
+ uint64_t nb_m1; // mixes row stride
+ float s0;
+ float c0;
+ float s1;
+ float c1;
+} ggml_metal_kargs_hy4_hc_pre_rms;
+
+// DSV4_HC_POST (no comb) + RMS_NORM of the flattened streams, see kernel_hy4_hc_post_rms_f32
+typedef struct {
+ int32_t n_embd;
+ int32_t ne00_t; // hc*n_embd/4
+ uint64_t nb_x0;
+ uint64_t nb_x1;
+ uint64_t nb_r0;
+ uint64_t nb_r1;
+ uint64_t nb_r2;
+ uint64_t nb_p0;
+ uint64_t nb_p1;
+ uint64_t nb_d1; // stream stride of the HC_POST output (contiguous: n_embd floats)
+ uint64_t nb_d2;
+ uint64_t nb_y1; // row stride of the normed output
+ float eps;
+} ggml_metal_kargs_hy4_hc_post_rms;
+
+// ADD + RMS_NORM + MUL, see kernel_hy4_add_rms_norm_mul_f32
+typedef struct {
+ int32_t ne00;
+ int32_t ne00_t; // ne00/4
+ uint64_t nb_a1; // row strides of the two addends, the weight and the output
+ uint64_t nb_b1;
+ uint64_t nb_y1;
+ float eps;
+} ggml_metal_kargs_hy4_add_rms;
+
typedef struct {
int32_t n_embd;
int32_t n_tokens;
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index cace34bd4..a2b407ed3 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -81,6 +81,16 @@ struct ggml_metal_op {
return ggml_graph_node(gf, idxs[i]);
}
+ // the whole graph and the graph index of node i (nodes outside this encoder's range included)
+ const ggml_cgraph * graph() const {
+ return gf;
+ }
+
+ int graph_idx(int i) const {
+ assert(i >= 0 && i < (int) idxs.size());
+ return idxs[i];
+ }
+
// consult the fusion table for the longest pattern starting at i0
// returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes
const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const {
@@ -821,6 +831,15 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) {
int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
+ if (op->op == GGML_OP_UNARY && ggml_get_unary_op(op) == GGML_UNARY_OP_SIGMOID && ctx->use_fusion()) {
+ int n = 1;
+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SIGMOID_MUL) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_sigmoid_mul_fused(ctx, idx);
+ }
+ }
+
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
@@ -1394,8 +1413,10 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK);
+ const ggml_tensor * r = op->src[4]; // optional list of the K rows
+
ggml_metal_kargs_lightning_indexer args = {
- /*.n_kv =*/ (int32_t) k->ne[2],
+ /*.n_kv =*/ (int32_t) (r ? r->ne[0] : k->ne[2]),
/*.n_batch =*/ (int32_t) q->ne[2],
/*.mask_ne3 =*/ (int32_t) m->ne[3],
/*.nb1 =*/ op->nb[1],
@@ -1409,6 +1430,7 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
/*.nbw3 =*/ w->nb[3],
/*.nbm1 =*/ m->nb[1],
/*.nbm3 =*/ m->nb[3],
+ /*.has_rows =*/ r ? 1 : 0,
};
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q), 1);
@@ -1416,6 +1438,7 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w), 3);
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m), 4);
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
+ ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(r ? r : op), 6);
const int nsg = OP_LIGHTNING_INDEXER_NSG;
const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg;
@@ -1425,16 +1448,97 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
ggml_metal_encoder_set_pipeline(enc, pipeline);
ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
ggml_metal_encoder_dispatch_threadgroups(enc,
- (k->ne[2] + nkptg - 1)/nkptg,
+ (args.n_kv + nkptg - 1)/nkptg,
(q->ne[2] + nbptg - 1)/nbptg,
q->ne[3], 32, nsg, 1);
return 1;
}
+static bool ggml_metal_op_hy4_mla_ok(const ggml_tensor * op);
+static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op);
+
+static const ggml_tensor * ggml_metal_op_view_base(const ggml_tensor * t) {
+ while (t && t->view_src) {
+ t = t->view_src;
+ }
+ return t;
+}
+
+// hy4: the single-token indexer selection (named indexer_sel by the model) feeds one flash attention, which can build
+// the list of selected cells from the top-k picks itself (kernel_hy4_mla_decode_v2 mode 4). True when the flash
+// attention fa reads the mask tkm that way.
+static bool ggml_metal_op_hy4_mode4_ok(const ggml_tensor * fa, const ggml_tensor * tkm) {
+ static const bool off = getenv("HY4_MLA_DISABLE") != nullptr;
+ if (off || fa->op != GGML_OP_FLASH_ATTN_EXT || tkm->op != GGML_OP_TOP_K_MASK ||
+ strncmp(tkm->name, "indexer_sel", 11) != 0 || !fa->src[3] ||
+ tkm->ne[1] != 1 || tkm->ne[2] != 1 || tkm->ne[3] != 1 || !tkm->src[2] || !tkm->src[3]) {
+ return false;
+ }
+
+ // the mask must be the selection itself through views only (an op writing into its storage, e.g. a copy into
+ // it or an in-place add, also flattens to it as view_src, but changes what the attention must see)
+ for (const ggml_tensor * t = fa->src[3]; t != tkm; t = t->src[0]) {
+ if (!t || (t->op != GGML_OP_RESHAPE && t->op != GGML_OP_VIEW)) {
+ return false;
+ }
+ }
+
+ // the picks must be extra sources of the attention (kept alive until it, see hy-v4-flash.cpp), the keep row a
+ // zero fill (it is not read), and the mask view must be the whole mask. The picked cells must be distinct (a
+ // repeated cell would count twice in the attention, once in the mask): hy4's pools and tail are disjoint and
+ // padding points past the cache.
+ const ggml_tensor * keep = tkm->src[0];
+ const int64_t n_list = tkm->src[1]->ne[0]*tkm->src[3]->ne[0] + (tkm->src[4] ? tkm->src[4]->ne[0] : 0);
+
+ return fa->src[6] == tkm->src[1] && fa->src[7] == tkm->src[2] && fa->src[8] == tkm->src[3] && fa->src[9] == tkm->src[4] &&
+ keep->op == GGML_OP_FILL && ggml_get_op_params_f32(keep, 0) == 0.0f &&
+ fa->src[3]->data == tkm->data && fa->src[3]->ne[0] == tkm->ne[0] && tkm->ne[0] <= fa->src[1]->ne[1] && n_list <= 4096 &&
+ ggml_metal_op_flash_attn_ext_n_kv_max_sparse(fa) > 0 && ggml_metal_op_hy4_mla_ok(fa);
+}
+
+static int ggml_metal_op_top_k_mask_impl(ggml_metal_op_t ctx, const ggml_tensor * op);
+
int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) {
- ggml_tensor * op = ctx->node(idx);
+ const ggml_tensor * op = ctx->node(idx);
+
+ // a selection that its flash attention reads through the picks is not built at all. Deferred only when the
+ // mask (through a chain of views) has that attention in this graph as its only reader, and is not an output
+ // (a graph split for an eval callback ends the graph at the mask, so the attention is not found). The use
+ // counts are those ggml_build_forward_expand computed: a graph edited afterwards (ggml_graph_add_node, sources
+ // retargeted) must not depend on this.
+ if (op->op == GGML_OP_TOP_K_MASK && strncmp(op->name, "indexer_sel", 11) == 0) {
+ const ggml_cgraph * gf = ctx->graph();
+ const ggml_tensor * head = op;
+ int i_head = ctx->graph_idx(idx);
+ for (int i = i_head + 1; i < gf->n_nodes; ++i) {
+ const ggml_tensor * node = gf->nodes[i];
+ bool reads = false;
+ for (int j = 0; j < GGML_MAX_SRC && !reads; ++j) {
+ reads = node->src[j] == head;
+ }
+ if (!reads) {
+ continue;
+ }
+ if ((head->flags & GGML_TENSOR_FLAG_OUTPUT) || ggml_node_get_use_count(gf, i_head) != 1) {
+ break;
+ }
+ if ((node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW) && node->src[0] == head) {
+ head = node;
+ i_head = i;
+ continue;
+ }
+ if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) && ggml_metal_op_hy4_mode4_ok(node, op)) {
+ return 1;
+ }
+ break;
+ }
+ }
+ return ggml_metal_op_top_k_mask_impl(ctx, op);
+}
+
+static int ggml_metal_op_top_k_mask_impl(ggml_metal_op_t ctx, const ggml_tensor * op) {
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
@@ -1550,9 +1654,191 @@ int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) {
return 1;
}
+// a fused kernel that replaces a rms_norm reduces in its order only with its thread count; a pipeline that could
+// not be created disables the fusion as well
+static bool ggml_metal_pipeline_nth_eq(
+ const ggml_metal_pipeline_with_params & fused, const ggml_metal_pipeline_with_params & unfused) {
+ return fused.pipeline && unfused.pipeline &&
+ std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(fused)) ==
+ std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(unfused));
+}
+
+// DSV4_HC_PRE + RMS_NORM (+ MUL), see ggml_metal_fusion_check_hc_pre_rms
+// n_gate == 4: the hc gate (MUL + ADD + SIGMOID + SCALE, see ggml_metal_op_hc_gate_fused) comes first and the
+// weights are computed in the kernel (ggml_metal_fusion_check_hc_gate_pre_rms)
+static int ggml_metal_op_hy4_hc_pre_rms(ggml_metal_op_t ctx, int idx, int n_fuse, int n_gate = 0) {
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ const ggml_tensor * pre = ctx->node(idx + n_gate);
+ const ggml_tensor * rms = ctx->node(idx + n_gate + 1);
+ const ggml_tensor * mul = n_fuse - n_gate > 2 ? ctx->node(idx + n_gate + 2) : nullptr;
+ ggml_tensor * dst = ctx->node(idx + n_fuse - 1);
+
+ const ggml_tensor * x = pre->src[0];
+ const ggml_tensor * w = pre->src[1];
+
+ // the gate: s1*(s0*sigmoid(mix*f + b) + c0) + c1 with one SCALE
+ const ggml_tensor * gmix = nullptr;
+ const ggml_tensor * gf = nullptr;
+ const ggml_tensor * gb = nullptr;
+ float s0 = 1.0f, c0 = 0.0f;
+ if (n_gate) {
+ GGML_ASSERT(n_gate == 4 && mul);
+ const ggml_tensor * gmul = ctx->node(idx);
+ const ggml_tensor * gadd = ctx->node(idx + 1);
+ const ggml_tensor * gsc = ctx->node(idx + 3);
+ gmix = gmul->src[0];
+ gf = gmul->src[1];
+ gb = gadd->src[0] == gmul ? gadd->src[1] : gadd->src[0];
+ s0 = ggml_get_op_params_f32(gsc, 0);
+ c0 = ggml_get_op_params_f32(gsc, 1);
+ }
+
+ for (int i = 1; i < n_fuse; ++i) {
+ if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) {
+ ggml_metal_op_concurrency_reset(ctx);
+ break;
+ }
+ }
+
+ float eps;
+ memcpy(&eps, rms->op_params, sizeof(float));
+
+ ggml_metal_kargs_hy4_hc_pre_rms args = {
+ /*.hc =*/ {
+ /*.n_embd =*/ (int32_t) x->ne[0],
+ /*.n_tokens =*/ (int32_t) x->ne[2],
+ /*.nb_x0 =*/ x->nb[0],
+ /*.nb_x1 =*/ x->nb[1],
+ /*.nb_x2 =*/ x->nb[2],
+ /*.nb_w0 =*/ w->nb[0],
+ /*.nb_w1 =*/ w->nb[1],
+ /*.nb_w2 =*/ w->nb[2],
+ /*.nb_d0 =*/ pre->nb[0],
+ /*.nb_d1 =*/ pre->nb[1],
+ /*.scale =*/ ggml_get_op_params_f32(pre, 0),
+ },
+ /*.ne00_t =*/ (int32_t) x->ne[0]/4,
+ /*.eps =*/ eps,
+ /*.nb_y1 =*/ dst->nb[1],
+ /*.nb_m1 =*/ gmix ? gmix->nb[1] : 0,
+ /*.s0 =*/ s0,
+ /*.c0 =*/ c0,
+ /*.s1 =*/ 1.0f,
+ /*.c1 =*/ 0.0f,
+ };
+
+ auto pipeline = ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, pre, mul != nullptr, n_gate != 0);
+
+ // the thread count of ggml_metal_op_norm, the sum depends on it
+ int nth = 32;
+ while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+ nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+ nth = std::min(nth, (args.ne00_t + 31)/32*32);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gmix ? gmix : w), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul ? mul->src[1] : x), 3);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gf ? gf : x), 5);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gb ? gb : x), 6);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
+ ggml_metal_encoder_dispatch_threadgroups(enc, args.hc.n_tokens, 1, 1, nth, 1, 1);
+
+ return n_fuse;
+}
+
+// DSV4_HC_POST + RESHAPE + RMS_NORM, see ggml_metal_fusion_check_hc_post_rms
+static int ggml_metal_op_hy4_hc_post_rms(ggml_metal_op_t ctx, int idx, int n_fuse) {
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ GGML_ASSERT(n_fuse == 2);
+
+ ggml_tensor * post = ctx->node(idx);
+ ggml_tensor * rms = ctx->node(idx + 1);
+
+ const ggml_tensor * x = post->src[0];
+ const ggml_tensor * r = post->src[1];
+ const ggml_tensor * pw = post->src[2];
+
+ if (!ggml_metal_op_concurrency_check(ctx, rms)) {
+ ggml_metal_op_concurrency_reset(ctx);
+ }
+
+ float eps;
+ memcpy(&eps, rms->op_params, sizeof(float));
+
+ ggml_metal_kargs_hy4_hc_post_rms args = {
+ /*.n_embd =*/ (int32_t) post->ne[0],
+ /*.ne00_t =*/ (int32_t) (rms->ne[0]/4),
+ /*.nb_x0 =*/ x->nb[0],
+ /*.nb_x1 =*/ x->nb[1],
+ /*.nb_r0 =*/ r->nb[0],
+ /*.nb_r1 =*/ r->nb[1],
+ /*.nb_r2 =*/ r->nb[2],
+ /*.nb_p0 =*/ pw->nb[0],
+ /*.nb_p1 =*/ pw->nb[1],
+ /*.nb_d1 =*/ post->nb[1],
+ /*.nb_d2 =*/ post->nb[2],
+ /*.nb_y1 =*/ rms->nb[1],
+ /*.eps =*/ eps,
+ };
+
+ auto pipeline = ggml_metal_library_get_pipeline_hy4_hc_post_rms(ctx->lib);
+
+ // the thread count of ggml_metal_op_norm, the sum depends on it
+ int nth = 32;
+ while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+ nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+ nth = std::min(nth, (args.ne00_t + 31)/32*32);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(r), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(pw), 3);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(post), 4);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(rms), 5);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
+ ggml_metal_encoder_dispatch_threadgroups(enc, rms->ne[1], 1, 1, nth, 1, 1);
+
+ return n_fuse;
+}
+
int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
+ // the fused kernels reduce in the order of the rms_norm kernel only with its thread count (capped at 1024)
+ auto same_nth = [&](const ggml_metal_pipeline_with_params & fused, int idx_rms, int n_norm) {
+ return ggml_metal_pipeline_nth_eq(fused, ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx_rms), n_norm));
+ };
+
+ if (op->op == GGML_OP_DSV4_HC_POST && ctx->use_fusion()) {
+ int n = 1;
+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_POST_RMS &&
+ same_nth(ggml_metal_library_get_pipeline_hy4_hc_post_rms(ctx->lib), idx + 1, 1)) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_hy4_hc_post_rms(ctx, idx, n);
+ }
+ }
+
+ if (op->op == GGML_OP_DSV4_HC_PRE && ctx->use_fusion()) {
+ int n = 1;
+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_PRE_RMS &&
+ same_nth(ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, op, n > 2, false), idx + 1, n - 1)) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_hy4_hc_pre_rms(ctx, idx, n);
+ }
+ }
+
ggml_metal_encoder_t enc = ctx->enc;
auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op);
@@ -3547,9 +3833,140 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const ggml_tensor * op) {
return k_size + v_size;
}
+// hy4 MLA decode (kernel_hy4_mla_decode_v2): one query token over one latent KV head (DK 576, V = the first 512
+// dims of K) for the sparse path (n_kv_max hint), over the compacted list of the visible cells. Every K row is read
+// once per 16 heads. Set HY4_MLA_DISABLE to fall back to the vec kernels.
+static bool ggml_metal_op_hy4_mla_ok(const ggml_tensor * op) {
+ static const bool off = getenv("HY4_MLA_DISABLE") != nullptr;
+ const ggml_tensor * q = op->src[0];
+ const ggml_tensor * k = op->src[1];
+ float max_bias = 0.0f;
+ float softcap = 0.0f;
+ memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));
+ memcpy(&softcap, (const float *) op->op_params + 2, sizeof(float));
+ const ggml_tensor * m = op->src[3];
+ // the kernel reads q as f32 and K rows as f16 with 16-byte loads, and one mask row
+ return !off && q->ne[1] == 1 && q->ne[0] == 576 && op->src[2]->ne[0] == 512 && k->ne[2] == 1 && k->ne[3] == 1 &&
+ q->ne[3] == 1 && q->ne[2] <= 128 && k->type == GGML_TYPE_F16 && max_bias == 0.0f && softcap == 0.0f &&
+ ggml_metal_op_flash_attn_ext_v_is_view_of_k(op) && op->src[2]->data == k->data && op->src[2]->nb[1] == k->nb[1] &&
+ q->nb[0] == 4 && k->nb[0] == 2 &&
+ q->nb[2] % 16 == 0 && k->nb[1] % 16 == 0 && (!m || (m->ne[2] == 1 && m->ne[3] == 1 && m->ne[0] == k->ne[1]));
+}
+
+static void ggml_metal_op_hy4_mla(ggml_metal_op_t ctx, const ggml_tensor * op,
+ ggml_metal_buffer_id bid_list, ggml_metal_buffer_id bid_mask, int32_t n_rows, int32_t n_valid_max, int32_t mask_mode, int32_t n_cells,
+ ggml_metal_buffer_id bid_tmp, ggml_metal_buffer_id bid_dst, float scale, const ggml_tensor * tkm = nullptr) {
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
+
+ const int32_t n_head = op->src[0]->ne[2];
+ const bool has_sinks = op->src[4] != nullptr;
+
+ constexpr int TR = 16; // K rows per tile
+ constexpr int NG = 2; // groups of 8 heads per threadgroup
+
+ // measured on M5 Max: 64 heads x 512 window rows best at 16, 32 heads x 2052 sparse rows at 32
+ // (the tmp buffer, ggml_metal_op_flash_attn_ext_extra_tmp, holds 32 partials per row)
+ const int nwg = n_head >= 64 ? 16 : 32;
+ GGML_ASSERT(nwg <= 32);
+ const int rows_per_wg = GGML_PAD((n_valid_max + nwg - 1)/nwg, TR);
+
+ ggml_metal_kargs_hy4_mla args = {
+ /*.n_rows =*/ n_rows,
+ /*.rows_per_wg =*/ rows_per_wg,
+ /*.n_head =*/ n_head,
+ /*.nwg =*/ nwg,
+ /*.nb_q_head =*/ op->src[0]->nb[2],
+ /*.nb_k_row =*/ op->src[1]->nb[1],
+ /*.scale =*/ scale,
+ /*.has_sinks =*/ has_sinks ? 1 : 0,
+ /*.has_mask =*/ mask_mode,
+ /*.n_cells =*/ n_cells,
+ /*.n_top =*/ tkm ? (int32_t) tkm->src[1]->ne[0] : 0,
+ /*.n_blk =*/ tkm ? (int32_t) tkm->src[3]->ne[0] : 0,
+ /*.n_pick =*/ tkm ? (int32_t) tkm->src[3]->ne[1] : 0,
+ /*.n_extra =*/ tkm && tkm->src[4] ? (int32_t) tkm->src[4]->ne[0] : 0,
+ /*.nbk0 =*/ tkm ? tkm->src[1]->nb[0] : 0,
+ /*.nbs0 =*/ tkm ? tkm->src[2]->nb[0] : 0,
+ /*.nbb0 =*/ tkm ? tkm->src[3]->nb[0] : 0,
+ /*.nbb1 =*/ tkm ? tkm->src[3]->nb[1] : 0,
+ /*.nbe0 =*/ tkm && tkm->src[4] ? tkm->src[4]->nb[0] : 0,
+ };
+
+ const char * kname = "kernel_hy4_mla_decode_v2_ng2";
+ ggml_metal_pipeline_with_params ppl = ggml_metal_library_get_pipeline(lib, kname);
+ if (!ppl.pipeline) {
+ ppl = ggml_metal_library_compile_pipeline(lib, kname, kname, nullptr);
+ }
+ GGML_ASSERT(32*4*NG <= ggml_metal_pipeline_max_theads_per_threadgroup(ppl));
+
+ // K tile [16][576] halfs, partial S [NG][4][8][16] floats, P [NG][8][16] halfs, diag [NG][64], M, S, row ids
+ // + the mode 3 list [1024] and simdgroup counts [32]
+ const size_t smem = GGML_PAD(2*(TR*576) + 4*(NG*4*8*TR) + 2*(NG*8*TR) + 4*(NG*64 + 2*8*NG) + 4*TR + 4*4 + 4*TR + 4*(1024 + 32), 16);
+ GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);
+
+ ggml_metal_buffer_id bid_q = ggml_metal_get_buffer_id(op->src[0]);
+ ggml_metal_buffer_id bid_k = ggml_metal_get_buffer_id(op->src[1]);
+ ggml_metal_buffer_id bid_sinks = has_sinks ? ggml_metal_get_buffer_id(op->src[4]) : bid_q;
+
+ ggml_metal_encoder_set_pipeline(enc, ppl);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, bid_q, 1);
+ ggml_metal_encoder_set_buffer (enc, bid_k, 2);
+ ggml_metal_encoder_set_buffer (enc, bid_list, 3);
+ ggml_metal_encoder_set_buffer (enc, bid_sinks, 4);
+ ggml_metal_encoder_set_buffer (enc, bid_tmp, 5);
+ ggml_metal_encoder_set_buffer (enc, bid_mask, 6);
+ ggml_metal_encoder_set_buffer (enc, tkm ? ggml_metal_get_buffer_id(tkm->src[2]) : bid_mask, 7);
+ ggml_metal_encoder_set_buffer (enc, tkm ? ggml_metal_get_buffer_id(tkm->src[3]) : bid_mask, 8);
+ ggml_metal_encoder_set_buffer (enc, tkm && tkm->src[4] ? ggml_metal_get_buffer_id(tkm->src[4]) : bid_mask, 9);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
+ ggml_metal_encoder_dispatch_threadgroups(enc, nwg, (n_head + 8*NG - 1)/(8*NG), 1, 32, 4*NG, 1);
+
+ ggml_metal_op_concurrency_reset(ctx);
+
+ const char * rname = "kernel_hy4_mla_reduce";
+ ggml_metal_pipeline_with_params ppl_r = ggml_metal_library_get_pipeline(lib, rname);
+ if (!ppl_r.pipeline) {
+ ppl_r = ggml_metal_library_compile_pipeline(lib, rname, rname, nullptr);
+ }
+
+ ggml_metal_encoder_set_pipeline(enc, ppl_r);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
+ ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
+ ggml_metal_encoder_dispatch_threadgroups(enc, n_head, 1, 1, 128, 1, 1);
+}
+
int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
+ // hy4 sparse decode: the selected cells come from the indexer picks (see ggml_metal_op_hy4_mode4_ok). The mask
+ // was not built when this op is its reader; if it was (another reader came first), it is simply not used.
+ {
+ const ggml_tensor * base = ggml_metal_op_view_base(op->src[3]);
+ if (base && ggml_metal_op_hy4_mode4_ok(op, base)) {
+ // the picks are tracked as extra sources, the deferred mask's own inputs are not: serialize
+ ggml_metal_op_concurrency_reset(ctx);
+ {
+ const int n_list = (int) (base->src[1]->ne[0]*base->src[3]->ne[0] + (base->src[4] ? base->src[4]->ne[0] : 0));
+
+ ggml_metal_buffer_id bid_tmp = ggml_metal_get_buffer_id(op);
+ bid_tmp.offs += ggml_nbytes(op) + ggml_metal_op_flash_attn_ext_extra_pad(op) + ggml_metal_op_flash_attn_ext_extra_blk(op);
+
+ float scale_mla;
+ memcpy(&scale_mla, (const float *) op->op_params + 0, sizeof(float));
+
+ ggml_metal_op_hy4_mla(ctx, op, ggml_metal_get_buffer_id(base->src[1]), ggml_metal_get_buffer_id(op->src[3]),
+ n_list, n_list, 4, (int32_t) base->ne[0], bid_tmp, ggml_metal_get_buffer_id(op), scale_mla, base);
+ ggml_metal_op_concurrency_reset(ctx);
+ return 1;
+ }
+ }
+ }
+
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
@@ -4105,6 +4522,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
const bool has_kvpad = !use_sparse && ne11_attn % ncpsg != 0;
+ // a short mask (e.g. the SWA cache) is compacted inside the hy4 MLA kernel, no separate pass
+ if (use_sparse && has_mask && ne30 <= 8192 && n_kv_max_sparse <= 1024 && ggml_metal_op_hy4_mla_ok(op)) {
+ ggml_metal_op_hy4_mla(ctx, op, bid_idx, bid_src3, n_kv_max_sparse, n_kv_max_sparse, 3, ne30, bid_tmp, bid_dst, scale);
+ return 1;
+ }
+
if (use_sparse) {
assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0);
@@ -4221,7 +4644,11 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
}
}
} else {
- nwg = 32;
+ // a short KV view (e.g. a sliding window) is cheaper scanned by one workgroup that writes
+ // dst directly than split 32 ways with a reduce pass (hy_v4_flash decode +3%, 64 heads at
+ // <= 1k rows); with few heads one workgroup per head would leave most of the GPU idle
+ const int64_t n_tg = ne02*ne03*((ne01 + nqptg - 1)/nqptg);
+ nwg = ne11_attn <= 1024 && n_tg >= 32 ? 1 : 32;
nsg = 1;
while (2*nwg*nsg*ncpsg < ne11_attn && nsg < 4) {
nsg *= 2;
@@ -4273,6 +4700,11 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.n_kv_max_padded =*/ n_kv_max_padded,
};
+ if (use_sparse && ggml_metal_op_hy4_mla_ok(op)) {
+ ggml_metal_op_hy4_mla(ctx, op, bid_idx, bid_src3, n_kv_max_padded, n_kv_max_sparse, has_mask ? 2 : 0, ne30, bid_tmp, bid_dst, scale);
+ return 1;
+ }
+
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, has_rows, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);
GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
@@ -4338,6 +4770,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
return 1;
}
+static int ggml_metal_op_hy4_add_rms(ggml_metal_op_t ctx, int idx);
+
+
int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
int n_fuse = 1;
const ggml_metal_fusion * fusion = nullptr;
@@ -4349,6 +4784,37 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
n_fuse = n;
+ // the residual sum computed inside RMS_NORM + MUL (same thread count as rms_norm)
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ADD_RMS) {
+ if (ggml_metal_pipeline_nth_eq(ggml_metal_library_get_pipeline_hy4_add_rms(ctx->lib),
+ ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx + 1), 2))) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_hy4_add_rms(ctx, idx);
+ }
+ // encode this ADD alone; the following nodes match again on their own
+ fusion = nullptr;
+ n_fuse = 1;
+ }
+
+ // the hyper-connection gate computed inside DSV4_HC_PRE + RMS_NORM + MUL (same thread count as rms_norm)
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_GATE_PRE_RMS) {
+ const ggml_tensor * pre = ctx->node(idx + 4);
+ if (ggml_metal_pipeline_nth_eq(ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, pre, true, true),
+ ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx + 5), 2))) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_hy4_hc_pre_rms(ctx, idx, n_fuse, 4);
+ }
+ // encode this MUL alone; the following nodes match again on their own (the gate alone was not checked)
+ fusion = nullptr;
+ n_fuse = 1;
+ }
+
+ // hyper-connection gate: mul -> add -> sigmoid -> scale (-> scale)
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_GATE) {
+ ctx->count_fusions(fusion);
+ return ggml_metal_op_hc_gate_fused(ctx, idx, n_fuse);
+ }
+
// snake activation autofuse: mul -> sin -> sqr -> mul -> add
if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SNAKE) {
ctx->count_fusions(fusion);
@@ -4628,6 +5094,59 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) {
return 1;
}
+// ADD + RMS_NORM + MUL, see ggml_metal_fusion_check_add_rms
+static int ggml_metal_op_hy4_add_rms(ggml_metal_op_t ctx, int idx) {
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ const ggml_tensor * add = ctx->node(idx);
+ const ggml_tensor * rms = ctx->node(idx + 1);
+ ggml_tensor * dst = ctx->node(idx + 2);
+
+ const ggml_tensor * a = add->src[0];
+ const ggml_tensor * b = add->src[1];
+ const ggml_tensor * w = dst->src[1];
+
+ for (int i = 1; i < 3; ++i) {
+ if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) {
+ ggml_metal_op_concurrency_reset(ctx);
+ break;
+ }
+ }
+
+ float eps;
+ memcpy(&eps, rms->op_params, sizeof(float));
+
+ ggml_metal_kargs_hy4_add_rms args = {
+ /*.ne00 =*/ (int32_t) add->ne[0],
+ /*.ne00_t =*/ (int32_t) add->ne[0]/4,
+ /*.nb_a1 =*/ a->nb[1],
+ /*.nb_b1 =*/ b->nb[1],
+ /*.nb_y1 =*/ dst->nb[1],
+ /*.eps =*/ eps,
+ };
+
+ auto pipeline = ggml_metal_library_get_pipeline_hy4_add_rms(ctx->lib);
+
+ // the thread count of ggml_metal_op_norm, the sum depends on it
+ int nth = 32;
+ while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+ nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+ nth = std::min(nth, (args.ne00_t + 31)/32*32);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(a), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(b), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(w), 3);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
+ ggml_metal_encoder_dispatch_threadgroups(enc, ggml_nrows(add), 1, 1, nth, 1, 1);
+
+ return 3;
+}
+
int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
@@ -4768,6 +5287,20 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) {
ggml_metal_library_t lib = ctx->lib;
ggml_metal_encoder_t enc = ctx->enc;
+ // ROPE + CONCAT: the rotation goes straight into the concatenation (see ggml_metal_fusion_check_rope_concat)
+ ggml_tensor * cat = nullptr;
+ if (ctx->use_fusion()) {
+ int n = 1;
+ const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n);
+ if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ROPE_CONCAT) {
+ ctx->count_fusions(fusion);
+ cat = ctx->node(idx + 1);
+ if (!ggml_metal_op_concurrency_check(ctx, cat)) {
+ ggml_metal_op_concurrency_reset(ctx);
+ }
+ }
+ }
+
GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
@@ -4810,7 +5343,7 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) {
const int n_offs = ((const int32_t *) op->op_params)[15];
// when dst aliases src0, the channels outside the rotated window already hold the correct data
- const bool inplace = op->data == op->src[0]->data;
+ const bool inplace = !cat && op->data == op->src[0]->data;
ggml_metal_kargs_rope args = {
/*.ne00 =*/ ne00,
@@ -4847,6 +5380,42 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) {
/* inplace =*/ inplace,
};
+ if (cat) {
+ const ggml_tensor * a = cat->src[0];
+
+ // the rotation is written with the strides of the concatenation
+ args.nb0 = cat->nb[0];
+ args.nb1 = cat->nb[1];
+ args.nb2 = cat->nb[2];
+ args.nb3 = cat->nb[3];
+
+ ggml_metal_kargs_concat cc = {};
+ cc.ne00 = (int32_t) a->ne[0];
+ cc.nb00 = a->nb[0];
+ cc.nb01 = a->nb[1];
+ cc.nb02 = a->nb[2];
+ cc.nb03 = a->nb[3];
+ cc.nb0 = cat->nb[0];
+ cc.nb1 = cat->nb[1];
+ cc.nb2 = cat->nb[2];
+ cc.nb3 = cat->nb[3];
+
+ auto ppl = ggml_metal_library_get_pipeline_hy4_rope_concat(lib);
+
+ ggml_metal_encoder_set_pipeline(enc, ppl);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_bytes (enc, &cc, sizeof(cc), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 3);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2] ? op->src[2] : op->src[0]), 4);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(a), 5);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(cat), 6);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);
+
+ return 2;
+ }
+
auto pipeline = ggml_metal_library_get_pipeline_rope(lib, op);
ggml_metal_encoder_set_pipeline(enc, pipeline);
@@ -5260,6 +5829,86 @@ int ggml_metal_op_col2im_1d(ggml_metal_op_t ctx, int idx) {
return 1;
}
+// Dispatch the fused hyper-connection gate from the matched mul -> add -> sigmoid -> scale (-> scale) chain.
+int ggml_metal_op_hc_gate_fused(ggml_metal_op_t ctx, int idx, int n_fuse) {
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ GGML_ASSERT(n_fuse == 4 || n_fuse == 5);
+
+ const ggml_tensor * mul = ctx->node(idx + 0);
+ const ggml_tensor * add = ctx->node(idx + 1);
+ const ggml_tensor * sc0 = ctx->node(idx + 3);
+ const ggml_tensor * sc1 = n_fuse == 5 ? ctx->node(idx + 4) : nullptr;
+ ggml_tensor * dst = ctx->node(idx + n_fuse - 1);
+
+ const ggml_tensor * x = mul->src[0];
+ const ggml_tensor * f = mul->src[1];
+ const ggml_tensor * b = add->src[0] == mul ? add->src[1] : add->src[0];
+
+ for (int i = 1; i < n_fuse; ++i) {
+ if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) {
+ ggml_metal_op_concurrency_reset(ctx);
+
+ break;
+ }
+ }
+
+ ggml_metal_kargs_hc_gate args = {
+ /*.n =*/ (int32_t) x->ne[0],
+ /*.n_rows =*/ (int32_t) x->ne[1],
+ /*.nb_x1 =*/ x->nb[1],
+ /*.s0 =*/ ggml_get_op_params_f32(sc0, 0),
+ /*.c0 =*/ ggml_get_op_params_f32(sc0, 1),
+ /*.s1 =*/ sc1 ? ggml_get_op_params_f32(sc1, 0) : 1.0f,
+ /*.c1 =*/ sc1 ? ggml_get_op_params_f32(sc1, 1) : 0.0f,
+ };
+
+ auto pipeline = ggml_metal_library_get_pipeline_hc_gate(lib);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(f), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(b), 3);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4);
+
+ const int total = args.n * args.n_rows;
+ const int nth = std::min(256, std::max(32, total));
+ ggml_metal_encoder_dispatch_threadgroups(enc, (total + nth - 1)/nth, 1, 1, nth, 1, 1);
+
+ return n_fuse;
+}
+
+// x * sigmoid(g): idx is the sigmoid, idx + 1 the mul. The caller has validated the pair.
+int ggml_metal_op_sigmoid_mul_fused(ggml_metal_op_t ctx, int idx) {
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ const ggml_tensor * sig = ctx->node(idx + 0);
+ ggml_tensor * mul = ctx->node(idx + 1);
+
+ if (!ggml_metal_op_concurrency_check(ctx, mul)) {
+ ggml_metal_op_concurrency_reset(ctx);
+ }
+
+ int32_t n = (int32_t) ggml_nelements(mul);
+
+ auto pipeline = ggml_metal_library_get_pipeline_sigmoid_mul(lib);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &n, sizeof(n), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(sig->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul->src[0]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul), 3);
+
+ const int nt = (n + 3)/4;
+ const int nth = std::min(256, std::max(32, nt));
+ ggml_metal_encoder_dispatch_threadgroups(enc, (nt + nth - 1)/nth, 1, 1, nth, 1, 1);
+
+ return 2;
+}
+
// Dispatch the fused snake kernel from the matched mul -> sin -> sqr -> mul -> add chain.
// idx points at the leading mul. The caller has validated the chain.
int ggml_metal_op_snake_fused(ggml_metal_op_t ctx, int idx) {
@@ -5978,6 +6627,50 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) {
GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
+ // the 11-bit variant (3 histogram passes, parallel bucket select, single emit pass); HY4_TOPK_R8 for the old one
+ static const bool r8 = getenv("HY4_TOPK_R8") != nullptr;
+ if (!r8 && op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32) {
+ const char * kname = "kernel_top_k_f32_i32_r11";
+ ggml_metal_pipeline_with_params ppl = ggml_metal_library_get_pipeline(lib, kname);
+ if (!ppl.pipeline) {
+ ppl = ggml_metal_library_compile_pipeline(lib, kname, kname, nullptr);
+ }
+
+ // its prefix sums assume 32-wide SIMD groups (at most 32 of them)
+ if (!ppl.pipeline || ggml_metal_pipeline_simd_width(ppl) != 32) {
+ goto r8_fallback;
+ }
+
+ int nth = 32; // power of two, the kernel splits the buckets evenly over the threads
+ while (2*nth <= std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(ppl))) {
+ nth *= 2;
+ }
+
+ ggml_metal_kargs_top_k args = {
+ /*.ne00 =*/ ne00,
+ /*.ne01 =*/ ne01,
+ /*.ne02 =*/ ne02,
+ /*.ne03 =*/ ne03,
+ /*.nb01 =*/ nb01,
+ /*.nb02 =*/ nb02,
+ /*.nb03 =*/ nb03,
+ /*.top_k =*/ (int32_t) op->ne[0],
+ };
+
+ ggml_metal_encoder_set_pipeline(enc, ppl);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);
+
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, 2048*sizeof(uint32_t), 0);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD(34*sizeof(uint32_t), 16), 1);
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD( 2*sizeof(uint32_t), 16), 2);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);
+ return;
+ }
+r8_fallback:
+
auto pipeline = ggml_metal_library_get_pipeline_top_k_radix(lib, op);
// one workgroup per row; radix-select the k-th largest value
@@ -6126,7 +6819,7 @@ int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) {
const int k = op->ne[0];
const int nrows = ggml_nrows(op->src[0]);
- const bool use_radix =
+ const bool use_radix = k > 0 &&
ncols > 2048 && (k > 64 || (nrows > 4 && ncols >= 8192));
if (use_radix) {
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h
index 47889fa84..4078cf77e 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.h
+++ b/ggml/src/ggml-metal/ggml-metal-ops.h
@@ -92,6 +92,8 @@ int ggml_metal_op_conv_transpose_1d (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_conv_transpose_2d (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_col2im_1d (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_snake_fused (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_hc_gate_fused (ggml_metal_op_t ctx, int idx, int n_fuse);
+int ggml_metal_op_sigmoid_mul_fused (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_upscale (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_pad (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_pad_reflect_1d (ggml_metal_op_t ctx, int idx);
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index 9bed6421b..4e247792b 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -9,6 +9,7 @@
#include "ggml-metal-ops.h"
#include "ggml-metal-tuning.h"
+#include <cstdlib>
#include <mutex>
#include <string>
@@ -657,6 +658,23 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
ggml_metal_set_n_cb(ctx, n_cb);
}
+// hy4: number of extra command-buffer encoder threads, GGML_METAL_N_CB overrides the default of 1
+// capped below GGML_METAL_MAX_COMMAND_BUFFERS (the free path only releases that many); note that with an
+// abort callback set, n_cb >= 2 leaves the main command buffer uncommitted (ggml-metal-context.m)
+static int ggml_backend_metal_default_n_cb() {
+ const char * env = getenv("GGML_METAL_N_CB");
+ if (env == nullptr || *env == '\0') {
+ return 1;
+ }
+ char * end = nullptr;
+ const long v = strtol(env, &end, 10);
+ if (*end != '\0' || v < 0 || v > 7) { // GGML_METAL_MAX_COMMAND_BUFFERS - 1
+ GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_CB='%s'\n", __func__, env);
+ return 1;
+ }
+ return (int) v;
+}
+
static ggml_backend_i ggml_backend_metal_i = {
/* .get_name = */ ggml_backend_metal_name,
/* .free = */ ggml_backend_metal_free,
@@ -700,7 +718,7 @@ ggml_backend_t ggml_backend_metal_init(void) {
/* .context = */ ctx,
};
- ggml_backend_metal_set_n_cb(backend, 1);
+ ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
return backend;
}
@@ -795,7 +813,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
/* .context = */ ctx,
};
- ggml_backend_metal_set_n_cb(backend, 1);
+ ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
return backend;
diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal
index 5231b8395..bb275b88c 100644
--- a/ggml/src/ggml-metal/kernels/argsort.metal
+++ b/ggml/src/ggml-metal/kernels/argsort.metal
@@ -342,6 +342,125 @@ kernel void kernel_top_k_f32_i32(
}
}
+// radix-select with 11-bit digits (3 histogram passes over the row instead of 4), the bucket holding the k-th value
+// found by a threadgroup prefix sum instead of a serial scan, and the strict winners and the ties written in one
+// pass. Same result contract as kernel_top_k_f32_i32 (unordered indices, ties in any order).
+kernel void kernel_top_k_f32_i32_r11(
+ constant ggml_metal_kargs_top_k & args,
+ device const char * src0,
+ device int32_t * dst,
+ threadgroup atomic_uint * histo [[threadgroup(0)]], // 2048
+ threadgroup uint * sh [[threadgroup(1)]], // 32 simdgroup sums + bucket + above
+ threadgroup atomic_uint * cnt [[threadgroup(2)]], // above / ties counters
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort3 tpitg[[thread_position_in_threadgroup]],
+ ushort3 ntg[[threads_per_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ const uint ncols = args.ne00;
+ const uint top_k = args.top_k;
+ const uint i01 = tgpig[0];
+ const uint i02 = tgpig[1];
+ const uint i03 = tgpig[2];
+
+ device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03);
+
+ device int32_t * dst_row = dst + top_k*(i01 + args.ne01*i02 + args.ne01*args.ne02*i03);
+
+ const uint tid = tpitg.x;
+ const uint nt = ntg.x; // power of two, <= 1024
+ const uint nsg = (nt + 31)/32;
+
+ uint prefix = 0; // fixed high bits of the threshold key
+ uint desired = top_k; // count still needed from the candidate range
+
+ for (int p = 0; p < 3; ++p) {
+ const uint shift = p == 0 ? 21u : p == 1 ? 10u : 0u;
+ const uint nbits = p == 2 ? 10u : 11u;
+ const uint nb = 1u << nbits;
+
+ for (uint i = tid; i < nb; i += nt) {
+ atomic_store_explicit(&histo[i], 0u, memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const uint hi_mask = p == 0 ? 0u : (0xFFFFFFFFu << (shift + nbits));
+ const uint prefix_hi = prefix & hi_mask;
+
+ for (uint i = tid; i < ncols; i += nt) {
+ const uint key = ggml_top_k_f2ui(src0_row[i]);
+ if ((key & hi_mask) == prefix_hi) {
+ atomic_fetch_add_explicit(&histo[(key >> shift) & (nb - 1)], 1u, memory_order_relaxed);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // buckets in descending order: thread t owns [nb - (t + 1)*per, nb - t*per)
+ const uint per = nb >= nt ? nb/nt : 1u;
+ const uint hi = tid*per < nb ? nb - tid*per : 0u; // exclusive top bucket of this thread
+ uint c = 0;
+ for (uint j = 0; j < per && hi > j; ++j) {
+ c += atomic_load_explicit(&histo[hi - 1 - j], memory_order_relaxed);
+ }
+ const uint incl = simd_prefix_inclusive_sum(c);
+ if (tiisg == 31 || tid == nt - 1) {
+ sh[sgitg] = incl;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ uint base = 0;
+ for (uint w = 0; w < sgitg; ++w) {
+ base += sh[w];
+ }
+ const uint my_incl = base + incl;
+ const uint my_excl = my_incl - c;
+ if (c > 0 && my_excl < desired && my_incl >= desired) {
+ uint acc = my_excl;
+ for (uint j = 0; j < per; ++j) {
+ const uint b = hi - 1 - j;
+ const uint h = atomic_load_explicit(&histo[b], memory_order_relaxed);
+ if (acc + h >= desired) {
+ sh[32] = b;
+ sh[33] = acc;
+ break;
+ }
+ acc += h;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ prefix |= sh[32] << shift;
+ desired -= sh[33];
+
+ // every thread has read sh before the next pass reuses it
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // prefix is the full key of the k-th largest value; desired is how many of its ties are needed
+ const uint threshold = prefix;
+ const uint n_above = top_k - desired;
+
+ if (tid == 0) {
+ atomic_store_explicit(&cnt[0], 0u, memory_order_relaxed);
+ atomic_store_explicit(&cnt[1], 0u, memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ for (uint i = tid; i < ncols; i += nt) {
+ const uint key = ggml_top_k_f2ui(src0_row[i]);
+ if (key > threshold) {
+ const uint pos = atomic_fetch_add_explicit(&cnt[0], 1u, memory_order_relaxed);
+ dst_row[pos] = (int32_t) i;
+ } else if (key == threshold) {
+ const uint t = atomic_fetch_add_explicit(&cnt[1], 1u, memory_order_relaxed);
+ if (t < desired) {
+ dst_row[n_above + t] = (int32_t) i;
+ }
+ }
+ }
+}
+
// fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing.
// One SIMDgroup handles one token row; n_expert is limited to 1024 by the host.
kernel void kernel_topk_moe_f32(
@@ -470,10 +589,16 @@ kernel void kernel_moe_reduce_f32(
const int n_expert_used = FC_moe_reduce_n_expert_used;
- const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col;
- float sum = 0.0f;
- FOR_UNROLL (int e = 0; e < n_expert_used; ++e) {
- sum += experts[base + e * args.ne00] * weights[token * n_expert_used + e];
+ // the unfused MUL and ADD chain: every product rounded, then added in expert order (no fused multiply-add)
+ {
+#pragma METAL fp math_mode(safe)
+#pragma METAL fp contract(off)
+ const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col;
+ float sum = experts[base] * weights[token * n_expert_used];
+ FOR_UNROLL (int e = 1; e < n_expert_used; ++e) {
+ const float p = experts[base + (int64_t) e * args.ne00] * weights[token * n_expert_used + e];
+ sum = sum + p;
+ }
+ dst[token * args.ne00 + col] = sum;
}
- dst[token * args.ne00 + col] = sum;
}
diff --git a/ggml/src/ggml-metal/kernels/fa_aux.metal b/ggml/src/ggml-metal/kernels/fa_aux.metal
index 6fe47470f..4b9081276 100644
--- a/ggml/src/ggml-metal/kernels/fa_aux.metal
+++ b/ggml/src/ggml-metal/kernels/fa_aux.metal
@@ -416,6 +416,7 @@ kernel void kernel_lightning_indexer(
device const char * w,
device const char * m,
device char * dst,
+ device const int * rows,
uint3 tgpig[[threadgroup_position_in_grid]],
ushort tiitg[[thread_index_in_threadgroup]],
ushort tiisg[[thread_index_in_simdgroup]],
@@ -448,7 +449,8 @@ kernel void kernel_lightning_indexer(
half4x4 tmp;
if (i_kv_0 + ik < args.n_kv) {
- device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3);
+ const int ir = args.has_rows ? rows[i_kv_0 + ik] : i_kv_0 + ik;
+ device const kd4x4_t * kr = (device const kd4x4_t *) (k + (int64_t) ir*args.nbk2 + i_stream*args.nbk3);
deq_k(kr + i16/nl_k, i16%nl_k, tmp);
} else {
@@ -557,3 +559,367 @@ template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_
template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_0, 2, dequantize_q5_0>;
template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_1, 2, dequantize_q5_1>;
template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q8_0, 2, dequantize_q8_0>;
+
+// merge the nwg partial results of kernel_hy4_mla_decode_v2 (nwg <= 32): one threadgroup per head
+kernel void kernel_hy4_mla_reduce(
+ constant ggml_metal_kargs_hy4_mla & args,
+ device const char * htmp,
+ device char * dst,
+ uint tgpig [[threadgroup_position_in_grid]],
+ ushort tiitg [[thread_index_in_threadgroup]],
+ ushort ntg [[threads_per_threadgroup]]) {
+ constexpr int DV4 = 512/4;
+ const int NWG = args.nwg;
+ const int rid = tgpig;
+
+ device const float * ss = (device const float *) htmp + (int64_t) args.n_head*512*NWG + rid*(2*NWG);
+
+ float m = -FLT_MAX/2;
+ for (int w = 0; w < NWG; ++w) {
+ m = max(m, ss[2*w + 1]);
+ }
+ float S = 0.0f;
+ float ms[32];
+ for (int w = 0; w < NWG; ++w) {
+ ms[w] = exp(ss[2*w + 1] - m);
+ S += ss[2*w + 0]*ms[w];
+ }
+ const float iS = S == 0.0f ? 0.0f : 1.0f/S;
+
+ // partials [head][workgroup][DV]: each workgroup writes its rows contiguously
+ device const float4 * h4 = (device const float4 *) htmp + (int64_t) rid*DV4*NWG;
+ device float4 * d4 = (device float4 *) dst + (int64_t) rid*DV4;
+
+ for (int i = tiitg; i < DV4; i += ntg) {
+ float4 v = 0.0f;
+ for (int w = 0; w < NWG; ++w) {
+ v += h4[w*DV4 + i]*ms[w];
+ }
+ d4[i] = v*iS;
+ }
+}
+
+#define HY4_MLA_LMAX 1024 // longest list mode 3 compacts in threadgroup memory
+
+// hy4 MLA decode: one query token, n_head query heads over one latent KV head (DK 576, V = the first 512 dims of
+// K). The rows come from the compacted list of the visible cells: built by kernel_flash_attn_ext_vec_idx (padded
+// with -1), or, for a short mask, by every threadgroup itself. NG groups of 8 heads per threadgroup share every K tile, so a row is read
+// once per NG*8 heads (the vec kernel reads it once per head), and both products run on 8x8 simdgroup matrices.
+// Simdgroup (g, s) owns head group g and quarter s of DK (for S) and of DV (for O); its Q quarter stays in
+// registers. The unnormalized O, S and M of every workgroup go to kernel_hy4_mla_reduce.
+template<short NG>
+kernel void kernel_hy4_mla_decode_v2(
+ constant ggml_metal_kargs_hy4_mla & args,
+ device const char * q,
+ device const char * k,
+ device const int * idx,
+ device const char * sinks,
+ device char * dst,
+ device const half * mask,
+ device const char * score, // mode 4
+ device const char * blocks, // mode 4
+ device const char * extra, // mode 4
+ threadgroup half * shmem [[threadgroup(0)]],
+ uint3 tgpig [[threadgroup_position_in_grid]],
+ ushort tiitg [[thread_index_in_threadgroup]],
+ ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+ constexpr int DK = 576;
+ constexpr int DV = 512;
+ constexpr int TR = 16; // K rows per tile
+ constexpr int NQ = 4; // quarters of DK/DV per head group
+ constexpr int NT = NG*NQ*32;
+ constexpr int DVS = DV/NQ; // 128 O columns per simdgroup
+ constexpr int KBS = DK/8/NQ; // 18 DK blocks per simdgroup
+ constexpr int HT = NG*8; // heads per threadgroup
+
+ constexpr float M_NONE = -FLT_MAX/2;
+
+ const int iwg = tgpig.x;
+ const int h0 = tgpig.y*HT;
+ const int NWG = args.nwg;
+
+ const short g = sgitg/NQ; // head group of this simdgroup
+ const short qs = sgitg%NQ; // DK/DV quarter of this simdgroup
+
+ threadgroup half * sk = shmem; // [TR][DK] (also Q staging and the O output)
+ threadgroup float * ss = (threadgroup float *) (sk + TR*DK); // [NG][NQ][8][TR] partial S
+ threadgroup half * sp = (threadgroup half *) (ss + NG*NQ*8*TR);// [NG][8][TR] P
+ threadgroup float * sd = (threadgroup float *) (sp + NG*8*TR); // [NG][8][8] diag(ms)
+ threadgroup float * sm = sd + NG*64; // M[HT]
+ threadgroup float * sS = sm + HT; // S[HT]
+ threadgroup int * sr = (threadgroup int *) (sS + HT); // row ids of the tile [TR], -1 = none
+ threadgroup int * sn = sr + TR; // valid rows in the tile
+ threadgroup float * sv = (threadgroup float *) (sn + 4); // mask value of each tile row [TR]
+ threadgroup int * sl = (threadgroup int *) (sv + TR); // mode 3: the compacted list [HY4_MLA_LMAX]
+ threadgroup int * sc = sl + HY4_MLA_LMAX; // mode 3: per-simdgroup counts [32]
+
+ // Q -> registers, staged through sk in halves of 16 heads at most (16*576 halfs == TR*DK)
+ simdgroup_half8x8 mq[KBS];
+ for (short gg = 0; gg < NG; gg += 2) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int nh = min(2, NG - gg)*8;
+ for (int i = tiitg; i < nh*DK/4; i += NT) {
+ const int h = h0 + gg*8 + i/(DK/4);
+ const int c = i%(DK/4);
+ const float4 v = h < args.n_head ? ((device const float4 *) (q + h*args.nb_q_head))[c] : float4(0.0f);
+ ((threadgroup half4 *) sk)[i] = (half4) v;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (g == gg || g == gg + 1) {
+ for (short b = 0; b < KBS; ++b) {
+ simdgroup_load(mq[b], sk + (g - gg)*8*DK + (qs*KBS + b)*8, DK);
+ }
+ }
+ }
+ if (tiitg < HT) {
+ sm[tiitg] = M_NONE;
+ sS[tiitg] = 0.0f;
+ }
+
+ simdgroup_float8x8 mo[DVS/8];
+ for (short i = 0; i < DVS/8; ++i) {
+ mo[i] = make_filled_simdgroup_matrix<float, 8>(0.0f);
+ }
+
+ // mode 3: every threadgroup compacts the finite cells of the (short) mask row itself, in cell order, so all
+ // the workgroups see the same list and no separate compaction dispatch is needed
+ int n_rows = args.n_rows;
+ if (args.has_mask == 3) {
+ const int n = args.n_cells;
+ const int qn = n/NT;
+ const int rn = n%NT;
+ const int c0 = qn*tiitg + min((int) tiitg, rn);
+ const int c1 = c0 + qn + (tiitg < rn ? 1 : 0);
+
+ int cnt = 0;
+ for (int c = c0; c < c1; ++c) {
+ cnt += isfinite((float) mask[c]) ? 1 : 0;
+ }
+ const int incl = simd_prefix_inclusive_sum(cnt);
+ if ((tiitg % 32) == 31) {
+ sc[sgitg] = incl;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ int base = incl - cnt;
+ int total = 0;
+ for (short w = 0; w < NT/32; ++w) {
+ base += w < sgitg ? sc[w] : 0;
+ total += sc[w];
+ }
+ int j = base;
+ for (int c = c0; c < c1 && j < HY4_MLA_LMAX; ++c) {
+ if (isfinite((float) mask[c])) {
+ sl[j++] = c;
+ }
+ }
+ n_rows = min(total, HY4_MLA_LMAX);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // mode 3 splits the list it counted, so cells beyond the host's hint (e.g. repeated positions) are not dropped
+ const int rows_per_wg = args.has_mask == 3 ? ((n_rows + NWG*TR - 1)/(NWG*TR))*TR : args.rows_per_wg;
+
+ const int r0 = iwg*rows_per_wg;
+ const int r1 = min(r0 + rows_per_wg, n_rows);
+
+ // softmax layout: thread t -> group t/128, head (t/16)%8, row t%16
+ const int tg = tiitg/128;
+ const int th = (tiitg/TR)%8;
+ const int tr = tiitg%TR;
+
+ for (int rt = r0; rt < r1; rt += TR) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tiitg < TR) {
+ const int r = rt + tiitg;
+ int row = -1;
+ if (r < r1) {
+ if (args.has_mask == 4) {
+ // the cells TOP_K_MASK would keep: the members of the picked pools with a finite score, then the tail
+ const int n_pool_rows = args.n_top*args.n_blk;
+ int c = -1;
+ if (r < n_pool_rows) {
+ const int p = *(device const int32_t *) ((device const char *) idx + (r/args.n_blk)*args.nbk0);
+ if (p >= 0 && p < args.n_pick && *(device const float *) (score + p*args.nbs0) > -INFINITY) {
+ c = *(device const int32_t *) (blocks + (r%args.n_blk)*args.nbb0 + p*args.nbb1);
+ }
+ } else {
+ c = *(device const int32_t *) (extra + (r - n_pool_rows)*args.nbe0);
+ }
+ row = c >= 0 && c < args.n_cells ? c : -1;
+ } else {
+ row = args.has_mask == 3 ? sl[r] : idx[r];
+ }
+ }
+ // mode 4 keeps a zero-filled row: the value is 0
+ float mv = 0.0f;
+ if (row >= 0 && args.has_mask && args.has_mask != 4) {
+ mv = (float) mask[row];
+ if (!isfinite(mv)) {
+ row = -1;
+ }
+ }
+ sr[tiitg] = row;
+ sv[tiitg] = mv;
+ }
+ if (tiitg == 0) {
+ sn[0] = 0;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tiitg < TR && sr[tiitg] >= 0) {
+ atomic_fetch_add_explicit((threadgroup atomic_int *) sn, 1, memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // a compacted list ends in padding; the mode 4 list can have empty tiles anywhere
+ if (sn[0] == 0) {
+ if (args.has_mask == 4) {
+ continue;
+ }
+ break;
+ }
+
+ for (int i = tiitg; i < TR*DK/8; i += NT) {
+ const int rr = i/(DK/8);
+ const int c = i%(DK/8);
+ const int row = sr[rr];
+ ((threadgroup float4 *) sk)[i] = row >= 0 ? ((device const float4 *) (k + (uint64_t) row*args.nb_k_row))[c] : float4(0.0f);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ simdgroup_float8x8 ms0 = make_filled_simdgroup_matrix<float, 8>(0.0f);
+ simdgroup_float8x8 ms1 = make_filled_simdgroup_matrix<float, 8>(0.0f);
+ for (short b = 0; b < KBS; ++b) {
+ const short kb = qs*KBS + b;
+ simdgroup_half8x8 mk0;
+ simdgroup_half8x8 mk1;
+ simdgroup_load(mk0, sk + 0*DK + kb*8, DK, 0, true);
+ simdgroup_load(mk1, sk + 8*DK + kb*8, DK, 0, true);
+ simdgroup_multiply_accumulate(ms0, mq[b], mk0, ms0);
+ simdgroup_multiply_accumulate(ms1, mq[b], mk1, ms1);
+ }
+ simdgroup_store(ms0, ss + (g*NQ + qs)*8*TR + 0, TR);
+ simdgroup_store(ms1, ss + (g*NQ + qs)*8*TR + 8, TR);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tg < NG) {
+ const bool valid = sr[tr] >= 0;
+ float s = 0.0f;
+ for (short n = 0; n < NQ; ++n) {
+ s += ss[(tg*NQ + n)*8*TR + th*TR + tr];
+ }
+ s = s*args.scale + sv[tr];
+
+ float mx = valid ? s : M_NONE;
+ mx = max(mx, simd_shuffle_xor(mx, 8));
+ mx = max(mx, simd_shuffle_xor(mx, 4));
+ mx = max(mx, simd_shuffle_xor(mx, 2));
+ mx = max(mx, simd_shuffle_xor(mx, 1));
+
+ const int hh = tg*8 + th;
+ const float mold = sm[hh];
+ const float mnew = max(mold, mx);
+ const float p = valid ? exp(s - mnew) : 0.0f;
+ const float msc = exp(mold - mnew);
+
+ float ps = p;
+ ps += simd_shuffle_xor(ps, 8);
+ ps += simd_shuffle_xor(ps, 4);
+ ps += simd_shuffle_xor(ps, 2);
+ ps += simd_shuffle_xor(ps, 1);
+
+ sp[(tg*8 + th)*TR + tr] = (half) p;
+ if (tr < 8) {
+ sd[tg*64 + th*8 + tr] = tr == th ? msc : 0.0f;
+ }
+ // every lane of the head read sm[hh] before lane 0 writes it: the 16 lanes are one simdgroup half
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ if (tr == 0) {
+ sm[hh] = mnew;
+ sS[hh] = sS[hh]*msc + ps;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ simdgroup_float8x8 md;
+ simdgroup_half8x8 mp0;
+ simdgroup_half8x8 mp1;
+ simdgroup_load(md, sd + g*64, 8);
+ simdgroup_load(mp0, sp + g*8*TR + 0, TR);
+ simdgroup_load(mp1, sp + g*8*TR + 8, TR);
+ for (short i = 0; i < DVS/8; ++i) {
+ const short d0 = qs*DVS + i*8;
+ simdgroup_half8x8 mv0;
+ simdgroup_half8x8 mv1;
+ simdgroup_load(mv0, sk + 0*DK + d0, DK);
+ simdgroup_load(mv1, sk + 8*DK + d0, DK);
+ simdgroup_multiply(mo[i], md, mo[i]);
+ simdgroup_multiply_accumulate(mo[i], mp0, mv0, mo[i]);
+ simdgroup_multiply_accumulate(mo[i], mp1, mv1, mo[i]);
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (args.has_sinks && iwg == 0) {
+ const int hh = tiitg/8;
+ const int c = tiitg%8;
+ float mn = 0.0f;
+ float msc = 0.0f;
+ float vs = 0.0f;
+ if (tiitg < HT*8) {
+ const float m = sm[hh];
+ const float snk = h0 + hh < args.n_head ? ((device const float *) sinks)[h0 + hh] : M_NONE;
+ mn = max(m, snk);
+ msc = exp(m - mn);
+ vs = h0 + hh < args.n_head ? exp(snk - mn) : 0.0f;
+ sd[(hh/8)*64 + (hh%8)*8 + c] = c == hh%8 ? msc : 0.0f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tiitg < HT*8 && c == 0) {
+ sm[hh] = mn;
+ sS[hh] = sS[hh]*msc + vs;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ simdgroup_float8x8 md;
+ simdgroup_load(md, sd + g*64, 8);
+ for (short i = 0; i < DVS/8; ++i) {
+ simdgroup_multiply(mo[i], md, mo[i]);
+ }
+ }
+
+ device float4 * dst4 = (device float4 *) dst;
+ device float * dst1 = (device float *) dst + (int64_t) args.n_head*DV*NWG;
+
+ // O out through sk, one head group at a time ([8][DV] floats fit in the [TR][DK] halfs)
+ for (short gg = 0; gg < NG; ++gg) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ threadgroup float * so = (threadgroup float *) sk;
+ if (g == gg) {
+ for (short i = 0; i < DVS/8; ++i) {
+ simdgroup_store(mo[i], so + qs*DVS + i*8, DV);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int i = tiitg; i < 8*DV/4; i += NT) {
+ const int h = h0 + gg*8 + i/(DV/4);
+ const int c = i%(DV/4);
+ if (h < args.n_head) {
+ dst4[((int64_t) h*NWG + iwg)*(DV/4) + c] = ((threadgroup float4 *) so)[i];
+ }
+ }
+ }
+ if (tiitg < HT && h0 + tiitg < args.n_head) {
+ dst1[(h0 + tiitg)*(2*NWG) + 2*iwg + 0] = sS[tiitg];
+ dst1[(h0 + tiitg)*(2*NWG) + 2*iwg + 1] = sm[tiitg];
+ }
+}
+
+typedef decltype(kernel_hy4_mla_decode_v2<4>) kernel_hy4_mla_decode_v2_t;
+
+template [[host_name("kernel_hy4_mla_decode_v2_ng2")]] kernel kernel_hy4_mla_decode_v2_t kernel_hy4_mla_decode_v2<2>;
diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal
index 5dfd4a60c..295fbe8b2 100644
--- a/ggml/src/ggml-metal/kernels/misc.metal
+++ b/ggml/src/ggml-metal/kernels/misc.metal
@@ -511,6 +511,52 @@ typedef decltype(kernel_count_equal<int32_t>) kernel_count_equal_t;
template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal<int32_t>;
+kernel void kernel_hc_gate_f32(
+ constant ggml_metal_kargs_hc_gate & args,
+ device const char * x,
+ device const float * f,
+ device const float * b,
+ device float * dst,
+ uint tgpig [[threadgroup_position_in_grid]],
+ uint tpitg [[thread_position_in_threadgroup]],
+ uint ntg [[threads_per_threadgroup]]) {
+ const int idx = tgpig * ntg + tpitg;
+ if (idx >= args.n * args.n_rows) {
+ return;
+ }
+
+ const int i = idx % args.n;
+ const int r = idx / args.n;
+
+ // same op order as the unfused chain: mul, add, sigmoid, then the scales
+ const float z = ((device const float *) (x + r*args.nb_x1))[i] * f[0] + b[i];
+ const float s = 1.0f / (1.0f + exp(-z));
+ dst[idx] = args.s1 * (args.s0 * s + args.c0) + args.c1;
+}
+
+// x * sigmoid(g) over n contiguous floats, 4 per thread; the same sigmoid as kernel_unary (sigmoid op)
+kernel void kernel_sigmoid_mul_f32(
+ constant int32_t & n,
+ device const float * g,
+ device const float * x,
+ device float * dst,
+ uint tpig [[thread_position_in_grid]]) {
+ const int i = tpig*4;
+ if (i >= n) {
+ return;
+ }
+ if (n - i >= 4) {
+ const float4 gv = ((device const float4 *) g)[tpig];
+ const float4 s = 1.0f / (1.0f + exp(-gv));
+ ((device float4 *) dst)[tpig] = ((device const float4 *) x)[tpig] * s;
+ } else {
+ for (int j = i; j < n; ++j) {
+ const float s = 1.0f / (1.0f + exp(-g[j]));
+ dst[j] = x[j] * s;
+ }
+ }
+}
+
template <typename T>
kernel void kernel_snake(
constant ggml_metal_kargs_snake & args,
@@ -810,6 +856,227 @@ kernel void kernel_dsv4_hc_pre_gated_f32(
*(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result;
}
+// DSV4_HC_PRE + RMS_NORM (+ MUL, F == 2): one threadgroup per token with the float4 thread mapping and the
+// reduction of kernel_rms_norm_fuse_impl<float4>, each element computed as kernel_dsv4_hc_pre(_gated) does, so
+// the result is the unfused one bit for bit. WM: 0 the weights, 1 gated (sigmoid of a per-element gate), 2 the
+// weights computed from the mixes as kernel_hc_gate_f32 does (the hc gate fused in front; the same bits as that
+// kernel followed by these ops)
+template <short WM, short F>
+kernel void kernel_hy4_hc_pre_rms_f32(
+ constant ggml_metal_kargs_hy4_hc_pre_rms & args,
+ device const char * x,
+ device const char * w,
+ device const char * f0,
+ device char * dst,
+ device const float * gf,
+ device const float * gb,
+ threadgroup float * shmem_f32 [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort3 tpitg[[thread_position_in_threadgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ constant ggml_metal_kargs_dsv4_hc_pre & hc = args.hc;
+
+ if (sgitg == 0) {
+ shmem_f32[tiisg] = 0.0f;
+ }
+
+ const int it = tgpig.x;
+
+ float wg[8]; // the host limits the fused gate to 8 streams
+ if (WM == 2) {
+ FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) {
+ const float z = ((device const float *) (w + it*args.nb_m1))[ih] * gf[0] + gb[ih];
+ const float s = 1.0f / (1.0f + exp(-z));
+ wg[ih] = args.s1 * (args.s0 * s + args.c0) + args.c1;
+ }
+ }
+
+ auto pre = [&](int i00) {
+ float4 v;
+ FOR_UNROLL (short c = 0; c < 4; ++c) {
+ const int i0 = 4*i00 + c;
+ device const char * xb = x + i0*hc.nb_x0 + it*hc.nb_x2;
+ float result = 0.0f;
+ FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) {
+ float wv;
+ if (WM == 2) {
+ wv = wg[ih];
+ } else if (WM == 1) {
+ wv = 1.0f/(1.0f + exp(-*(device const float *) (w + i0*hc.nb_w0 + ih*hc.nb_w1 + it*hc.nb_w2)));
+ } else {
+ wv = *(device const float *) (w + ih*hc.nb_w0 + it*hc.nb_w1);
+ }
+ const float xv = *(device const float *) (xb + ih*hc.nb_x1);
+ result = fma(xv, wv, result);
+ }
+ v[c] = hc.scale*result;
+ }
+ return v;
+ };
+
+ float sumf = 0.0f;
+
+ // parallel sum
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ const float4 v = pre(i00);
+ sumf += dot(v, v);
+ }
+ sumf = simd_sum(sumf);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ shmem_f32[sgitg] = sumf;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ sumf = shmem_f32[tiisg];
+ sumf = simd_sum(sumf);
+
+ const float mean = sumf/hc.n_embd;
+ const float scale = 1.0f/sqrt(mean + args.eps);
+
+ device float4 * y = (device float4 *) (dst + it*args.nb_y1);
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ const float4 v = pre(i00);
+ if (F == 1) {
+ y[i00] = (v*scale);
+ }
+ if (F == 2) {
+ y[i00] = (v*scale)*((device const float4 *) f0)[i00];
+ }
+ }
+}
+
+typedef decltype(kernel_hy4_hc_pre_rms_f32<0, 1>) kernel_hy4_hc_pre_rms_t;
+
+template [[host_name("kernel_hy4_hc_pre_rms_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<0, 1>;
+template [[host_name("kernel_hy4_hc_pre_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<0, 2>;
+template [[host_name("kernel_hy4_hc_pre_gated_rms_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<1, 1>;
+template [[host_name("kernel_hy4_hc_pre_gated_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<1, 2>;
+template [[host_name("kernel_hy4_hc_gate_pre_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<2, 2>;
+
+// ADD + RMS_NORM + MUL: the sum is formed in registers in both passes instead of being stored and read back.
+// Same float4 thread mapping and reduction as kernel_rms_norm_fuse_impl<float4, 2>, so the result is the unfused
+// one bit for bit (the sum of two floats rounds the same way in registers as in memory).
+kernel void kernel_hy4_add_rms_norm_mul_f32(
+ constant ggml_metal_kargs_hy4_add_rms & args,
+ device const char * a,
+ device const char * b,
+ device const char * w,
+ device char * dst,
+ threadgroup float * shmem_f32 [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort3 tpitg[[thread_position_in_threadgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ if (sgitg == 0) {
+ shmem_f32[tiisg] = 0.0f;
+ }
+
+ const int i01 = tgpig.x;
+
+ device const float4 * av = (device const float4 *) (a + i01*args.nb_a1);
+ device const float4 * bv = (device const float4 *) (b + i01*args.nb_b1);
+ device const float4 * wv = (device const float4 *) w;
+
+ float sumf = 0.0f;
+
+ // parallel sum
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ const float4 v = av[i00] + bv[i00];
+ sumf += dot(v, v);
+ }
+ sumf = simd_sum(sumf);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ shmem_f32[sgitg] = sumf;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ sumf = shmem_f32[tiisg];
+ sumf = simd_sum(sumf);
+
+ const float mean = sumf/args.ne00;
+ const float scale = 1.0f/sqrt(mean + args.eps);
+
+ device float4 * y = (device float4 *) (dst + i01*args.nb_y1);
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ const float4 v = av[i00] + bv[i00];
+ y[i00] = (v*scale)*wv[i00];
+ }
+}
+
+// DSV4_HC_POST (no comb) + RMS_NORM of the flattened streams: one threadgroup per token with the float4 mapping
+// and the reduction of kernel_rms_norm_fuse_impl<float4>; the streams are computed as kernel_dsv4_hc_post_nocomb
+// does, written out, and read back for the norm
+kernel void kernel_hy4_hc_post_rms_f32(
+ constant ggml_metal_kargs_hy4_hc_post_rms & args,
+ device const char * x,
+ device const char * residual,
+ device const char * post,
+ device char * dst,
+ device char * dst_rms,
+ threadgroup float * shmem_f32 [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort3 tpitg[[thread_position_in_threadgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ if (sgitg == 0) {
+ shmem_f32[tiisg] = 0.0f;
+ }
+
+ const int it = tgpig.x;
+ const int ne4 = args.n_embd/4;
+
+ device float4 * ys = (device float4 *) (dst + it*args.nb_d2);
+
+ float sumf = 0.0f;
+
+ // parallel sum
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ const int idst = i00/ne4;
+ const int i0 = 4*(i00%ne4);
+ const float p = *(device const float *) (post + idst*args.nb_p0 + it*args.nb_p1);
+ float4 v;
+ FOR_UNROLL (short c = 0; c < 4; ++c) {
+ const float xv = *(device const float *) (x + (i0 + c)*args.nb_x0 + it*args.nb_x1);
+ const float rv = *(device const float *) (residual + (i0 + c)*args.nb_r0 + idst*args.nb_r1 + it*args.nb_r2);
+ v[c] = xv*p + rv;
+ }
+ ys[i00] = v;
+ sumf += dot(v, v);
+ }
+ sumf = simd_sum(sumf);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (tiisg == 0) {
+ shmem_f32[sgitg] = sumf;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ sumf = shmem_f32[tiisg];
+ sumf = simd_sum(sumf);
+
+ const float mean = sumf/(4*args.ne00_t);
+ const float scale = 1.0f/sqrt(mean + args.eps);
+
+ device float4 * y = (device float4 *) (dst_rms + it*args.nb_y1);
+ for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) {
+ y[i00] = (ys[i00]*scale);
+ }
+}
+
kernel void kernel_dsv4_hc_post_nocomb_f32(
constant ggml_metal_kargs_dsv4_hc_post & args,
device const char * x,
diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal
index 5d012e484..41b6b6f4c 100644
--- a/ggml/src/ggml-metal/kernels/mul_mv.metal
+++ b/ggml/src/ggml-metal/kernels/mul_mv.metal
@@ -597,6 +597,89 @@ kernel void kernel_mul_mv_q8_0_f32(
kernel_mul_mv_q8_0_f32_impl<N_R0_Q8_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
}
+// Q8_0 mat-vec for short rows (ne00 <= 512, e.g. the per-head MLA projections wk_b/wv_b): each simdgroup owns
+// R whole rows and reduces each with one simd_sum, so there is no cross-simdgroup reduction and no barrier. A
+// lane keeps up to two 8-wide chunks of the activation in registers and reuses them for all R rows, and the
+// loads of all R rows are issued before any reduction.
+template<short R>
+kernel void kernel_mul_mv_q8_0_f32_short(
+ constant ggml_metal_kargs_mul_mv & args,
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ threadgroup char * shmem [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ const short NSG = FC_mul_mv_nsg;
+
+ const int nq = args.ne00/8; // 8-wide chunks per row, at most 64
+
+ const int r0 = (tgpig.x*NSG + sgitg)*R;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const uint i12 = im%FC_mul_mv_ne12;
+ const uint i13 = im/FC_mul_mv_ne12;
+
+ device const float * y = (device const float *) (src1 + r1*args.nb11 + i12*args.nb12 + i13*args.nb13);
+
+ const int c0 = tiisg;
+ const int c1 = tiisg + 32;
+
+ float y0[8];
+ float y1[8];
+ FOR_UNROLL (short i = 0; i < 8; ++i) {
+ y0[i] = c0 < nq ? y[c0*8 + i] : 0.0f;
+ y1[i] = c1 < nq ? y[c1*8 + i] : 0.0f;
+ }
+
+ device float * dst_f32 = (device float *) dst + (uint64_t) im*args.ne0*args.ne1 + (uint64_t) r1*args.ne0;
+
+ const uint64_t offset02 = (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
+
+ // issue the loads of all R rows before reducing, so they are in flight together
+ float sum[R];
+ FOR_UNROLL (short row = 0; row < R; ++row) {
+ const int r = min(r0 + row, args.ne01 - 1);
+
+ device const block_q8_0 * ax = (device const block_q8_0 *) (src0 + r*args.nb01 + offset02);
+
+ float acc = 0.0f;
+ if (c0 < nq) {
+ device const int8_t * qs = ax[c0/4].qs + (c0%4)*8;
+ float s = 0.0f;
+ FOR_UNROLL (short i = 0; i < 8; ++i) {
+ s += qs[i]*y0[i];
+ }
+ acc += s*(float) ax[c0/4].d;
+ }
+ if (c1 < nq) {
+ device const int8_t * qs = ax[c1/4].qs + (c1%4)*8;
+ float s = 0.0f;
+ FOR_UNROLL (short i = 0; i < 8; ++i) {
+ s += qs[i]*y1[i];
+ }
+ acc += s*(float) ax[c1/4].d;
+ }
+ sum[row] = acc;
+ }
+
+ FOR_UNROLL (short row = 0; row < R; ++row) {
+ const float tot = simd_sum(sum[row]);
+ if (tiisg == 0 && r0 + row < args.ne01) {
+ dst_f32[r0 + row] = tot;
+ }
+ }
+
+ (void) shmem;
+}
+
+typedef decltype(kernel_mul_mv_q8_0_f32_short<4>) kernel_mul_mv_q8_0_f32_short_t;
+
+template [[host_name("kernel_mul_mv_q8_0_f32_short_r4")]] kernel kernel_mul_mv_q8_0_f32_short_t kernel_mul_mv_q8_0_f32_short<4>;
+template [[host_name("kernel_mul_mv_q8_0_f32_short_r2")]] kernel kernel_mul_mv_q8_0_f32_short_t kernel_mul_mv_q8_0_f32_short<2>;
+
// mat-vec kernel processing in chunks of float4
// chpb - chunks per quantization block
template<short r1ptg, typename q_t, short chpb, void (*deq_t4)(device const q_t *, short, thread float4 &) >
diff --git a/ggml/src/ggml-metal/kernels/rope.metal b/ggml/src/ggml-metal/kernels/rope.metal
index 401ceacb0..7cae7dc49 100644
--- a/ggml/src/ggml-metal/kernels/rope.metal
+++ b/ggml/src/ggml-metal/kernels/rope.metal
@@ -44,20 +44,16 @@ static void rope_yarn_corr_dims(
dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base)));
}
+// one row of kernel_rope_norm: dst is the row's base (the channels at i0*nb0 from it)
template<typename T>
-kernel void kernel_rope_norm(
+static inline void rope_norm_row(
constant ggml_metal_kargs_rope & args,
device const char * src0,
device const char * src1,
device const char * src2,
device char * dst,
- ushort tiitg[[thread_index_in_threadgroup]],
- ushort3 tptg [[threads_per_threadgroup]],
- uint3 tgpig[[threadgroup_position_in_grid]]) {
- const int i3 = tgpig[2];
- const int i2 = tgpig[1];
- const int i1 = tgpig[0];
-
+ const int i1, const int i2, const int i3,
+ ushort tiitg, ushort3 tptg) {
float corr_dims[2];
rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims);
@@ -81,7 +77,7 @@ kernel void kernel_rope_norm(
rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, iw, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta);
device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);
- device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);
+ device T * dst_data = (device T *)( dst + i0*args.nb0);
const float x0 = src[0];
const float x1 = src[1];
@@ -94,7 +90,7 @@ kernel void kernel_rope_norm(
}
device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00);
- device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);
+ device T * dst_data = (device T *)( dst + i0*args.nb0);
dst_data[0] = src[0];
dst_data[1] = src[1];
@@ -102,6 +98,50 @@ kernel void kernel_rope_norm(
}
}
+template<typename T>
+kernel void kernel_rope_norm(
+ constant ggml_metal_kargs_rope & args,
+ device const char * src0,
+ device const char * src1,
+ device const char * src2,
+ device char * dst,
+ ushort tiitg[[thread_index_in_threadgroup]],
+ ushort3 tptg [[threads_per_threadgroup]],
+ uint3 tgpig[[threadgroup_position_in_grid]]) {
+ const int i3 = tgpig[2];
+ const int i2 = tgpig[1];
+ const int i1 = tgpig[0];
+
+ rope_norm_row<T>(args, src0, src1, src2, dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1, i1, i2, i3, tiitg, tptg);
+}
+
+// ROPE (norm) + CONCAT along dim 0 with the rotated tensor second (hy_v4_flash: q and k as [nope | rope]): the
+// row of the first source is copied, the rotation written right after it, as the two kernels would
+kernel void kernel_hy4_rope_concat_f32(
+ constant ggml_metal_kargs_rope & args,
+ constant ggml_metal_kargs_concat & cc,
+ device const char * src0,
+ device const char * src1,
+ device const char * src2,
+ device const char * srcc,
+ device char * dst,
+ ushort tiitg[[thread_index_in_threadgroup]],
+ ushort3 tptg [[threads_per_threadgroup]],
+ uint3 tgpig[[threadgroup_position_in_grid]]) {
+ const int i3 = tgpig[2];
+ const int i2 = tgpig[1];
+ const int i1 = tgpig[0];
+
+ device char * row = dst + i3*cc.nb3 + i2*cc.nb2 + i1*cc.nb1;
+
+ device const char * crow = srcc + i3*cc.nb03 + i2*cc.nb02 + i1*cc.nb01;
+ for (int j = tiitg; j < cc.ne00; j += tptg.x) {
+ *((device float *) (row + j*cc.nb0)) = *((device const float *) (crow + j*cc.nb00));
+ }
+
+ rope_norm_row<float>(args, src0, src1, src2, row + cc.ne00*cc.nb0, i1, i2, i3, tiitg, tptg);
+}
+
template<typename T>
kernel void kernel_rope_neox(
constant ggml_metal_kargs_rope & args,
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 9999aac90..7de14d158 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -11815,7 +11815,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op) &&
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_argmax_f32_i32) >= 64;
case GGML_OP_LIGHTNING_INDEXER:
- return op->src[0]->type == GGML_TYPE_F32 &&
+ return op->src[4] == nullptr && op->src[0]->type == GGML_TYPE_F32 &&
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_BF16) &&
op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F16 &&
op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type) &&
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 9ee16423a..3ca421c3b 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -7394,7 +7394,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
(op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) &&
op->type == GGML_TYPE_F32;
case GGML_OP_LIGHTNING_INDEXER:
- return op->src[0]->type == GGML_TYPE_F32 &&
+ return op->src[4] == nullptr && op->src[0]->type == GGML_TYPE_F32 &&
(op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32 ||
op->src[1]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_Q8_0 ||
op->src[1]->type == GGML_TYPE_Q5_1 || op->src[1]->type == GGML_TYPE_Q5_0 ||
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index ba8578716..afb0fedc9 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -16070,6 +16070,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[0] == 64;
case GGML_OP_LIGHTNING_INDEXER:
{
+ if (op->src[4] != nullptr) {
+ return false; // the row-list form is Metal/CPU only
+ }
const ggml_tensor * q = op->src[0];
const ggml_tensor * k = op->src[1];
const ggml_tensor * w = op->src[2];
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index 3d60c44e5..970a043f5 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -6553,6 +6553,44 @@ struct ggml_tensor * ggml_lightning_indexer(
return result;
}
+struct ggml_tensor * ggml_lightning_indexer_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * weights,
+ struct ggml_tensor * mask,
+ struct ggml_tensor * rows) {
+
+ GGML_ASSERT( q->type == GGML_TYPE_F32);
+ GGML_ASSERT( weights->type == GGML_TYPE_F32);
+ GGML_ASSERT( mask->type == GGML_TYPE_F16);
+ GGML_ASSERT( rows->type == GGML_TYPE_I32);
+ GGML_ASSERT(ggml_is_contiguous(rows) && ggml_nrows(rows) == 1);
+ GGML_ASSERT( q->ne[0] == k->ne[0]);
+ GGML_ASSERT( mask->ne[0] == rows->ne[0]);
+ GGML_ASSERT( q->ne[1] == weights->ne[0]);
+ GGML_ASSERT( k->ne[1] == 1);
+ GGML_ASSERT( mask->ne[1] == q->ne[2]);
+ GGML_ASSERT( q->ne[2] == weights->ne[1]);
+ GGML_ASSERT(weights->ne[2] == 1);
+ GGML_ASSERT( mask->ne[2] == 1);
+ GGML_ASSERT( q->ne[3] == k->ne[3]);
+ GGML_ASSERT( k->ne[3] == weights->ne[3]);
+ GGML_ASSERT(weights->ne[3] % mask->ne[3] == 0);
+
+ int64_t ne[4] = { rows->ne[0], q->ne[2], 1, q->ne[3] };
+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
+
+ result->op = GGML_OP_LIGHTNING_INDEXER;
+ result->src[0] = q;
+ result->src[1] = k;
+ result->src[2] = weights;
+ result->src[3] = mask;
+ result->src[4] = rows;
+
+ return result;
+}
+
// ggml_dsv4_hc_comb
struct ggml_tensor * ggml_dsv4_hc_comb(
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 97250c2e8..d15912560 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -32,6 +32,7 @@ set(LLAMA_CORE_SOURCES
llama-memory-hybrid.cpp
llama-memory-hybrid-iswa.cpp
llama-memory-hybrid-idx.cpp
+ llama-memory-idx-iswa.cpp
llama-memory-recurrent.cpp
llama-mmap.cpp
llama-model-loader.cpp
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 40f608371..b4df866e8 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -132,6 +132,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_HUNYUAN_VL, "hunyuan_vl" },
{ LLM_ARCH_HY_V3, "hy_v3" },
{ LLM_ARCH_HY_V4, "hy_v4" },
+ { LLM_ARCH_HY_V4_FLASH, "hy_v4_flash" },
{ LLM_ARCH_SMOLLM3, "smollm3" },
{ LLM_ARCH_OPENAI_MOE, "gpt-oss" },
{ LLM_ARCH_LFM2, "lfm2" },
@@ -808,6 +809,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "blk.%d.indexer_compressor_norm" },
{ LLM_TENSOR_INDEXER_KPOOL_GATE, "blk.%d.indexer_compressor_gate" },
{ LLM_TENSOR_INDEXER_KPOOL_APE, "blk.%d.indexer_compressor_ape" },
+ { LLM_TENSOR_INDEXER_KPOOL_VALUE, "blk.%d.indexer_compressor_value" },
+ { LLM_TENSOR_INDEXER_KPOOL_NORM, "blk.%d.indexer_compressor_norm" },
{ LLM_TENSOR_FFN_GATE_TID2EID, "blk.%d.ffn_gate_tid2eid" },
{ LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" },
{ LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" },
@@ -820,6 +823,10 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_DSPARK_MASK_EMBD, "mask_embd" },
{ LLM_TENSOR_ENGRAM_KV, "blk.%d.engram_kv" },
{ LLM_TENSOR_ENGRAM_GATE, "blk.%d.engram_gate" },
+ { LLM_TENSOR_ENGRAM_KV_B, "blk.%d.engram_kv_b" },
+ { LLM_TENSOR_ENGRAM_LOOKUP_NORM, "blk.%d.engram_lookup_norm" },
+ { LLM_TENSOR_ENGRAM_PRE_GAMMA, "blk.%d.engram_pre_gamma" },
+ { LLM_TENSOR_ENGRAM_HC_SCALE, "blk.%d.engram_hc_scale" },
{ LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" },
{ LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" },
{ LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" },
@@ -1139,6 +1146,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_INDEXER_COMPRESSOR_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_INDEXER_KPOOL_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_INDEXER_KPOOL_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
+ {LLM_TENSOR_INDEXER_KPOOL_VALUE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_INDEXER_KPOOL_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_GATE_TID2EID, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
{LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
@@ -1175,6 +1184,10 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_DSPARK_MASK_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
{LLM_TENSOR_ENGRAM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ENGRAM_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ENGRAM_KV_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
+ {LLM_TENSOR_ENGRAM_LOOKUP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ENGRAM_PRE_GAMMA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ENGRAM_HC_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
{LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -1372,6 +1385,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_HY_V4:
+ case LLM_ARCH_HY_V4_FLASH:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_BITNET:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index b98be40cf..afff4ca7e 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -136,6 +136,7 @@ enum llm_arch {
LLM_ARCH_HUNYUAN_VL,
LLM_ARCH_HY_V3,
LLM_ARCH_HY_V4,
+ LLM_ARCH_HY_V4_FLASH,
LLM_ARCH_SMOLLM3,
LLM_ARCH_OPENAI_MOE,
LLM_ARCH_LFM2,
@@ -801,6 +802,8 @@ enum llm_tensor {
LLM_TENSOR_INDEXER_COMPRESSOR_NORM,
LLM_TENSOR_INDEXER_KPOOL_GATE, // glm5-next: k-pool gate scores
LLM_TENSOR_INDEXER_KPOOL_APE, // glm5-next: k-pool position bias
+ LLM_TENSOR_INDEXER_KPOOL_VALUE, // hy_v4_flash: what the k-pool actually pools
+ LLM_TENSOR_INDEXER_KPOOL_NORM, // hy_v4_flash: norm of the pooled key
LLM_TENSOR_FFN_GATE_TID2EID,
LLM_TENSOR_NEXTN_PROJ_PRE,
LLM_TENSOR_NEXTN_PROJ_POST,
@@ -825,6 +828,10 @@ enum llm_tensor {
LLM_TENSOR_DSPARK_MASK_EMBD,
LLM_TENSOR_ENGRAM_KV,
LLM_TENSOR_ENGRAM_GATE,
+ LLM_TENSOR_ENGRAM_KV_B, // hy_v4_flash: bias on the fused key/value projections
+ LLM_TENSOR_ENGRAM_LOOKUP_NORM, // hy_v4_flash: per-head RMSNorm of the looked-up rows
+ LLM_TENSOR_ENGRAM_PRE_GAMMA, // hy_v4_flash: scales the normed value before the gate
+ LLM_TENSOR_ENGRAM_HC_SCALE, // hy_v4_flash: per-hc per-channel additive gate offset
LLM_TENSOR_DFLASH_ATTN_CONV_BASE,
LLM_TENSOR_DFLASH_ATTN_CONV_PROJ,
LLM_TENSOR_DFLASH_FFN_CONV_BASE,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 0cd795a56..0095311c6 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2715,7 +2715,8 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_NANBEIGE ||
model.arch == LLM_ARCH_MINIMAX_01 ||
model.arch == LLM_ARCH_MINIMAX_M3 ||
- model.arch == LLM_ARCH_HY_V4) {
+ model.arch == LLM_ARCH_HY_V4 ||
+ model.arch == LLM_ARCH_HY_V4_FLASH) {
res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
} else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) {
// DFlash2's convolutions and selector are shape work rather than matmuls,
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index fe8bdebe5..91955668a 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -834,8 +834,12 @@ void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) {
mctx->get_swa()->set_input_attn_mask(self_attn_mask_swa, ubatch, cparams.causal_attn);
}
+const llama_kv_cache_iswa_context * llm_graph_input_attn_k_iswa::mctx_of(const llm_graph_params & params) const {
+ return static_cast<const llama_kv_cache_iswa_context *>(params.mctx);
+}
+
bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) {
- const auto * mctx = static_cast<const llama_kv_cache_iswa_context *>(params.mctx);
+ const auto * mctx = mctx_of(params);
this->mctx = mctx;
@@ -846,8 +850,10 @@ bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) {
res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;
}
+ // build_attn_inp_k_iswa() builds plain masks without K/V row lists, so check them as such:
+ // can_reuse_attn_mask() expects row lists whenever fused_kv_rows yields a row count, and so never reused these
if (self_attn_mask.raw && self_attn_mask.raw->buffer) {
- res &= can_reuse_attn_mask(self_attn_mask, mctx->get_base(), params.ubatch, params.cparams);
+ res &= can_reuse_kq_mask(self_attn_mask.raw, mctx->get_base(), params.ubatch, params.cparams);
}
// swa tensors may not be allocated if there are no SWA attention layers
@@ -856,7 +862,7 @@ bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) {
}
if (self_attn_mask_swa.raw && self_attn_mask_swa.raw->buffer) {
- res &= can_reuse_attn_mask(self_attn_mask_swa, mctx->get_swa(), params.ubatch, params.cparams);
+ res &= can_reuse_kq_mask(self_attn_mask_swa.raw, mctx->get_swa(), params.ubatch, params.cparams);
}
return res;
@@ -2452,7 +2458,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
const float limit = hparams.swiglu_clamp_exp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
- if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) {
+ if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4 || arch == LLM_ARCH_HY_V4_FLASH) {
cur = ggml_swiglu_clamp(ctx0, cur, up, limit);
} else {
up = ggml_clamp(ctx0, up, -limit, limit);
@@ -3043,7 +3049,10 @@ ggml_tensor * llm_graph_context::build_attn_mha(
cur = ggml_mul_mat(ctx0, v_mla, cur);
cb(cur, "fattn_mla", il);
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
- cur = ggml_cont(ctx0, cur); // Needed because ggml_reshape_2d expects contiguous inputs.
+ // ggml_reshape_2d needs contiguous data; a single token is already laid out that way after the permute
+ if (!ggml_is_contiguous(cur)) {
+ cur = ggml_cont(ctx0, cur);
+ }
#endif
}
@@ -3668,7 +3677,17 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
+ // A standard sliding window shows each query at most n_swa cells, so a short decode (one token, or the block a
+ // speculative draft proposes) can attend over the compacted list of the cells it sees instead of scanning the
+ // whole SWA cache (Metal: the hy4 MLA decode kernel for one token, the per-row index lists above it).
+ // The bound assumes causal masking and one cell per position; the extra n_tokens - 1 entries cover a batch whose
+ // positions repeat, which a draft block does not produce.
+ const int64_t n_tok_max = 8; // the largest DFly block
+ const int64_t n_kv_max = is_swa && hparams.swa_type == LLAMA_SWA_TYPE_STANDARD && !kv_rows &&
+ q_cur->ne[2] <= n_tok_max && hparams.n_swa > 0 &&
+ hparams.n_swa + q_cur->ne[2] - 1 < (int64_t) k->ne[2] ? hparams.n_swa + q_cur->ne[2] - 1 : 0;
+
+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, n_kv_max, kq_scale, il);
cb(cur, "kqv_out", il);
if (k_rot) {
@@ -3868,7 +3887,13 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const
llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const {
const auto * mctx_cur = static_cast<const llama_kv_cache_iswa_context *>(mctx);
- auto inp = std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur);
+ return build_attn_inp_k_iswa(
+ std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur), mctx_cur);
+}
+
+llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa(
+ std::unique_ptr<llm_graph_input_attn_k_iswa> inp,
+ const llama_kv_cache_iswa_context * mctx_cur) const {
{
inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch);
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 37b47acbb..688fda7a0 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -562,12 +562,17 @@ public:
cparams(cparams),
mctx(mctx) {
}
- ~llm_graph_input_attn_k_iswa() = default;
+ virtual ~llm_graph_input_attn_k_iswa() = default;
void set_input(const llama_ubatch * ubatch) override;
bool can_reuse(const llm_graph_params & params) override;
+ // Where this input finds its context in the next batch's params. The default is that the
+ // memory is the iSWA cache itself; an arch whose memory merely composes one overrides this,
+ // because params.mctx is then that outer context and the cast would be wrong.
+ virtual const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const;
+
ggml_tensor * get_k_idxs() const { return self_k_idxs; }
ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; }
@@ -1370,6 +1375,12 @@ struct llm_graph_context {
llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const;
+ // same, for a memory that composes an iSWA pair rather than being one: the arch supplies the
+ // input, so it can override mctx_of(), and the attention context out of its own
+ llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa(
+ std::unique_ptr<llm_graph_input_attn_k_iswa> inp,
+ const llama_kv_cache_iswa_context * mctx_cur) const;
+
// note: if k_cur is not provided, it will not be stored in the memory
// note: the K cache is used as V (MLA-style attention)
ggml_tensor * build_attn(
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 4d2cb8931..19748815b 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -10,7 +10,10 @@
// bump if necessary
#define LLAMA_MAX_LAYERS 512
#define LLAMA_MAX_EXPERTS 1024 // Kimi K3
-#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp, deepseek41 (all hash modules)
+// multipliers are stored flat as [module][ngram] and indexed at stride ple_ngram_size
+// (deepseek41.cpp:247), so this bounds ple_n_modules*ple_ngram_size, not ngram_size alone.
+// hy_v4_flash needs 4 modules x 3 = 12.
+#define LLAMA_MAX_PLE_NGRAM 16 // qwen4exp, deepseek41, hy_v4_flash (all hash modules)
#define LLAMA_MAX_PLE_HEADS 64 // qwen4exp, deepseek41 (all hash modules)
enum llama_expert_gating_func_type {
diff --git a/src/llama-memory-idx-iswa.cpp b/src/llama-memory-idx-iswa.cpp
new file mode 100644
index 000000000..dae3a814a
--- /dev/null
+++ b/src/llama-memory-idx-iswa.cpp
@@ -0,0 +1,989 @@
+#include "llama-memory-idx-iswa.h"
+
+#include "llama-impl.h"
+#include "llama-batch.h"
+#include "llama-io.h"
+#include "llama-model.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <type_traits>
+
+//
+// llama_memory_idx_iswa
+//
+
+llama_memory_idx_iswa::llama_memory_idx_iswa(
+ const llama_model & model,
+ bool no_alloc,
+ ggml_type type_k,
+ ggml_type type_v,
+ bool v_trans,
+ bool offload,
+ bool swa_full,
+ bool unified,
+ uint32_t kv_size,
+ uint32_t n_seq_max,
+ uint32_t n_ubatch,
+ const layer_filter_cb & filter,
+ const layer_filter_cb & filter_idx) :
+ hparams_idx(model.hparams),
+ mem_attn(new llama_kv_cache_iswa(
+ model, no_alloc, type_k, type_v, v_trans, offload, swa_full, unified,
+ kv_size, n_seq_max, n_ubatch,
+ nullptr, filter, nullptr, nullptr)),
+ mem_idx([&] {
+ // a model without an indexer uses llama_kv_cache_iswa directly, so there is no null case here
+ GGML_ASSERT(filter_idx != nullptr);
+
+ // the indexer mirrors the base child alone, so a SWA layer has no cells of its own to index
+ for (uint32_t il = 0; il < model.hparams.n_layer_all; ++il) {
+ GGML_ASSERT(!(filter_idx(il) && model.hparams.is_swa(il)) && "an indexer layer cannot be a SWA layer");
+ }
+
+ // MQA with a single key head of indexer_head_size, as llama_kv_cache_dsa shapes its own
+ std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1);
+ // a k-pool indexer caches its per-token rows and the pooled key side by side
+ // (glm5-next: key | gate | pooled, qwen4exp: key | pooled)
+ hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size * (model.hparams.indexer_kpool > 0 ? model.hparams.indexer_kpool_row : 1);
+
+ // the cached indexer keys are raw, rotation happens after pooling at read time, so a
+ // K-shift must not rotate them while the stream copies in the same update still apply
+ hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE;
+
+ // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors
+ hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size;
+ hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size;
+
+ LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
+
+ // the indexer mirrors the base child, so it takes its size and its (absent) SWA as well
+ return new llama_kv_cache(
+ model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified,
+ kv_size, n_seq_max, 0, LLAMA_SWA_TYPE_NONE,
+ nullptr, filter_idx, nullptr, nullptr, "idx_");
+ }()) {}
+
+llama_memory_context_ptr llama_memory_idx_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
+ // note: repeats llama_kv_cache_iswa::init_batch, as the indexer needs the base slot infos that its context hides
+ do {
+ balloc.split_reset();
+
+ std::vector<llama_ubatch> ubatches;
+
+ while (true) {
+ llama_ubatch ubatch;
+
+ if (embd_all) {
+ // if all tokens are output, split by sequence
+ ubatch = balloc.split_seq(n_ubatch);
+ } else {
+ // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice)
+ const bool unified = (mem_attn->get_base()->get_n_stream() == 1);
+
+ ubatch = balloc.split_equal(n_ubatch, !unified, 0);
+ }
+
+ if (ubatch.n_tokens == 0) {
+ break;
+ }
+
+ ubatches.push_back(std::move(ubatch)); // NOLINT
+ }
+
+ if (balloc.get_n_used() < balloc.get_n_tokens()) {
+ // failed to find a suitable split
+ break;
+ }
+
+ auto sinfos_base = mem_attn->get_base()->prepare(ubatches);
+ if (sinfos_base.empty()) {
+ LLAMA_LOG_ERROR("%s: failed to prepare non-SWA attention ubatches\n", __func__);
+ break;
+ }
+
+ auto sinfos_swa = mem_attn->get_swa()->prepare(ubatches);
+ if (sinfos_swa.empty()) {
+ LLAMA_LOG_ERROR("%s: failed to prepare SWA attention ubatches\n", __func__);
+ break;
+ }
+
+ assert(sinfos_base.size() == sinfos_swa.size());
+
+ // the indexer uses the base cache's slot layout; a separate one can drift from it
+ llama_kv_cache::slot_info_vec_t sinfos_idx = sinfos_base;
+
+ return std::make_unique<llama_memory_idx_iswa_context>(
+ this, std::move(sinfos_base), std::move(sinfos_swa), std::move(sinfos_idx), std::move(ubatches));
+ } while(false);
+
+ return std::make_unique<llama_memory_idx_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
+}
+
+llama_memory_context_ptr llama_memory_idx_iswa::init_full() {
+ return std::make_unique<llama_memory_idx_iswa_context>(this);
+}
+
+llama_memory_context_ptr llama_memory_idx_iswa::init_update(llama_context * lctx, bool optimize) {
+ return std::make_unique<llama_memory_idx_iswa_context>(this, lctx, optimize);
+}
+
+bool llama_memory_idx_iswa::get_can_shift() const {
+ // the indexer holds raw keys, so a shift never has to rotate them: only attention can refuse one
+ return mem_attn->get_can_shift();
+}
+
+void llama_memory_idx_iswa::clear(bool data) {
+ mem_attn->clear(data);
+ mem_idx ->clear(data);
+
+ mem_idx_stale_set(-1, 0);
+}
+
+// A pooled key is only valid while the grouping that produced it holds. Grouping is sequence relative,
+// so an edit at p0 leaves every pool that ends before p0 alone.
+void llama_memory_idx_iswa::mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0) {
+ p0 = std::max<llama_pos>(p0, 0);
+
+ if (seq_id < 0) {
+ for (auto & p : mem_idx_stale) {
+ p = std::min(p, p0);
+ }
+
+ return;
+ }
+
+ GGML_ASSERT(seq_id < (llama_seq_id) LLAMA_MAX_SEQ);
+
+ mem_idx_stale[seq_id] = std::min(mem_idx_stale[seq_id], p0);
+}
+
+// An edit at or below the first position moves pos_min, which regroups the whole sequence.
+llama_pos llama_memory_idx_iswa::mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const {
+ if (seq_id < 0 || p0 <= mem_idx->seq_pos_min(seq_id)) {
+ return 0;
+ }
+
+ return p0;
+}
+
+bool llama_memory_idx_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
+ // the stale position is relative to the first position the indexer still holds, so read it before the removal
+ const llama_pos stale = mem_idx_stale_pos(seq_id, p0);
+
+ bool res = true;
+
+ res = res & mem_attn->seq_rm(seq_id, p0, p1);
+ res = res & mem_idx ->seq_rm(seq_id, p0, p1);
+
+ mem_idx_stale_set(seq_id, stale);
+
+ return res;
+}
+
+void llama_memory_idx_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {
+ // only whole sequences are copied: a shared cell holds a single pool grouping
+ GGML_ASSERT(p0 <= 0 && p1 < 0 && "partial seq_cp is not supported");
+
+ mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1);
+ mem_idx ->seq_cp(seq_id_src, seq_id_dst, p0, p1);
+
+ // a whole sequence copy gives the destination the source's pools, rep rows included: the source keeps its
+ // pooled keys, the destination rebuilds its layout and re-pools into the same rows
+ mem_idx_stale_set(seq_id_dst, 0);
+}
+
+void llama_memory_idx_iswa::seq_keep(llama_seq_id seq_id) {
+ mem_attn->seq_keep(seq_id);
+ mem_idx ->seq_keep(seq_id);
+
+ // every other sequence loses its cells, so their layouts must rebuild
+ mem_idx_stale_set(-1, 0);
+}
+
+void llama_memory_idx_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {
+ // a negative shift moves the cells below p0, so they regroup as well
+ const llama_pos stale = mem_idx_stale_pos(seq_id, shift < 0 ? p0 + shift : p0);
+
+ mem_attn->seq_add(seq_id, p0, p1, shift);
+ mem_idx ->seq_add(seq_id, p0, p1, shift);
+
+ mem_idx_stale_set(seq_id, stale);
+}
+
+void llama_memory_idx_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {
+ mem_attn->seq_div(seq_id, p0, p1, d);
+ mem_idx ->seq_div(seq_id, p0, p1, d);
+
+ mem_idx_stale_set(seq_id, 0);
+}
+
+llama_pos llama_memory_idx_iswa::seq_pos_min(llama_seq_id seq_id) const {
+ return mem_attn->seq_pos_min(seq_id);
+}
+
+llama_pos llama_memory_idx_iswa::seq_pos_max(llama_seq_id seq_id) const {
+ return mem_attn->seq_pos_max(seq_id);
+}
+
+std::map<ggml_backend_buffer_type_t, size_t> llama_memory_idx_iswa::memory_breakdown() const {
+ std::map<ggml_backend_buffer_type_t, size_t> mb = mem_attn->memory_breakdown();
+
+ for (const auto & buft_size : mem_idx->memory_breakdown()) {
+ mb[buft_size.first] += buft_size.second;
+ }
+
+ return mb;
+}
+
+void llama_memory_idx_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {
+ mem_attn->state_write(io, seq_id, flags);
+
+ // the indexer section goes last, so it is a pure suffix: an iSWA-only reader stops early instead of misparsing it
+ mem_idx->state_write(io, seq_id, flags);
+}
+
+void llama_memory_idx_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
+ // note: repeats llama_kv_cache_iswa::state_read
+ // the indexer needs the base cache's cells, and a half-failed restore must leave both caches alike
+
+ // the indexer restore adopts the base cache's layout instead of searching for cells of its own
+ // two find_slot calls agree only while both caches see the same occupancy, which a restore cannot promise
+ llama_kv_cache::slot_info_vec_t sinfos_base;
+
+ try {
+ mem_attn->get_base()->state_read_sinfo(io, seq_id, flags, &sinfos_base, nullptr);
+ mem_attn->get_swa ()->state_read(io, seq_id, flags);
+
+ // must mirror the write order in state_write
+ // the indexer has the SWA type of the base child, so both read the same partial state
+ mem_idx->state_read_sinfo(io, seq_id, flags, nullptr, &sinfos_base);
+
+ // a restore that rewrites cells does it behind the pool layout's back
+ if (!sinfos_base.empty()) {
+ mem_idx_stale_set(seq_id, 0);
+ }
+ } catch (...) {
+ // a half-restored context is the one state the indexer cannot fix by itself: attention holds new cells, the indexer old ones
+ // drop what was being restored from both of them, which is a state they do agree on.
+ state_drop(seq_id);
+
+ throw;
+ }
+}
+
+void llama_memory_idx_iswa::state_drop(llama_seq_id seq_id) {
+ if (seq_id < 0) {
+ clear(true);
+
+ return;
+ }
+
+ mem_attn->get_base()->state_clear(seq_id);
+ mem_attn->get_swa ()->state_clear(seq_id);
+
+ mem_idx->state_clear(seq_id);
+ mem_idx_stale_set(seq_id, 0);
+}
+
+llama_kv_cache_iswa * llama_memory_idx_iswa::get_mem_attn() const {
+ return mem_attn.get();
+}
+
+llama_kv_cache * llama_memory_idx_iswa::get_mem_idx() const {
+ return mem_idx.get();
+}
+
+//
+// llama_memory_idx_iswa_context
+//
+
+// streams in each ubatch's slot info, matching get_k/get_v's `ns`
+static std::vector<uint32_t> llama_memory_idx_iswa_ns(const llama_kv_cache::slot_info_vec_t & sinfos) {
+ std::vector<uint32_t> res;
+ res.reserve(sinfos.size());
+
+ for (const auto & sinfo : sinfos) {
+ res.push_back(sinfo.s1 - sinfo.s0 + 1);
+ }
+
+ return res;
+}
+
+// Which cells of a sequence make up which pool, for the whole cache.
+struct llama_memory_idx_iswa::kpool_layout {
+ struct seq {
+ llama_pos pos_min = 0;
+ uint32_t strm = 0; // Stream holding this sequence's cells
+ std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position.
+ std::vector<uint32_t> pools;
+
+ // Where the pool scan stopped, so an append resumes instead of starting over.
+ size_t j_next = 0;
+ };
+
+ std::array<seq, LLAMA_MAX_SEQ> seqs;
+
+ uint32_t n_pool_real = 0;
+};
+
+// Which pools of the layout the current ubatch must re-pool, in the layout's pool order.
+struct llama_memory_idx_iswa_context::kpool_state {
+ std::vector<uint32_t> is_new;
+ std::vector<uint32_t> rep_gen; // per global cell, the generation that last marked a pool with that rep
+ uint32_t generation = 0;
+
+ uint32_t n_pool_real = 0;
+ uint32_t n_new = 0;
+ uint32_t n_new_g = 1; // graph size of the new pool list, stable across decode steps
+};
+
+namespace {
+
+// The last padded pool is always unused.
+uint32_t kpool_pad(uint32_t n_pool) {
+ return std::max<uint32_t>(64u, GGML_PAD(n_pool + 1, 64u));
+}
+
+// Rank of (pos, cell) in a sequence's cells sorted by position then cell, or -1 when absent.
+// In order mode the rank alone places a token: cells sharing a position (M-RoPE images) have distinct ranks.
+int64_t kpool_rank(const std::vector<std::pair<llama_pos, uint32_t>> & cells, llama_pos pos, uint32_t cell) {
+ auto it = std::lower_bound(cells.begin(), cells.end(), std::make_pair(pos, cell));
+ return it != cells.end() && it->second == cell && it->first == pos ? it - cells.begin() : -1;
+}
+
+}
+
+llama_memory_idx_iswa::~llama_memory_idx_iswa() = default;
+
+const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_get() const {
+ GGML_ASSERT(kpool_lay != nullptr);
+
+ return *kpool_lay;
+}
+
+// Pools are fixed by the positions relative to the sequence's first one, so the layout survives a plain
+// append. A sequence edit can regroup them, and mem_idx_stale tells us it happened.
+const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_update() {
+ if (!kpool_lay) {
+ kpool_lay = std::make_unique<kpool_layout>();
+ }
+
+ auto & lay = *kpool_lay;
+
+ const uint32_t kpool = get_kpool();
+ const uint32_t n_stream_kv = mem_idx->get_n_stream();
+ const bool unified = n_stream_kv == 1;
+
+ lay.n_pool_real = 0;
+
+ for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) {
+ auto & sq = lay.seqs[s];
+
+ // a non unified cache gives each sequence its own stream, with stream local cell indices
+ if (!unified && s >= (llama_seq_id) n_stream_kv) {
+ sq = kpool_layout::seq();
+ continue;
+ }
+
+ const auto & cells = mem_idx->get_cells(unified ? 0 : s);
+ const auto & sp = cells.seq_pos_get(s);
+
+ sq.strm = unified ? 0 : mem_idx->get_stream(s);
+
+ if (mem_idx_stale[s] == POS_CLEAN && !sq.cells.empty() && !sp.empty() &&
+ sq.pos_min == sp.begin()->first) {
+ for (auto it = sp.upper_bound(sq.cells.back()); it != sp.end(); ++it) {
+ sq.cells.push_back(*it);
+ }
+ }
+
+ // the appended tail accounts for every cell only if nothing before it was dropped, but an edit can
+ // regroup a sequence without changing its cell count, so a stale sequence must rebuild regardless
+ if (sq.cells.size() != sp.size() || mem_idx_stale[s] != POS_CLEAN) {
+ sq.cells.assign(sp.begin(), sp.end());
+ sq.pools.clear();
+ sq.j_next = 0;
+ sq.pos_min = sp.empty() ? 0 : sp.begin()->first;
+ }
+
+ // Pools start at the first valid token
+ size_t j = sq.j_next;
+ if (hparams_idx.indexer_kpool_by_order) {
+ // consecutive cells in sequence order, whatever their positions
+ for (; j + kpool <= sq.cells.size(); j += kpool) {
+ sq.pools.push_back((uint32_t) j);
+ }
+ } else {
+ while (j + kpool <= sq.cells.size()) {
+ const llama_pos p0 = sq.cells[j].first;
+ if ((p0 - sq.pos_min) % (llama_pos) kpool != 0) {
+ ++j;
+ continue;
+ }
+ bool ok = true;
+ for (uint32_t k = 1; k < kpool; ++k) {
+ if (sq.cells[j + k].first != p0 + (llama_pos) k) {
+ ok = false;
+ break;
+ }
+ }
+ if (ok) {
+ sq.pools.push_back((uint32_t) j);
+ j += kpool;
+ } else {
+ ++j;
+ }
+ }
+ }
+ sq.j_next = j;
+
+ lay.n_pool_real += (uint32_t) sq.pools.size();
+ }
+
+ return lay;
+}
+
+llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_status status) : status(status) {}
+
+llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem) :
+ mem(mem),
+ // graph reservation walks a full context, and the sparse attention is built only when this is set
+ // without it the reserved worst case is the dense graph, so ggml-alloc must grow the buffer on the first decode
+ ns_ubatch(std::vector<uint32_t>{ mem->get_mem_idx()->get_n_stream() }),
+ ctx_attn(mem->get_mem_attn()->init_full()),
+ ctx_idx (mem->get_mem_idx ()->init_full()),
+ status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) {
+ if (kpool_track()) {
+ mem->kpool_layout_update();
+ auto st = kpool_build_sizes();
+ const auto * idx = mem->get_mem_idx();
+ const uint64_t n_pool_max = uint64_t(idx->get_size() / mem->get_kpool()) * idx->get_n_seq_max();
+ GGML_ASSERT(n_pool_max <= UINT32_MAX - 64);
+ st.n_pool_real = std::max(st.n_pool_real, uint32_t(n_pool_max));
+ st.n_new = st.n_pool_real;
+ st.n_new_g = std::max(st.n_new, 1u);
+ kpool_st = std::make_unique<kpool_state>(std::move(st));
+ i_kpool = 0;
+ }
+}
+
+llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(
+ llama_memory_idx_iswa * mem,
+ llama_context * lctx,
+ bool optimize) :
+ mem(mem),
+ ctx_attn(mem->get_mem_attn()->init_update(lctx, optimize)),
+ // update() applies a pending cross-stream seq_cp, else the copy keeps stale indexer keys
+ ctx_idx (mem->get_mem_idx ()->init_update(lctx, optimize)),
+ status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) {}
+
+llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(
+ llama_memory_idx_iswa * mem,
+ slot_info_vec_t sinfos_base,
+ slot_info_vec_t sinfos_swa,
+ slot_info_vec_t sinfos_idx,
+ std::vector<llama_ubatch> ubatches) :
+ mem(mem),
+ ubatches(std::move(ubatches)),
+ ns_ubatch(llama_memory_idx_iswa_ns(sinfos_idx)),
+ sinfos_kpool(mem->get_kpool() > 0 && mem->get_kpool_by_order() ? sinfos_idx : slot_info_vec_t()),
+ // note: the children copy the ubatches; ctx_idx gets a copy of its own
+ ctx_attn(new llama_kv_cache_iswa_context(mem->get_mem_attn(), std::move(sinfos_base), std::move(sinfos_swa), this->ubatches)),
+ ctx_idx (new llama_kv_cache_context(mem->get_mem_idx(), std::move(sinfos_idx), this->ubatches)),
+ status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) {
+ // Sequence edits force the touched positions to re-pool.
+ mem_idx_stale_batch = mem->mem_idx_stale_get();
+}
+
+llama_memory_idx_iswa_context::~llama_memory_idx_iswa_context() = default;
+
+bool llama_memory_idx_iswa_context::next() {
+ assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+ // Clear only after a successful ubatch.
+ if (i_next == 0 && mem != nullptr) {
+ mem->mem_idx_stale_clear();
+ }
+
+ ctx_attn->next();
+ ctx_idx ->next();
+
+ if (++i_next >= ubatches.size()) {
+ return false;
+ }
+
+ return true;
+}
+
+bool llama_memory_idx_iswa_context::apply() {
+ assert(!llama_memory_status_is_fail(status));
+
+ bool res = true;
+
+ res = res & ctx_attn->apply();
+ res = res & ctx_idx ->apply();
+
+ // Extend the pool layout with this ubatch's cells, then pick what it must re-pool.
+ if (res && kpool_track()) {
+ mem->kpool_layout_update();
+ if (!kpool_st) {
+ kpool_st = std::make_unique<kpool_state>();
+ }
+ kpool_build_state(get_ubatch());
+ i_kpool = i_next;
+ }
+
+ return res;
+}
+
+llama_memory_status llama_memory_idx_iswa_context::get_status() const {
+ return status;
+}
+
+const llama_ubatch & llama_memory_idx_iswa_context::get_ubatch() const {
+ assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
+
+ return ubatches[i_next];
+}
+
+bool llama_memory_idx_iswa_context::kpool_track() const {
+ // Derived from mem instead of being cached.
+ return mem != nullptr && mem->get_kpool() > 0 && !ns_ubatch.empty();
+}
+
+const llama_kv_cache_iswa_context * llama_memory_idx_iswa_context::get_attn() const {
+ return static_cast<const llama_kv_cache_iswa_context *>(ctx_attn.get());
+}
+
+const llama_kv_cache_context * llama_memory_idx_iswa_context::get_base() const {
+ return get_attn()->get_base();
+}
+
+const llama_kv_cache_context * llama_memory_idx_iswa_context::get_swa() const {
+ return get_attn()->get_swa();
+}
+
+const llama_kv_cache_context * llama_memory_idx_iswa_context::get_idx() const {
+ return static_cast<const llama_kv_cache_context *>(ctx_idx.get());
+}
+
+uint32_t llama_memory_idx_iswa_context::get_n_stream() const {
+ GGML_ASSERT(i_next < ns_ubatch.size());
+
+ return ns_ubatch[i_next];
+}
+
+llama_memory_idx_iswa_context::kpool_access::kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd) : ctx(ctx) {
+ // rows are the per-token part (glm5-next: key | gate, qwen4exp: key), then the pooled key
+ const int64_t n_tok = k->ne[0] - n_embd;
+ GGML_ASSERT(n_tok > 0 && n_tok % n_embd == 0);
+
+ const int64_t n_cells = k->ne[1]*k->ne[2];
+
+ // Pool indices can refer to other streams. Revisit these full-storage views if that changes:
+ // https://github.com/ggml-org/llama.cpp/pull/27773#discussion_r4130905603
+ key_gate = ggml_view_2d(ctx, k, n_tok, n_cells, k->nb[1], 0);
+ pooled = ggml_view_2d(ctx, k, n_embd, n_cells, k->nb[1], ggml_row_size(k->type, n_tok));
+}
+
+ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_key_gate(ggml_tensor * idxs) const {
+ return ggml_get_rows(ctx, key_gate, idxs);
+}
+
+ggml_tensor * llama_memory_idx_iswa_context::kpool_access::scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const {
+ return ggml_set_rows(ctx, pooled, values, idxs);
+}
+
+ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_pooled(ggml_tensor * idxs) const {
+ return ggml_get_rows(ctx, pooled, idxs);
+}
+
+ggml_tensor * llama_memory_idx_iswa_context::kpool_access::pooled_rows() const {
+ return ggml_view_4d(ctx, pooled, pooled->ne[0], 1, pooled->ne[1], 1,
+ pooled->nb[1], pooled->nb[1], pooled->nb[1]*pooled->ne[1], 0);
+}
+
+llama_memory_idx_iswa_context::kpool_access llama_memory_idx_iswa_context::get_kpool_access(
+ ggml_context * ctx, int32_t il, int64_t n_embd) const {
+ GGML_ASSERT(mem != nullptr);
+
+ return kpool_access(ctx, mem->get_mem_idx()->get_k_storage(il), n_embd);
+}
+
+// k-pool DSA indexer (hy_v4_flash, glm5-next, qwen4exp QSA)
+
+// Sizes only, used by the full cache context so get_n_kpool() works during graph reserve.
+llama_memory_idx_iswa_context::kpool_state llama_memory_idx_iswa_context::kpool_build_sizes() const {
+ const auto & lay = mem->kpool_layout_get();
+
+ kpool_state st;
+ st.n_pool_real = lay.n_pool_real;
+
+ return st;
+}
+
+// Which pools this ubatch must re-pool.
+// Pool cache lifecycle:
+// 1. cpy_k writes each token's key | gate into its idx cache row, pooled slot are zeroed.
+// 2. This marks the pools the ubatch touches or completes as new, during decode that's one pool every kpool tokens, zero elsewise.
+// 3. The graph pools only the new pools and set_rows each result into the pooled slot of the pool's last member row.
+// 4. All pools are gathered in one get_rows via pool_cells, fresh ones just written, older ones from whatever batch last wrote them.
+// A seq_* edit regroups the pools from the edited position on, so it stales them and the first ubatch of the next batch
+// rebuilds them from the still-valid key | gate rows, rewriting the (possibly different) rep rows.
+// Orphaned pooled slots are never cleared, a slot is only ever read through pool_cells, which follows the current grouping.
+void llama_memory_idx_iswa_context::kpool_build_state(const llama_ubatch & ubatch) {
+ const auto & lay = mem->kpool_layout_get();
+ auto & st = *kpool_st;
+
+ const auto * idx = mem->get_mem_idx();
+ const uint32_t kv_size = idx->get_size();
+ const uint32_t kpool = mem->get_kpool();
+
+ st.n_pool_real = lay.n_pool_real;
+ st.n_new = 0;
+ if (++st.generation == 0) {
+ std::fill(st.is_new.begin(), st.is_new.end(), 0);
+ std::fill(st.rep_gen.begin(), st.rep_gen.end(), 0);
+ st.generation = 1;
+ }
+ st.is_new.resize(lay.n_pool_real, 0);
+ st.rep_gen.resize((size_t) kv_size*idx->get_n_stream(), 0);
+
+ std::array<uint32_t, LLAMA_MAX_SEQ> pool_start;
+
+ // a pool is marked once per rep: sequences sharing cells (a seq_cp, or tokens decoded for several sequences)
+ // share their pools, whose single pooled row they all read through pool_cells, so the scatter rows stay unique
+ auto mark = [&](llama_seq_id s, size_t k) {
+ const auto & sq = lay.seqs[s];
+ const size_t rep = (size_t) sq.strm*kv_size + sq.cells[sq.pools[k] + kpool - 1].second;
+ if (st.rep_gen[rep] != st.generation) {
+ st.rep_gen[rep] = st.generation;
+ st.is_new[pool_start[s] + k] = st.generation;
+ ++st.n_new;
+ }
+ };
+
+ uint32_t ip = 0;
+ for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) {
+ const auto & sq = lay.seqs[s];
+ pool_start[s] = ip;
+ ip += (uint32_t) sq.pools.size();
+
+ // A sequence edit invalidates only pools ending after the edited position.
+ const llama_pos stale_from = i_next == 0 ?
+ mem_idx_stale_batch[s] : llama_memory_idx_iswa::POS_CLEAN;
+ if (stale_from == llama_memory_idx_iswa::POS_CLEAN) {
+ continue;
+ }
+
+ auto first = std::lower_bound(sq.pools.begin(), sq.pools.end(), stale_from,
+ [&](uint32_t j, llama_pos p) { return sq.cells[j + kpool - 1].first < p; });
+ for (auto it = first; it != sq.pools.end(); ++it) {
+ mark(s, it - sq.pools.begin());
+ }
+ }
+ GGML_ASSERT(ip == st.is_new.size());
+
+ // in order mode a token's cell gives its rank, and the rank its pool: positions cannot, as an image shares one
+ const bool by_order = mem->get_kpool_by_order();
+ const auto * sinfo = by_order ? &sinfos_kpool[i_next] : nullptr;
+ const uint32_t n_tps = by_order ? (uint32_t) sinfo->size() : 0;
+
+ for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
+ const llama_pos p = ubatch.pos[i];
+ for (int32_t k = 0; k < ubatch.n_seq_id[i]; ++k) {
+ const llama_seq_id s = ubatch.seq_id[i][k];
+ const auto & sq = lay.seqs[s];
+ if (by_order) {
+ const int64_t r = kpool_rank(sq.cells, p, sinfo->idxs[i / n_tps][i % n_tps]);
+ GGML_ASSERT(r >= 0);
+ if ((size_t) r / kpool < sq.pools.size()) {
+ mark(s, (size_t) r / kpool);
+ }
+ continue;
+ }
+ auto it = std::upper_bound(sq.pools.begin(), sq.pools.end(), p,
+ [&](llama_pos pos, uint32_t j) { return pos < sq.cells[j].first; });
+ if (it == sq.pools.begin()) {
+ continue;
+ }
+ --it;
+ if (p <= sq.cells[*it + kpool - 1].first) {
+ mark(s, it - sq.pools.begin());
+ }
+ }
+ }
+
+ // a ubatch touches at most t_s/kpool + 1 pools per sequence, pad to that bound so the graph keeps its shape
+ // as the count moves; reserve sizes the list for every pool the cache can hold, so never pad past n_pool_max
+ const uint32_t n_pool_max = kv_size / kpool * idx->get_n_seq_max();
+ const uint32_t bound = ubatch.n_tokens/kpool + ubatch.n_seqs_unq;
+ st.n_new_g = std::max({st.n_new, 1u, std::min({bound, kpool_pad(st.n_pool_real) - 1, n_pool_max})});
+}
+
+const llama_memory_idx_iswa_context::kpool_state & llama_memory_idx_iswa_context::kpool_cur() const {
+ GGML_ASSERT(kpool_st != nullptr && i_kpool == i_next && "k-pool state read before apply()");
+
+ return *kpool_st;
+}
+
+uint32_t llama_memory_idx_iswa_context::get_n_kpool() const {
+ return kpool_pad(kpool_cur().n_pool_real);
+}
+
+uint32_t llama_memory_idx_iswa_context::get_n_kpool_new() const {
+ return kpool_cur().n_new_g;
+}
+
+void llama_memory_idx_iswa_context::set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs,
+ ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep,
+ const llama_ubatch * ubatch, ggml_tensor * new_pool_pos) const {
+ GGML_ASSERT(mem != nullptr);
+ GGML_ASSERT(ggml_backend_buffer_is_host(pool_cells->buffer));
+ GGML_ASSERT(ggml_backend_buffer_is_host(pool_idxs->buffer));
+ GGML_ASSERT(ggml_backend_buffer_is_host(pool_mask->buffer));
+ GGML_ASSERT(ggml_backend_buffer_is_host(tail_idxs->buffer));
+
+ const uint32_t kpool = mem->get_kpool();
+ const uint32_t n_kv = get_idx()->get_n_kv();
+
+ const auto & st = kpool_cur();
+ const auto & lay = mem->kpool_layout_get();
+
+ const uint32_t n_tokens = ubatch->n_tokens;
+ const uint32_t n_pool = (uint32_t) pool_cells->ne[0];
+ const uint32_t n_new = st.n_new;
+ // the graph always pools at least one entry, padded to a stable bound, see kpool_build_state
+ const uint32_t n_new_g = st.n_new_g;
+
+ const bool by_order = mem->get_kpool_by_order();
+
+ const uint32_t n_tail_slots = (uint32_t) tail_idxs->ne[0];
+ const bool tail_full = n_tail_slots == kpool;
+
+ GGML_ASSERT(n_pool == kpool_pad(st.n_pool_real));
+ GGML_ASSERT(st.is_new.size() == st.n_pool_real);
+ GGML_ASSERT(pool_mask->ne[0] == (int64_t) n_pool && pool_mask->ne[1] == (int64_t) n_tokens);
+ GGML_ASSERT((n_tail_slots == kpool - 1 || tail_full) && tail_idxs->ne[1] == (int64_t) n_tokens);
+ GGML_ASSERT(pool_idxs->ne[0] == (int64_t) kpool && pool_idxs->ne[1] == (int64_t) n_pool);
+ GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_idxs->buffer));
+ GGML_ASSERT(ggml_nelements(new_pool_idxs) == (int64_t) kpool*n_new_g);
+ // the graph always scatters the fresh pooled keys back into the cache
+ GGML_ASSERT(new_pool_rep != nullptr && ggml_backend_buffer_is_host(new_pool_rep->buffer));
+ GGML_ASSERT(new_pool_rep->ne[0] == (int64_t) n_new_g);
+ if (new_pool_pos != nullptr) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_pos->buffer));
+ GGML_ASSERT(new_pool_pos->ne[0] == 4*(int64_t) n_new_g);
+ }
+
+ const uint32_t kv_size = mem->get_mem_idx()->get_size();
+ const uint32_t n_stream_kv = mem->get_mem_idx()->get_n_stream();
+
+ auto gcell = [&](const llama_memory_idx_iswa::kpool_layout::seq & sq, uint32_t cell) {
+ return (int64_t) sq.strm*kv_size + cell;
+ };
+
+ // Sequences present in this ubatch, pools of absent sequences must fall on the scatter sentinel row.
+ std::vector<uint8_t> seq_in_ub(LLAMA_MAX_SEQ, 0);
+ for (uint32_t i = 0; i < n_tokens; ++i) {
+ for (int32_t k = 0; k < ubatch->n_seq_id[i]; ++k) {
+ seq_in_ub[ubatch->seq_id[i][k]] = 1;
+ }
+ }
+
+ // a cell of this ubatch, written before any read, so the padded pools read a finite K row
+ int64_t dummy_cell = 0;
+ {
+ const llama_seq_id s = ubatch->seq_id[0][0];
+ const auto & sq = lay.seqs[s];
+ auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(ubatch->pos[0], 0u));
+ GGML_ASSERT(it != sq.cells.end() && it->first == ubatch->pos[0]);
+ dummy_cell = gcell(sq, it->second);
+ }
+
+ // in order mode a token sees the pools and the tail up to its own rank in the sequence, which its cell pins down
+ std::vector<int64_t> rank;
+ if (by_order) {
+ const auto & sinfo = sinfos_kpool[i_next];
+ const uint32_t n_tps = (uint32_t) sinfo.size();
+
+ rank.resize(n_tokens);
+ for (uint32_t i = 0; i < n_tokens; ++i) {
+ rank[i] = kpool_rank(lay.seqs[ubatch->seq_id[i][0]].cells, ubatch->pos[i], sinfo.idxs[i / n_tps][i % n_tps]);
+ GGML_ASSERT(rank[i] >= 0);
+ }
+ }
+
+ // padding and absent cells point at the n_kv sentinel row, one past the live cells
+ const int32_t sentinel = (int32_t) n_kv;
+
+ float * gm = nullptr;
+ uint32_t n_sel = 0;
+ uint32_t n_top = 0; // Pools per token in the selection.
+ uint32_t n_tail_sel = 0;
+ if (sel_mask != nullptr) {
+ GGML_ASSERT(ggml_backend_buffer_is_host(sel_mask->buffer));
+ GGML_ASSERT(sel_mask->type == GGML_TYPE_F32);
+ GGML_ASSERT(sel_mask->ne[3] == (int64_t) n_tokens && sel_mask->ne[1] == 1 && sel_mask->ne[2] == 1);
+ n_sel = (uint32_t) sel_mask->ne[0];
+ n_tail_sel = tail_full ? kpool : n_sel % kpool;
+ GGML_ASSERT(n_sel >= n_tail_sel && (n_sel - n_tail_sel) % kpool == 0);
+ GGML_ASSERT(n_tail_sel == 0 || n_tail_sel == n_tail_slots);
+ n_top = (n_sel - n_tail_sel) / kpool;
+ gm = (float *) sel_mask->data;
+ }
+
+ // pools are laid out per sequence
+ std::vector<uint32_t> seq_pool_start(LLAMA_MAX_SEQ, 0);
+ std::vector<llama_pos> pool_end;
+ pool_end.reserve(n_pool);
+
+ int32_t * pcell = (int32_t *) pool_cells->data;
+ int32_t * pidx = (int32_t *) pool_idxs->data;
+ int32_t * nidx = (int32_t *) new_pool_idxs->data;
+ int64_t * nrep = (int64_t *) new_pool_rep->data;
+ int32_t * npos = new_pool_pos != nullptr ? (int32_t *) new_pool_pos->data : nullptr;
+
+ if (npos != nullptr) {
+ std::fill(npos, npos + 4*n_new_g, 0);
+ }
+
+ uint32_t i_new = 0;
+ for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) {
+ const auto & sq = lay.seqs[s];
+ seq_pool_start[s] = (uint32_t) pool_end.size();
+
+ const bool inert = n_stream_kv > 1 && !seq_in_ub[s];
+
+ for (size_t pi = 0; pi < sq.pools.size(); ++pi) {
+ const uint32_t j = sq.pools[pi];
+ const uint32_t ip = (uint32_t) pool_end.size();
+ GGML_ASSERT(ip + 1 < n_pool);
+
+ // The pooled key lives in the last member's row.
+ const uint32_t rep = sq.cells[j + kpool - 1].second;
+ pcell[ip] = (int32_t) gcell(sq, rep);
+
+ for (uint32_t k = 0; k < kpool; ++k) {
+ pidx[(size_t) ip*kpool + k] = inert ? sentinel : (int32_t) sq.cells[j + k].second;
+ }
+
+ if (st.is_new[ip] == st.generation) {
+ GGML_ASSERT(i_new < n_new);
+ for (uint32_t k = 0; k < kpool; ++k) {
+ nidx[(size_t) i_new*kpool + k] = (int32_t) gcell(sq, sq.cells[j + k].second);
+ }
+ nrep[i_new] = gcell(sq, rep);
+ if (npos != nullptr) {
+ // a pooled key is rotated to the M-RoPE position of its first member
+ const uint32_t c = sq.cells[j].second;
+ const auto & e = mem->get_mem_idx()->get_cells(s).ext_get(c);
+ npos[0*n_new_g + i_new] = sq.cells[j].first;
+ npos[1*n_new_g + i_new] = e.y;
+ npos[2*n_new_g + i_new] = e.x;
+ npos[3*n_new_g + i_new] = sq.cells[j].first;
+ }
+ ++i_new;
+ }
+
+ pool_end.push_back(sq.cells[j + kpool - 1].first);
+ }
+ }
+ GGML_ASSERT(i_new == n_new);
+
+ // Padded entries re-pool cells whose pooled slot is never read: only the reps of complete pools are read.
+ // Each entry takes its own cell, entries sharing one would write it from several threads in the scatter.
+ if (n_new_g > n_new) {
+ std::vector<int64_t> reps(pcell, pcell + pool_end.size());
+ std::sort(reps.begin(), reps.end());
+
+ int64_t pad_cell = 0;
+ for (uint32_t i = n_new; i < n_new_g; ++i, ++pad_cell) {
+ while (std::binary_search(reps.begin(), reps.end(), pad_cell)) {
+ ++pad_cell;
+ }
+ GGML_ASSERT(pad_cell < (int64_t) kv_size*n_stream_kv);
+ for (uint32_t k = 0; k < kpool; ++k) {
+ nidx[(size_t) i*kpool + k] = (int32_t) pad_cell;
+ }
+ nrep[i] = pad_cell;
+ }
+ }
+
+ const uint32_t n_pool_real = (uint32_t) pool_end.size();
+ for (uint32_t ip = n_pool_real; ip < n_pool; ++ip) {
+ pcell[ip] = (int32_t) dummy_cell; // pool_cells always addresses the K storage
+ for (uint32_t k = 0; k < kpool; ++k) {
+ pidx[(size_t) ip*kpool + k] = sentinel;
+ }
+ }
+
+ // a pool is visible when it belongs to the token's sequence and ends at or before it, unless reserved by a full-width tail
+ auto fill_mask = [&](auto * data) {
+ using T = std::remove_pointer_t<decltype(data)>;
+ const T keep = llama_cast<T>(0.0f);
+ const T drop = llama_cast<T>(-INFINITY);
+
+ for (uint32_t i = 0; i < n_tokens; ++i) {
+ const llama_seq_id s = ubatch->seq_id[i][0];
+ const llama_pos p = ubatch->pos[i];
+
+ T * row = data + (size_t) i*n_pool;
+ std::fill(row, row + n_pool, drop);
+
+ const uint32_t p0 = seq_pool_start[s];
+ const uint32_t p1 = p0 + (uint32_t) lay.seqs[s].pools.size();
+ const uint32_t nv = by_order ? std::min(p1 - p0, (uint32_t) ((rank[i] + !tail_full)/kpool)) :
+ (uint32_t) ((tail_full ? std::lower_bound(pool_end.begin() + p0, pool_end.begin() + p1, p) :
+ std::upper_bound(pool_end.begin() + p0, pool_end.begin() + p1, p)) - (pool_end.begin() + p0));
+ std::fill(row + p0, row + p0 + nv, keep);
+
+ // Finite visible pools occupy the first min(nv, n_top) ranked slots.
+ if (gm != nullptr) {
+ const uint32_t nvc = std::min(nv, n_top);
+ float * grow = gm + (size_t) i*n_sel;
+ std::fill(grow, grow + (size_t) nvc*kpool, 0.0f);
+ std::fill(grow + (size_t) nvc*kpool, grow + (size_t) n_top*kpool, -INFINITY);
+ }
+ }
+ };
+ if (pool_mask->type == GGML_TYPE_F16) {
+ fill_mask((ggml_fp16_t *) pool_mask->data);
+ } else {
+ fill_mask((float *) pool_mask->data);
+ }
+
+ int32_t * tidx = (int32_t *) tail_idxs->data;
+ for (uint32_t i = 0; i < n_tokens; ++i) {
+ const llama_seq_id s = ubatch->seq_id[i][0];
+ const llama_pos p = ubatch->pos[i];
+ const auto & sq = lay.seqs[s];
+
+ const int64_t pos = by_order ? rank[i] : p - sq.pos_min;
+ const uint32_t n_tail = tail_full ? (uint32_t) (pos % kpool) + 1 : (uint32_t) ((pos + 1) % kpool);
+
+ for (uint32_t k = 0; k < n_tail_slots; ++k) {
+ int32_t cell = sentinel;
+ bool real = false;
+ if (k < n_tail && by_order) {
+ const uint32_t c = sq.cells[rank[i] - k].second;
+ cell = (int32_t) c;
+ real = true;
+ } else if (k < n_tail) {
+ const llama_pos pt = p - (llama_pos) k;
+ auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(pt, 0u));
+ if (it != sq.cells.end() && it->first == pt) {
+ cell = (int32_t) it->second;
+ real = true;
+ }
+ }
+ tidx[(size_t) i*n_tail_slots + k] = cell;
+
+ if (gm != nullptr && n_tail_sel != 0) {
+ gm[(size_t) i*n_sel + (size_t) n_top*kpool + k] = real ? 0.0f : -INFINITY;
+ }
+ }
+ }
+}
diff --git a/src/llama-memory-idx-iswa.h b/src/llama-memory-idx-iswa.h
new file mode 100644
index 000000000..8827773d7
--- /dev/null
+++ b/src/llama-memory-idx-iswa.h
@@ -0,0 +1,252 @@
+#pragma once
+
+#include "llama-kv-cache-iswa.h"
+
+#include <array>
+#include <limits>
+#include <memory>
+#include <vector>
+
+//
+// llama_memory_idx_iswa
+//
+
+// llama_kv_cache_iswa plus a second cache with one indexer key per token, for block-sparse attention (hy_v4_flash)
+// only the non-SWA layers own an indexer, so it mirrors the base child alone: same size, padding, streams and
+// slots, so cell j is one token in both
+
+class llama_memory_idx_iswa : public llama_memory_i {
+public:
+ llama_memory_idx_iswa(
+ const llama_model & model,
+ bool no_alloc,
+ ggml_type type_k,
+ ggml_type type_v,
+ bool v_trans,
+ bool offload,
+ bool swa_full,
+ bool unified,
+ uint32_t kv_size,
+ uint32_t n_seq_max,
+ uint32_t n_ubatch,
+ /* layer filters */
+ const layer_filter_cb & filter,
+ /* the SWA layers never own an indexer, so this must select non-SWA layers only */
+ const layer_filter_cb & filter_idx);
+
+ // Defined out of line because kpool_layout is incomplete here.
+ ~llama_memory_idx_iswa();
+
+ //
+ // llama_memory_i
+ //
+
+ llama_memory_context_ptr init_batch(
+ llama_batch_allocr & balloc,
+ uint32_t n_ubatch,
+ bool embd_all) override;
+
+ llama_memory_context_ptr init_full() override;
+
+ llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;
+
+ bool get_can_shift() const override;
+
+ void clear(bool data) override;
+
+ bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override;
+ void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;
+ void seq_keep(llama_seq_id seq_id) override;
+ void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override;
+ void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override;
+
+ llama_pos seq_pos_min(llama_seq_id seq_id) const override;
+ llama_pos seq_pos_max(llama_seq_id seq_id) const override;
+
+ std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;
+
+ // state write/load
+
+ void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;
+ void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
+
+ //
+ // llama_memory_idx_iswa specific API
+ //
+
+ llama_kv_cache_iswa * get_mem_attn() const;
+ llama_kv_cache * get_mem_idx () const;
+
+ // The model's indexer pool size.
+ uint32_t get_kpool() const { return hparams_idx.indexer_kpool; }
+
+ // Whether pools are kpool consecutive cells in sequence order (qwen4exp) instead of kpool consecutive positions.
+ bool get_kpool_by_order() const { return hparams_idx.indexer_kpool_by_order; }
+
+ // Which cells of a sequence make up which pool of kpool consecutive positions (or cells, in order mode).
+ // It is kept here because it outlives the batch: pools are fixed by the positions relative to the
+ // sequence's first one, so a ubatch only ever appends to it. Sequence edits drop it, see mem_idx_stale.
+ struct kpool_layout;
+
+ const kpool_layout & kpool_layout_update();
+ const kpool_layout & kpool_layout_get() const;
+
+ // The pooled keys persist in the idx cache across batches. A sequence edit can regroup the pools
+ // from some position on, which stales every pooled key at or after it. POS_CLEAN means none.
+ using stale_pos_t = std::array<llama_pos, LLAMA_MAX_SEQ>;
+
+ static constexpr llama_pos POS_CLEAN = std::numeric_limits<llama_pos>::max();
+
+ static stale_pos_t stale_pos_clean() {
+ stale_pos_t res;
+ res.fill(POS_CLEAN);
+ return res;
+ }
+
+ const stale_pos_t & mem_idx_stale_get() const { return mem_idx_stale; }
+ void mem_idx_stale_clear() { mem_idx_stale.fill(POS_CLEAN); }
+
+private:
+ // forget seq_id (all of it if seq_id < 0) in both caches at once, so a failed restore cannot leave them out of step
+ // seq_id < 0 drops the whole context, as the caches themselves do on a failed restore
+ void state_drop(llama_seq_id seq_id);
+
+ // the indexer cache holds one key head per layer, so it needs its own hparams:
+ // llama_kv_cache keeps a reference to what it is given
+ llama_hparams hparams_idx;
+
+ const std::unique_ptr<llama_kv_cache_iswa> mem_attn;
+ const std::unique_ptr<llama_kv_cache> mem_idx;
+
+ // unique_ptr because kpool_layout is incomplete here
+ std::unique_ptr<kpool_layout> kpool_lay;
+
+ // seq_id < 0 stales every sequence, p0 < 0 stales the sequence from its first position
+ void mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0);
+
+ // the position an edit at p0 stales the sequence from
+ llama_pos mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const;
+
+ stale_pos_t mem_idx_stale = stale_pos_clean();
+};
+
+class llama_memory_idx_iswa_context : public llama_memory_context_i {
+public:
+ class kpool_access {
+ public:
+ ggml_tensor * gather_key_gate(ggml_tensor * idxs) const;
+ ggml_tensor * scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const;
+ ggml_tensor * gather_pooled(ggml_tensor * idxs) const;
+
+ // the pooled key of every cell as [n_embd, 1, n_cells, 1], for ops that read the rows by index
+ ggml_tensor * pooled_rows() const;
+
+ private:
+ friend class llama_memory_idx_iswa_context;
+
+ kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd);
+
+ ggml_context * ctx;
+ ggml_tensor * key_gate;
+ ggml_tensor * pooled;
+ };
+
+ using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;
+
+ // used for errors
+ explicit llama_memory_idx_iswa_context(llama_memory_status status);
+
+ // used to create a full-cache context
+ explicit llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem);
+
+ // used to create an update context
+ llama_memory_idx_iswa_context(
+ llama_memory_idx_iswa * mem,
+ llama_context * lctx,
+ bool optimize);
+
+ // used to create a batch processing context from a batch
+ llama_memory_idx_iswa_context(
+ llama_memory_idx_iswa * mem,
+ slot_info_vec_t sinfos_base,
+ slot_info_vec_t sinfos_swa,
+ slot_info_vec_t sinfos_idx,
+ std::vector<llama_ubatch> ubatches);
+
+ ~llama_memory_idx_iswa_context(); // Defined out of line because kpool_state is incomplete here.
+
+ //
+ // llama_memory_context_i
+ //
+
+ bool next() override;
+ bool apply() override;
+
+ llama_memory_status get_status() const override;
+ const llama_ubatch & get_ubatch() const override;
+
+ //
+ // llama_memory_idx_iswa_context specific API
+ //
+
+ const llama_kv_cache_iswa_context * get_attn() const;
+
+ // the per-layer attention context: base for the non-SWA layers, swa for the rest
+ const llama_kv_cache_context * get_base() const;
+ const llama_kv_cache_context * get_swa () const;
+
+ const llama_kv_cache_context * get_idx () const;
+
+ // streams in the current slot info, the `ns` of get_k/get_v; 1 if unified
+ uint32_t get_n_stream() const;
+
+ // hy_v4_flash, complete pools of kpool cells per sequence, scored as whole pools.
+ uint32_t get_n_kpool () const; // Padded pool count, where the last pool is always unused.
+ uint32_t get_n_kpool_new() const; // Pools to re-pool this ubatch, padded to a stable bound, never below 1.
+ kpool_access get_kpool_access(ggml_context * ctx, int32_t il, int64_t n_embd) const;
+ // sel_mask (F32 [n_sel, 1, 1, n_tokens], can be null): 0 for the live selection slots, -inf for the dead ones
+ // new_pool_pos (I32 [4*n_new]): M-RoPE position of each new pool's first member, for pooled keys rotated at pooling time
+ // tail_idxs width: kpool reserves the current group; kpool - 1 keeps only the incomplete tail.
+ void set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs,
+ ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep,
+ const llama_ubatch * ubatch, ggml_tensor * new_pool_pos = nullptr) const;
+
+private:
+ llama_memory_idx_iswa * mem = nullptr;
+
+ // the index of the next ubatch to process
+ size_t i_next = 0;
+
+ std::vector<llama_ubatch> ubatches;
+
+ // streams per ubatch, read from the slot infos before ctx_idx takes them
+ // declared before sinfos_kpool and ctx_idx, so it is initialised while sinfos_idx is still intact
+ const std::vector<uint32_t> ns_ubatch;
+
+ // the indexer cells of each ubatch, kept for pools in cache order (qwen4exp): token s*n + i of ubatch u
+ // sits in cell idxs[s][i] of stream strm[s] of sinfos_kpool[u], and several cells can share a position
+ const slot_info_vec_t sinfos_kpool;
+
+ const llama_memory_context_ptr ctx_attn;
+ const llama_memory_context_ptr ctx_idx;
+
+ const llama_memory_status status;
+
+ // Which pools of the layout this ubatch must re-pool. The layout itself belongs to the memory.
+ struct kpool_state;
+ kpool_state kpool_build_sizes() const;
+ void kpool_build_state(const llama_ubatch & ubatch);
+ const kpool_state & kpool_cur() const;
+
+ // unique_ptr because kpool_state is incomplete here.
+ std::unique_ptr<kpool_state> kpool_st;
+
+ // The ubatch kpool_st was built for, guards against reads before apply.
+ size_t i_kpool = SIZE_MAX;
+
+ // Whether this context tracks k-pool states.
+ bool kpool_track() const;
+
+ // Positions each sequence must re-pool from, cleared only after the first ubatch succeeds
+ llama_memory_idx_iswa::stale_pos_t mem_idx_stale_batch = llama_memory_idx_iswa::stale_pos_clean();
+};
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 36f3850d5..1287e8051 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -357,10 +357,14 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks);
add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size);
}
- if (hparams.ple_n_heads > 0) {
- add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id);
- add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map);
- }
+ }
+
+ // any arch with an Engram module needs these, not just deepseek41. The token map is what says
+ // the arch has one at all: qwen4exp sets ple_n_heads for a plain per-layer embedding and has no
+ // hashed n-gram table, so it must not pick up a stray pad id.
+ if (hparams.ple_n_heads > 0 && !model->engram_token_map.empty()) {
+ add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id);
+ add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map);
}
// the PLE group only means anything whole: write all of it or none
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 7529f55c2..a07ab1226 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -17,6 +17,7 @@
#include "llama-memory-hybrid.h"
#include "llama-memory-hybrid-iswa.h"
#include "llama-memory-hybrid-idx.h"
+#include "llama-memory-idx-iswa.h"
#include "llama-memory-recurrent.h"
#include "llama.h"
@@ -306,6 +307,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_hy_v3(params);
case LLM_ARCH_HY_V4:
return new llama_model_hy_v4(params);
+ case LLM_ARCH_HY_V4_FLASH:
+ return new llama_model_hy_v4_flash(params);
case LLM_ARCH_SMOLLM3:
return new llama_model_smollm3(params);
case LLM_ARCH_OPENAI_MOE:
@@ -2638,7 +2641,8 @@ void llama_model::print_info() const {
if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR ||
arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA ||
arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4 ||
- arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4) {
+ arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4 ||
+ arch == LLM_ARCH_HY_V4_FLASH) {
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv);
@@ -3029,6 +3033,75 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
nullptr);
}
} break;
+ case LLM_ARCH_HY_V4_FLASH:
+ {
+ GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
+
+ if (hparams.indexer_top_k == 0) {
+ // no indexer, so the sparse layers attend their whole context: a plain iSWA pair is enough
+ llama_kv_cache_iswa::layer_filter_cb filter = [&](uint32_t il) {
+ return il < hparams.n_layer();
+ };
+
+ // the draft context holds the MTP block alone
+ if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
+ if (hparams.n_layer_nextn == 0) {
+ throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp");
+ }
+ filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
+ }
+
+ res = new llama_kv_cache_iswa(
+ *this,
+ params.no_alloc,
+ params.type_k,
+ params.type_v,
+ !cparams.flash_attn,
+ cparams.offload_kqv,
+ params.swa_full,
+ cparams.kv_unified,
+ cparams.n_ctx_seq,
+ cparams.n_seq_max,
+ cparams.n_ubatch,
+ nullptr,
+ filter,
+ nullptr,
+ nullptr);
+ } else {
+ // the indexer lives on the sparse layers, which are the non-SWA ones
+ llama_memory_idx_iswa::layer_filter_cb filter = [&](uint32_t il) {
+ return il < hparams.n_layer();
+ };
+ llama_memory_idx_iswa::layer_filter_cb filter_idx = [&](uint32_t il) {
+ return il < hparams.n_layer() && !hparams.is_swa(il) && hparams.is_indexer_full(il);
+ };
+
+ // the draft context holds the MTP block alone, which is a sparse layer and so
+ // owns an indexer of its own
+ if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
+ if (hparams.n_layer_nextn == 0) {
+ throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp");
+ }
+ filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
+ filter_idx = [&](uint32_t il) { return il >= hparams.n_layer(); };
+ }
+
+ res = new llama_memory_idx_iswa(
+ *this,
+ params.no_alloc,
+ params.type_k,
+ params.type_v,
+ !cparams.flash_attn,
+ cparams.offload_kqv,
+ params.swa_full,
+ cparams.kv_unified,
+ cparams.n_ctx_seq,
+ cparams.n_seq_max,
+ cparams.n_ubatch,
+ std::move(filter),
+ std::move(filter_idx));
+ }
+ } break;
case LLM_ARCH_DOTS3NOTE:
{
GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
@@ -3616,6 +3689,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM;
// HY_V4 rotates consecutive pairs, matching the reference implementation
case LLM_ARCH_HY_V4:
+ case LLM_ARCH_HY_V4_FLASH:
return LLAMA_ROPE_TYPE_NORM;
// the pairs of head values are offset by n_rot/2
diff --git a/src/llama-model.h b/src/llama-model.h
index da4964c98..9d11ce517 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -612,10 +612,20 @@ struct llama_layer {
// glm5-next k-pool indexer
struct ggml_tensor * indexer_kpool_gate = nullptr;
struct ggml_tensor * indexer_kpool_ape = nullptr;
+ // hy_v4_flash: the pool runs over a projection of the normed key, and the pooled result is
+ // normed again before it is scored
+ struct ggml_tensor * indexer_kpool_value = nullptr;
+ struct ggml_tensor * indexer_kpool_norm = nullptr;
// deepseek41
struct ggml_tensor * engram_kv = nullptr;
struct ggml_tensor * engram_gate = nullptr; // q_weight*k_weight folded
+ // hy_v4_flash only: biases on engram_kv, the per-head norm of the looked-up rows, and the
+ // scale/offset of the delta the module writes back
+ struct ggml_tensor * engram_kv_b = nullptr;
+ struct ggml_tensor * engram_lookup_norm = nullptr;
+ struct ggml_tensor * engram_pre_gamma = nullptr;
+ struct ggml_tensor * engram_hc_scale = nullptr;
// MSA
struct ggml_tensor * index_q_proj = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 34c247858..3f1b58e00 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -326,6 +326,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
// Engram gate is an elementwise weight (ggml_mul), not a matmul operand
quantize &= name.find("engram_gate.weight") == std::string::npos;
+ // same for these: they are the second operand of a ggml_add / ggml_mul, which has no
+ // quantized kernel, so quantizing them aborts at run time rather than degrading quality
+ quantize &= name.find("engram_kv_b.weight") == std::string::npos;
+ quantize &= name.find("engram_hc_scale.weight") == std::string::npos;
+ quantize &= name.find("indexer_compressor_ape.weight") == std::string::npos;
+
// these are very small (e.g. 4x4)
quantize &= name.find("altup") == std::string::npos;
quantize &= name.find("laurel") == std::string::npos;
diff --git a/src/models/hy-v4-flash.cpp b/src/models/hy-v4-flash.cpp
new file mode 100644
index 000000000..18deb364f
--- /dev/null
+++ b/src/models/hy-v4-flash.cpp
@@ -0,0 +1,1376 @@
+#include "models.h"
+
+#include "llama-kv-cache.h"
+#include "llama-kv-cache-iswa.h"
+#include "llama-memory-idx-iswa.h"
+
+#include <cmath>
+
+// HY_V4_FLASH reuses hy_v4's iHC, gated MLA and learnable sink. What it adds:
+// - a hybrid stack: "sparse" full-attention layers and sliding-window layers, with their own
+// head count and rope base, over one shared MLA latent geometry
+// - sandwich norms after the attention output and after the FFN
+// - routed experts that run at n_expert_latent instead of n_embd
+
+// a token that cannot be hashed: outside the map, or an embedding (image) position
+static constexpr int64_t HY_V4_FLASH_ENGRAM_DEAD = -1;
+
+// The reference builds every Engram norm as an RMSNorm with eps left unset, which resolves to the
+// fp32 machine epsilon rather than to the model's rms_norm_eps.
+static constexpr float HY_V4_FLASH_ENGRAM_EPS = 1.1920929e-07f;
+
+// cache view length (n_kv) up to which a prefill ubatch attends the indexer selection through dense attention kernels
+static constexpr int64_t HY_V4_FLASH_DENSE_FA_MAX_KV = 12288;
+
+static size_t hy_v4_flash_elem_offset(const ggml_tensor * t, int64_t i) {
+ return ggml_row_size(t->type, i);
+}
+
+static ggml_tensor * hy_v4_flash_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) {
+ return ggml_view_1d(ctx, t, ne0, hy_v4_flash_elem_offset(t, i0));
+}
+
+static ggml_tensor * hy_v4_flash_view_2d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t ne1, int64_t i0) {
+ return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], hy_v4_flash_elem_offset(t, i0));
+}
+
+void llama_model_hy_v4_flash::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
+ ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
+ ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
+ ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false);
+
+ // routed experts only; the shared and dense FFNs are not clamped
+ ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);
+
+ ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
+ ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
+ ml.get_key(LLM_KV_HYPER_CONNECTION_MAGNITUDE, hparams.hc_magnitude);
+
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+
+ // Compressed indexer, optional: present on the sparse layers only. A file without it falls
+ // back to those layers attending their whole context, which is what phase 1 shipped, so the
+ // keys are read only when indexer.top_k says there is an indexer at all.
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k, false);
+ if (hparams.indexer_top_k > 0) {
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL, hparams.indexer_kpool);
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail, false);
+ ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false);
+
+ // indexer.types is what says which layers own one. Without it nothing does, and a file that
+ // carries the other indexer keys anyway must not select a cache this graph cannot drive.
+ bool any = false;
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ any = any || hparams.is_indexer_full(il);
+ }
+
+ if (!any) {
+ hparams.indexer_top_k = 0;
+ } else {
+ if (hparams.indexer_kpool < 2 || hparams.indexer_top_k % hparams.indexer_kpool != 0) {
+ throw std::runtime_error(format(
+ "hy_v4_flash: indexer top_k %u must be a multiple of the pool size %u",
+ hparams.indexer_top_k, hparams.indexer_kpool));
+ }
+ if (hparams.indexer_kpool_select_tail && hparams.indexer_top_k / hparams.indexer_kpool < 2) {
+ throw std::runtime_error("hy_v4_flash: indexer top_k must reserve at least one history group and one tail group");
+ }
+ // the indexer cache mirrors the full-attention child, so a window layer cannot own one
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (!hparams.is_indexer_full(il)) {
+ continue;
+ }
+ if (hparams.is_swa(il)) {
+ throw std::runtime_error(format(
+ "hy_v4_flash: layer %u is a sliding-window layer and cannot own an indexer", il));
+ }
+ // the indexer splits each head into a nope front and a rope tail, so the tail must fit
+ if (hparams.n_rot(il) >= hparams.indexer_head_size) {
+ throw std::runtime_error(format(
+ "hy_v4_flash: layer %u indexer key length %u must be larger than the rope dimension %u",
+ il, hparams.indexer_head_size, hparams.n_rot(il)));
+ }
+ }
+ }
+ }
+
+ GGML_ASSERT(hparams.is_mla());
+
+ if (!hparams.is_swa_any()) {
+ throw std::runtime_error("hy_v4_flash: no sliding-window layer, expected a hybrid stack");
+ }
+
+ // One latent geometry for both layer types: only the head count differs, so the graph keeps a
+ // single latent cache and reads one set of MLA dimensions. dots3note is the arch that varies
+ // these per layer, and a file written for that shape would be read here as if it did not, so
+ // refuse it instead of building mis-sized wk_b / wv_b / wo views.
+ uint32_t n_embd_head_k_mla_swa = 0;
+ uint32_t n_embd_head_v_mla_swa = 0;
+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, n_embd_head_k_mla_swa, false);
+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, n_embd_head_v_mla_swa, false);
+
+ if ((n_embd_head_k_mla_swa && n_embd_head_k_mla_swa != hparams.n_embd_head_k_mla()) ||
+ (n_embd_head_v_mla_swa && n_embd_head_v_mla_swa != hparams.n_embd_head_v_mla())) {
+ throw std::runtime_error(format(
+ "hy_v4_flash: needs one MLA geometry for both layer types, but the "
+ "sliding-window layers declare key/value %u/%u against %u/%u",
+ n_embd_head_k_mla_swa, n_embd_head_v_mla_swa,
+ hparams.n_embd_head_k_mla(), hparams.n_embd_head_v_mla()));
+ }
+
+ // MLA is driven as MQA here: one shared latent row per token, so a file claiming more
+ // key/value heads would silently disagree with the cache this graph builds
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (hparams.n_head_kv(il) != 1) {
+ throw std::runtime_error(format(
+ "hy_v4_flash: layer %u has n_head_kv %u, expected 1 for MLA",
+ il, hparams.n_head_kv(il)));
+ }
+ }
+
+ // Engram, optional: the same n-gram hash tables deepseek41 carries, with one module per Engram
+ // layer. Multipliers, primes and offsets are all stored flat as [module][...] and indexed at
+ // module stride, so their lengths pin down the module count.
+ std::vector<uint32_t> e_layers;
+ ml.get_arr(LLM_KV_PLE_LAYERS, e_layers, false);
+ if (!e_layers.empty()) {
+ std::vector<uint64_t> mult;
+ std::vector<uint64_t> offsets;
+ std::vector<uint64_t> primes;
+ ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);
+ ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
+ ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim);
+ ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult);
+ ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets);
+ ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes);
+ ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id);
+ ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map);
+
+ const size_t n_e = e_layers.size();
+ const size_t n_gram = hparams.ple_ngram_size;
+ const size_t n_cols = n_gram >= 2 ? (n_gram - 1)*hparams.ple_heads_per_ngram : 0;
+ if (n_cols == 0 || hparams.ple_head_dim == 0 ||
+ mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols ||
+ mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS ||
+ hparams.engram_pad_id >= engram_token_map.size()) {
+ throw std::runtime_error("hy_v4_flash engram metadata is inconsistent");
+ }
+ for (size_t e = 0; e < n_e; ++e) {
+ if (e_layers[e] >= hparams.n_layer() || (e > 0 && e_layers[e] <= e_layers[e - 1])) {
+ throw std::runtime_error(format("hy_v4_flash engram layer %u is out of range or out of order", e_layers[e]));
+ }
+ hparams.is_ple_impl.set(e_layers[e]);
+ }
+ for (size_t c = 0; c < n_e*n_cols; ++c) {
+ // a bucket count of 0 traps in the hash; row ids are int32
+ if (primes[c] < 2 || offsets[c] + primes[c] > (uint64_t) INT32_MAX) {
+ throw std::runtime_error(format("hy_v4_flash engram head %zu: bucket range out of range", c));
+ }
+ hparams.ple_head_offsets[c] = (uint32_t) offsets[c];
+ hparams.ple_head_vocab_sizes[c] = (uint32_t) primes[c];
+ }
+ std::copy(mult.begin(), mult.end(), hparams.ple_layer_multipliers.begin());
+ hparams.ple_n_heads = (uint32_t) n_cols;
+ hparams.ple_n_modules = (uint32_t) n_e;
+ }
+
+ type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_hy_v4_flash::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();
+ const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();
+ const int64_t n_embd_head_qk_rope = hparams.n_rot();
+ const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
+ GGML_ASSERT(n_embd_head_qk_nope >= 1);
+
+ const int64_t q_lora_rank = hparams.n_lora_q;
+ const int64_t kv_lora_rank = hparams.n_lora_kv;
+ const int64_t n_ff_exp = hparams.n_ff_exp();
+ const int64_t n_expert_shared = hparams.n_expert_shared;
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t n_embd_latent = hparams.n_expert_latent ? hparams.n_expert_latent : n_embd;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
+
+ hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc * n_embd, hc}, 0);
+ hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc}, 0);
+ hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
+
+ const int64_t ple_n_heads = hparams.ple_n_heads;
+ const int64_t ple_head_dim = hparams.ple_head_dim;
+ if (ple_n_heads > 0) {
+ // every module's buckets live in one table, with the converter rebasing each module's head
+ // offsets into it, so the row count is the end of the last bucket
+ int64_t n_rows = 0;
+ for (size_t c = 0; c < (size_t) hparams.ple_n_modules*ple_n_heads; ++c) {
+ n_rows = std::max(n_rows, (int64_t) hparams.ple_head_offsets[c] + hparams.ple_head_vocab_sizes[c]);
+ }
+ // 118 GiB at the full size: read lazily, so only the rows a batch hashes to are faulted in
+ per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {ple_head_dim, n_rows}, TENSOR_READ_LAZY);
+ }
+
+ // the MTP block sits after the trunk and is a sparse layer in its own right, minus the
+ // hyper-connections: it is not part of the hc stack the trunk runs
+ for (int i = 0; i < (int) hparams.n_layer_all; ++i) {
+ auto & layer = layers[i];
+ const bool is_nextn = i >= n_layer;
+
+ // the sliding-window layers carry more heads over the same latent geometry
+ const int64_t n_head_l = hparams.n_head(i);
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head_l}, 0);
+
+ layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
+ layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, 0);
+ layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0);
+ layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);
+ layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, 0);
+ layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, 0);
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l * n_embd_head_v_mla}, 0);
+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+ if (!is_nextn) {
+ layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0);
+ layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {2 * hc}, 0);
+ layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {2}, 0);
+ layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0);
+ layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {2 * hc}, 0);
+ layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {2}, 0);
+ } else {
+ // the draft block folds the next token's embedding into the trunk's final hidden state
+ layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, 0);
+ layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, 0);
+ layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, 0);
+ layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, 0);
+ }
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+ // is_indexer_full() only covers the trunk, and the draft block always owns an indexer
+ if (hparams.indexer_top_k > 0 && (is_nextn || hparams.is_indexer_full(i))) {
+ const int64_t n_idx = hparams.indexer_head_size;
+ const int64_t n_idx_head = hparams.indexer_n_head;
+ const int64_t kpool = hparams.indexer_kpool;
+
+ layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd, n_idx}, 0);
+ layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx}, 0);
+ // the pooled quantity is a projection of the normed key, not the key itself, and the
+ // gate that weights the pooling is another one: both square at the indexer width
+ layer.indexer_kpool_value = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_VALUE, "weight", i), {n_idx, n_idx}, 0);
+ layer.indexer_kpool_gate = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_GATE, "weight", i), {n_idx, n_idx}, 0);
+ layer.indexer_kpool_ape = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_APE, "weight", i), {n_idx, kpool}, 0);
+ layer.indexer_kpool_norm = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_NORM, "weight", i), {n_idx}, 0);
+ layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx*n_idx_head}, 0);
+ layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0);
+ }
+
+ if (!is_nextn && engram_index(i) >= 0) {
+ // the hc key projections and the single value projection are fused into one matmul, as
+ // deepseek41 does; unlike deepseek41 they carry a bias
+ layer.engram_kv = create_tensor(tn(LLM_TENSOR_ENGRAM_KV, "weight", i), {ple_n_heads * ple_head_dim, n_embd * (hc + 1)}, 0);
+ layer.engram_kv_b = create_tensor(tn(LLM_TENSOR_ENGRAM_KV_B, "weight", i), {n_embd, hc + 1}, 0);
+ layer.engram_gate = create_tensor(tn(LLM_TENSOR_ENGRAM_GATE, "weight", i), {n_embd, hc}, 0);
+ layer.engram_lookup_norm = create_tensor(tn(LLM_TENSOR_ENGRAM_LOOKUP_NORM, "weight", i), {ple_head_dim, ple_n_heads}, 0);
+ layer.engram_pre_gamma = create_tensor(tn(LLM_TENSOR_ENGRAM_PRE_GAMMA, "weight", i), {n_embd}, 0);
+ layer.engram_hc_scale = create_tensor(tn(LLM_TENSOR_ENGRAM_HC_SCALE, "weight", i), {n_embd, hc}, 0);
+ }
+
+ if (!is_nextn && i < (int) hparams.n_layer_dense_lead) {
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ } else {
+ if (n_expert == 0 || n_expert_used == 0) {
+ throw std::runtime_error("hy_v4_flash: n_expert and n_expert_used must be > 0");
+ }
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+
+ if (hparams.n_expert_latent) {
+ layer.ffn_routed_down = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_DOWN, "weight", i), {n_embd, n_embd_latent}, 0);
+ layer.ffn_routed_up = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_UP, "weight", i), {n_embd_latent, n_embd}, 0);
+ }
+
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_latent, n_expert}, 0);
+
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd}, 0);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ }
+ }
+}
+
+int llama_model_hy_v4_flash::engram_index(int il) const {
+ if (hparams.ple_n_heads == 0 || !hparams.is_ple(il)) {
+ return -1;
+ }
+ int e = 0;
+ for (int j = 0; j < il; ++j) {
+ e += hparams.is_ple(j);
+ }
+ return e;
+}
+
+void llama_model_hy_v4_flash::engram_hash(int e, const llama_ubatch & ubatch,
+ const std::vector<llama_token> & prev,
+ std::vector<int32_t> & ids, std::vector<float> & valid) const {
+ const int64_t n_gram = hparams.ple_ngram_size;
+ const int64_t n_prev = n_gram - 1;
+ const int64_t n_head = hparams.ple_heads_per_ngram;
+ const int64_t n_cols = hparams.ple_n_heads;
+
+ const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram;
+ const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols;
+ const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols;
+ const int64_t pad = engram_token_map[hparams.engram_pad_id];
+
+ const auto cid = [&](llama_token t) -> int64_t {
+ return t < 0 || t >= (llama_token) engram_token_map.size() ? HY_V4_FLASH_ENGRAM_DEAD : engram_token_map[t];
+ };
+
+ ids.resize(ubatch.n_tokens*n_cols);
+ valid.resize(ubatch.n_tokens*n_cols);
+
+ std::vector<uint64_t> toks(n_gram);
+
+ for (int64_t i = 0; i < ubatch.n_tokens; ++i) {
+ // A history shorter than the n-gram is padded, not dropped: the reference shifts the token
+ // window and fills the overhang with the pad id, so a real row is looked up even at the
+ // start of a sequence. Only an embedding (image) position has no n-gram at all, and there
+ // the reference marks every column invalid and masks the whole contribution.
+ const bool is_embd = !ubatch.token || (ubatch.is_mixed() && ubatch.type[i]);
+ int64_t c = is_embd ? HY_V4_FLASH_ENGRAM_DEAD : cid(ubatch.token[i]);
+ const bool dead = c == HY_V4_FLASH_ENGRAM_DEAD;
+ toks[0] = dead ? pad : c;
+ for (int64_t s = 1; s < n_gram; ++s) {
+ c = ubatch.pos[i] - s < 0 ? HY_V4_FLASH_ENGRAM_DEAD : cid(prev[i*n_prev + (n_prev - s)]);
+ toks[s] = c == HY_V4_FLASH_ENGRAM_DEAD ? pad : c;
+ }
+
+ uint64_t rolling = toks[0]*mult[0];
+ for (int64_t o = 1; o < n_gram; ++o) {
+ rolling ^= toks[o]*mult[o];
+ for (int64_t h = 0; h < n_head; ++h) {
+ const int64_t col = (o - 1)*n_head + h;
+ ids [i*n_cols + col] = (int32_t) (rolling % primes[col] + offs[col]);
+ valid[i*n_cols + col] = dead ? 0.0f : 1.0f;
+ }
+ }
+ }
+}
+
+// the row ids and their validity are host-side, so they are graph inputs rather than computed
+class llm_graph_input_hy_v4_flash : public llm_graph_input_i {
+public:
+ llm_graph_input_hy_v4_flash(const llama_model_hy_v4_flash & model,
+ const llama_kv_cache_context * mctx) : model(model), mctx(mctx) {}
+
+ void set_input(const llama_ubatch * ubatch) override;
+ bool can_reuse(const llm_graph_params & params) override;
+
+ std::vector<ggml_tensor *> engram_ids;
+ std::vector<ggml_tensor *> engram_valid;
+
+ const llama_model_hy_v4_flash & model;
+ const llama_kv_cache_context * mctx;
+};
+
+void llm_graph_input_hy_v4_flash::set_input(const llama_ubatch * ubatch) {
+ if (engram_ids.empty()) {
+ return;
+ }
+
+ // a token shared by several sequences has the same history in each, so its first one resolves it
+ std::vector<llama_token> prev;
+ mctx->get_prev_tokens(*ubatch, model.hparams.ple_ngram_size - 1, prev);
+
+ // hash every module before the first gather, so a lazily read table gets one batched readahead
+ std::vector<std::vector<int32_t>> ids(engram_ids.size());
+ std::vector<std::vector<float>> valid(engram_ids.size());
+ std::vector<int32_t> all;
+ for (size_t e = 0; e < engram_ids.size(); ++e) {
+ model.engram_hash((int) e, *ubatch, prev, ids[e], valid[e]);
+ all.insert(all.end(), ids[e].begin(), ids[e].end());
+ }
+ if (model.can_prefetch.count(model.per_layer_tok_embd)) {
+ llama_prefetch_rows(model.per_layer_tok_embd, all.data(), all.size());
+ }
+ for (size_t e = 0; e < engram_ids.size(); ++e) {
+ if (engram_ids[e]->buffer) {
+ ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t));
+ }
+ if (engram_valid[e]->buffer) {
+ ggml_backend_tensor_set(engram_valid[e], valid[e].data(), 0, valid[e].size()*sizeof(float));
+ }
+ }
+}
+
+bool llm_graph_input_hy_v4_flash::can_reuse(const llm_graph_params & params) {
+ // The memory context is rebuilt for every batch, so the one captured at graph build time is
+ // dangling by now and set_input would read freed memory. Re-fetch it, exactly as deepseek41
+ // does for its own Engram input.
+ const auto * iswa = model.hparams.indexer_top_k > 0
+ ? static_cast<const llama_memory_idx_iswa_context *>(params.mctx)->get_attn()
+ : static_cast<const llama_kv_cache_iswa_context *>(params.mctx);
+ mctx = iswa->get_base();
+
+ bool res = true;
+ for (auto * t : engram_ids) {
+ res &= t->ne[0] == (int64_t) (model.hparams.ple_n_heads*params.ubatch.n_tokens);
+ }
+ return res;
+}
+
+std::unique_ptr<llm_graph_context> llama_model_hy_v4_flash::build_arch_graph(const llm_graph_params & params) const {
+ if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
+ return std::make_unique<graph_mtp>(*this, params);
+ }
+ return std::make_unique<graph>(*this, params);
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_engram(
+ const llama_model & model,
+ ggml_tensor * x,
+ ggml_tensor * rows,
+ ggml_tensor * valid,
+ int il) const {
+ const auto & layer = model.layers[il];
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[2];
+ const int64_t n_cols = hparams.ple_n_heads;
+ const int64_t head_dim = hparams.ple_head_dim;
+
+ GGML_ASSERT(rows->ne[0] == head_dim && rows->ne[1] == n_cols && rows->ne[2] == nt);
+
+ // delta 1: each looked-up row is RMS-normed on its own and scaled by its head's weight.
+ // The per-column scale by valid is belt and braces: valid is uniform across a token's columns
+ // (a short history is padded, not invalidated), and the whole-token mask below is what actually
+ // suppresses an embedding position.
+ ggml_tensor * r = ggml_rms_norm(ctx0, rows, HY_V4_FLASH_ENGRAM_EPS);
+ r = ggml_mul(ctx0, r, layer.engram_lookup_norm);
+ r = ggml_mul(ctx0, r, ggml_reshape_3d(ctx0, valid, 1, n_cols, nt));
+ r = ggml_reshape_2d(ctx0, r, head_dim*n_cols, nt);
+ cb(r, "engram_rows_normed", il);
+
+ // delta 2: the fused key/value projection carries a bias
+ ggml_tensor * kv = build_lora_mm(layer.engram_kv, r);
+ kv = ggml_add(ctx0, ggml_reshape_3d(ctx0, kv, n_embd, hc + 1, nt), layer.engram_kv_b);
+ cb(kv, "engram_kv", il);
+
+ ggml_tensor * key = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, hc, nt, kv->nb[1], kv->nb[2], 0));
+ ggml_tensor * value = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, 1, nt, kv->nb[1], kv->nb[2], hc*kv->nb[1]));
+
+ // the gate is deepseek41's: engram_gate folds norm_key.w * norm_query.w
+ ggml_tensor * egate = ggml_is_quantized(layer.engram_gate->type) ? ggml_cast(ctx0, layer.engram_gate, GGML_TYPE_F32) : layer.engram_gate;
+ ggml_tensor * dot = ggml_mul(ctx0, ggml_mul(ctx0, ggml_rms_norm(ctx0, x, HY_V4_FLASH_ENGRAM_EPS), ggml_rms_norm(ctx0, key, HY_V4_FLASH_ENGRAM_EPS)), egate);
+ dot = ggml_scale(ctx0, ggml_sum_rows(ctx0, dot), 1.0f/sqrtf((float) n_embd));
+ cb(dot, "engram_dot", il);
+
+ // sigmoid(copysign(sqrt(max(|dot|, 1e-6)), dot)); sign(0) = +1 as in copysign
+ ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY));
+ ggml_tensor * sign = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_neg(ctx0, dot)), -2.0f, 1.0f);
+ ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, sign));
+ cb(gate, "engram_gate", il);
+
+ // delta 3: the delta is the normed value scaled by a learned gamma, not the raw value
+ ggml_tensor * signal = ggml_mul(ctx0, ggml_rms_norm(ctx0, value, HY_V4_FLASH_ENGRAM_EPS), layer.engram_pre_gamma);
+ // delta 4: the per-hc offset is additive and per channel, so it broadcasts over tokens only
+ ggml_tensor * scale = ggml_add(ctx0, ggml_repeat_4d(ctx0, gate, n_embd, hc, nt, 1), layer.engram_hc_scale);
+ cb(scale, "engram_scale", il);
+
+ ggml_tensor * delta = ggml_mul(ctx0, ggml_repeat_4d(ctx0, signal, n_embd, hc, nt, 1), scale);
+
+ // A token with no valid n-gram contributes nothing. Zeroing its columns is not enough: the
+ // projection bias would still put a constant through the value path. This is the first token
+ // of every sequence, and any embedding (image) position.
+ ggml_tensor * any = ggml_sum_rows(ctx0, ggml_reshape_2d(ctx0, valid, n_cols, nt));
+ any = ggml_step(ctx0, any);
+ delta = ggml_mul(ctx0, delta, ggml_reshape_3d(ctx0, any, 1, 1, nt));
+ cb(delta, "engram_delta", il);
+
+ return ggml_add(ctx0, x, delta);
+}
+
+// reduce hc streams x[:,i,:] weighted by w[i,:] -> [n_embd, n_tokens]
+ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_reduce(
+ ggml_tensor * x,
+ ggml_tensor * weights,
+ int il) const {
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[2];
+ GGML_ASSERT(x->ne[0] == n_embd);
+ GGML_ASSERT(x->ne[1] == hc);
+
+ // the fused node is registered per layer, so the head reduce (il < 0) keeps the loop
+ if (cparams.fused_dsv4_hc_pre && il >= 0) {
+ ggml_tensor * result = ggml_dsv4_hc_pre(ctx0, x, weights);
+ res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, result, il});
+ return result;
+ }
+
+ ggml_tensor * result = nullptr;
+ for (int64_t ih = 0; ih < hc; ++ih) {
+ ggml_tensor * xh = ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih * x->nb[1]);
+ ggml_tensor * wh = ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih * weights->nb[0]);
+ ggml_tensor * cur = ggml_mul(ctx0, xh, wh);
+ result = result ? ggml_add(ctx0, result, cur) : cur;
+ }
+ return result;
+}
+
+llama_model_hy_v4_flash::graph::hc_pre
+llama_model_hy_v4_flash::graph::build_hc_pre(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base,
+ int il) const {
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[2];
+ GGML_ASSERT(x->ne[0] == n_embd && x->ne[1] == hc);
+
+ ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt);
+ ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps);
+ ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm);
+ cb(mixes, "hc_mixes", il);
+
+ ggml_tensor * scale_pre = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 0);
+ ggml_tensor * scale_post = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 1);
+ ggml_tensor * base_pre = hy_v4_flash_view_1d(ctx0, hc_base, hc, 0);
+ ggml_tensor * base_post = hy_v4_flash_view_1d(ctx0, hc_base, hc, hc);
+
+ ggml_tensor * pre = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, 0);
+ pre = ggml_mul(ctx0, pre, scale_pre);
+ pre = ggml_add(ctx0, pre, base_pre);
+ pre = ggml_sigmoid(ctx0, pre);
+ pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
+ cb(pre, "hc_pre", il);
+
+ ggml_tensor * post = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, hc);
+ post = ggml_mul(ctx0, post, scale_post);
+ post = ggml_add(ctx0, post, base_post);
+ post = ggml_sigmoid(ctx0, post);
+ post = ggml_scale(ctx0, post, hparams.hc_magnitude);
+ post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps);
+ cb(post, "hc_post_gate", il);
+
+ return { build_hc_reduce(x, pre, il), post };
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_post(
+ ggml_tensor * x,
+ ggml_tensor * residual,
+ ggml_tensor * post,
+ int il) const {
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[1];
+ GGML_ASSERT(x->ne[0] == n_embd);
+ GGML_ASSERT(residual->ne[1] == hc);
+
+ // comb is the identity here: each stream keeps its own residual, hy_v4 has no cross-stream mix
+ if (cparams.fused_dsv4_hc_post) {
+ ggml_tensor * result = ggml_dsv4_hc_post(ctx0, x, residual, post, nullptr);
+ res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, result, il});
+ return result;
+ }
+
+ ggml_tensor * out = nullptr;
+ for (int64_t i = 0; i < hc; ++i) {
+ ggml_tensor * res_i = ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], i * residual->nb[1]);
+ ggml_tensor * post_i = ggml_view_2d(ctx0, post, 1, nt, post->nb[1], i * post->nb[0]);
+ ggml_tensor * cur = ggml_add(ctx0, res_i, ggml_mul(ctx0, x, post_i));
+ cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, nt);
+ out = out ? ggml_concat(ctx0, out, cur, 1) : cur;
+ }
+
+ return out;
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_head(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base) const {
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[2];
+
+ ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt);
+ ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps);
+ ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm);
+ cb(mixes, "hc_head_mixes", -1);
+
+ ggml_tensor * pre = ggml_mul(ctx0, mixes, hc_scale);
+ pre = ggml_add(ctx0, pre, hc_base);
+ pre = ggml_sigmoid(ctx0, pre);
+ pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
+ cb(pre, "hc_head_pre", -1);
+
+ return build_hc_reduce(x, pre, -1);
+}
+
+// The memory composes an iSWA pair rather than being one, so the standard input cannot recover its
+// context from params.mctx on its own.
+class llama_model_hy_v4_flash::llm_graph_input_attn_idx : public llm_graph_input_attn_k_iswa {
+public:
+ using llm_graph_input_attn_k_iswa::llm_graph_input_attn_k_iswa;
+
+ const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const override {
+ return static_cast<const llama_memory_idx_iswa_context *>(params.mctx)->get_attn();
+ }
+};
+
+class llama_model_hy_v4_flash::llm_graph_input_kpool : public llm_graph_input_i {
+public:
+ llm_graph_input_kpool(const llama_memory_idx_iswa_context * mctx, uint32_t kpool) : mctx(mctx), kpool(kpool) {}
+ virtual ~llm_graph_input_kpool() = default;
+
+ void set_input(const llama_ubatch * ubatch) override {
+ mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch);
+ mctx->set_input_kpool(pool_cells, pool_idxs, pool_mask, tail_idxs, sel_mask,
+ new_pool_idxs, new_pool_rep, ubatch, new_pool_pos);
+ }
+
+ bool can_reuse(const llm_graph_params & params) override {
+ mctx = static_cast<const llama_memory_idx_iswa_context *>(params.mctx);
+
+ bool res = true;
+
+ res &= k_idxs->ne[0] == params.ubatch.n_tokens;
+ res &= pool_cells->ne[0] == mctx->get_n_kpool();
+ res &= pool_mask->ne[1] == params.ubatch.n_tokens;
+ res &= tail_idxs->ne[1] == params.ubatch.n_tokens;
+ res &= n_kv == mctx->get_idx()->get_n_kv();
+ res &= n_new == std::max(mctx->get_n_kpool_new(), 1u);
+
+ return res;
+ }
+
+ ggml_tensor * k_idxs = nullptr; // I64 [n_tokens]
+ ggml_tensor * pool_cells = nullptr; // I32 [n_pool] cell caching each pool's pooled key
+ ggml_tensor * pool_idxs = nullptr; // I32 [kpool, n_pool] member cells per pool
+ ggml_tensor * pool_mask = nullptr; // F32/F16 [n_pool, n_tokens]
+ ggml_tensor * tail_idxs = nullptr; // I32 [kpool, n_tokens] when the current group is forced
+ ggml_tensor * sel_mask = nullptr; // F32 [n_sel, 1, 1, n_tokens]
+ ggml_tensor * new_pool_idxs = nullptr; // I32 [kpool*n_new]
+ ggml_tensor * new_pool_rep = nullptr; // I64 [n_new]
+ ggml_tensor * new_pool_pos = nullptr; // I32 [4*n_new] first member's position per new pool
+
+ const llama_memory_idx_iswa_context * mctx;
+ const uint32_t kpool;
+ uint32_t n_new = 0;
+ uint32_t n_sel = 0;
+ uint32_t n_kv = 0;
+};
+
+llama_model_hy_v4_flash::llm_graph_input_kpool * llama_model_hy_v4_flash::graph::build_inp_kpool(
+ const llama_memory_idx_iswa_context * mctx_idx) {
+ const auto * mctx_lid = mctx_idx->get_idx();
+
+ const uint32_t kpool = hparams.indexer_kpool;
+ const uint32_t n_pool = mctx_idx->get_n_kpool();
+ // a ubatch that completes no pool still builds one dummy entry, so the graph does not change
+ // shape every kpool tokens
+ const uint32_t n_new = std::max(mctx_idx->get_n_kpool_new(), 1u);
+
+ const auto type_mask = cparams.fused_lid ? GGML_TYPE_F16 : GGML_TYPE_F32;
+
+ auto inp = std::make_unique<llm_graph_input_kpool>(mctx_idx, kpool);
+
+ inp->k_idxs = mctx_lid->build_input_k_idxs(ctx0, ubatch);
+ inp->pool_cells = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pool);
+ inp->pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_pool);
+ inp->pool_mask = ggml_new_tensor_2d(ctx0, type_mask, n_pool, n_tokens);
+ inp->tail_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, hparams.indexer_kpool_select_tail ? kpool : kpool - 1, n_tokens);
+ ggml_set_input(inp->pool_cells);
+ ggml_set_input(inp->pool_idxs);
+ ggml_set_input(inp->pool_mask);
+ ggml_set_input(inp->tail_idxs);
+
+ ggml_build_forward_expand(gf, inp->pool_cells);
+ ggml_build_forward_expand(gf, inp->pool_idxs);
+ ggml_build_forward_expand(gf, inp->pool_mask);
+ ggml_build_forward_expand(gf, inp->tail_idxs);
+
+ inp->n_kv = mctx_lid->get_n_kv();
+
+ {
+ // The reference keeps group_topk - 1 scored groups and forces the one the current token
+ // sits in, which the tail covers even when that group is complete.
+ const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail);
+ const int64_t n_sel = kpool*n_top_pool + (hparams.indexer_kpool_select_tail ? kpool : 0);
+ inp->n_sel = (uint32_t) n_sel;
+
+ if (!cparams.fused_top_k_mask) {
+ inp->sel_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_sel, 1, 1, n_tokens);
+ ggml_set_input(inp->sel_mask);
+ ggml_build_forward_expand(gf, inp->sel_mask);
+ }
+ }
+
+ inp->n_new = n_new;
+
+ inp->new_pool_idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kpool*n_new);
+ ggml_set_input(inp->new_pool_idxs);
+ inp->new_pool_rep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_new);
+ ggml_set_input(inp->new_pool_rep);
+ // 4 planes so the vision case can rotate pooled keys with mrope; text uses plane 0
+ inp->new_pool_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 4*n_new);
+ ggml_set_input(inp->new_pool_pos);
+ ggml_build_forward_expand(gf, inp->new_pool_pos);
+
+ return (llm_graph_input_kpool *) res->add_input(std::move(inp));
+}
+
+// rotate only the trailing n_rot of each indexer head, leaving the head's front half untouched
+ggml_tensor * llama_model_hy_v4_flash::graph::build_indexer_rope(
+ ggml_tensor * x, ggml_tensor * pos, int64_t n_idx, float freq_base, int il) const {
+ const int64_t n_nope = n_idx - n_rot;
+ GGML_ASSERT(n_nope > 0);
+
+ ggml_tensor * front = ggml_view_3d(ctx0, x, n_nope, x->ne[1], x->ne[2],
+ x->nb[1], x->nb[2], 0);
+ ggml_tensor * back = ggml_view_3d(ctx0, x, n_rot, x->ne[1], x->ne[2],
+ x->nb[1], x->nb[2], ggml_row_size(x->type, n_nope));
+
+ // the same base the layer's own attention uses, so a cparams override reaches the indexer too
+ back = ggml_rope_ext(ctx0, ggml_cont(ctx0, back), pos, nullptr, n_rot, rope_type, n_ctx_orig,
+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(back, "indexer_rope", il);
+
+ return ggml_concat(ctx0, ggml_cont(ctx0, front), back, 0);
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_kpool_select(
+ ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * inp_pos, float freq_base,
+ const llama_layer & layer,
+ const llama_memory_idx_iswa_context * mctx_idx, llm_graph_input_kpool * inp_kpool, int il) {
+ const auto * mctx_lid = mctx_idx->get_idx();
+
+ const int64_t n_idx_head = hparams.indexer_n_head;
+ const int64_t n_idx = hparams.indexer_head_size;
+ const int64_t kpool = hparams.indexer_kpool;
+ const int64_t n_pool = inp_kpool->pool_cells->ne[0];
+ const int64_t n_new = inp_kpool->n_new;
+ const int64_t n_kv = mctx_lid->get_n_kv();
+
+ // the query comes off the same lora residual the main attention uses, and is rotated at the
+ // token's own position
+ ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr);
+ iq = ggml_reshape_3d(ctx0, iq, n_idx, n_idx_head, n_tokens);
+ iq = build_indexer_rope(iq, inp_pos, n_idx, freq_base, il);
+ cb(iq, "indexer_q", il);
+
+ // key side: weight-only RMSNorm, unlike glm5-next's LayerNorm with a bias this model lacks
+ ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur);
+ ik = build_norm(ik, layer.indexer_k_norm, nullptr, LLM_NORM_RMS, il);
+ cb(ik, "indexer_k", il);
+
+ // what the pool averages is a projection of the normed key, and the weights come from another
+ // projection of it: glm5-next pools the key itself and gates off the hidden state
+ ggml_tensor * iv = ggml_mul_mat(ctx0, layer.indexer_kpool_value, ik);
+ ggml_tensor * ig = ggml_mul_mat(ctx0, layer.indexer_kpool_gate, ik);
+ cb(ig, "indexer_gate", il);
+
+ // cache rows hold value | gate | pooled
+ ggml_tensor * pzero = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_idx, n_tokens), 0.0f);
+ ggml_tensor * packed = ggml_concat(ctx0, ggml_concat(ctx0, iv, ig, 0), pzero, 0);
+ packed = ggml_reshape_3d(ctx0, packed, 3*n_idx, 1, n_tokens);
+ ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, packed, inp_kpool->k_idxs, il));
+
+ auto kpool_cache = mctx_idx->get_kpool_access(ctx0, il, n_idx);
+
+ const ggml_type type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
+
+ // pool the groups this ubatch completed; the last entry is a dummy when it completed none
+ ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_idxs);
+ rows = ggml_reshape_3d(ctx0, rows, 2*n_idx, kpool, n_new);
+
+ ggml_tensor * pk = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2], 0);
+ ggml_tensor * pg = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2],
+ ggml_row_size(rows->type, n_idx));
+
+ ggml_tensor * logits = ggml_add(ctx0, pg, layer.indexer_kpool_ape);
+ logits = ggml_cont(ctx0, ggml_permute(ctx0, logits, 1, 0, 2, 3)); // [kpool, n_idx, n_new]
+ // fold the pool axis into rows: soft_max maps ne[2] to a CUDA gridDim.y that caps at 65535
+ ggml_tensor * probs = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, logits, kpool, n_idx*n_new));
+ probs = ggml_reshape_3d(ctx0, probs, kpool, n_idx, n_new);
+
+ pk = ggml_cont(ctx0, ggml_permute(ctx0, pk, 1, 0, 2, 3));
+ ggml_tensor * pooled_new = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, pk)); // [1, n_idx, n_new]
+ pooled_new = ggml_reshape_2d(ctx0, pooled_new, n_idx, n_new);
+
+ // the pooled key is normed and then rotated at its group's first position, so the score keeps
+ // the same relative geometry as the main attention
+ pooled_new = build_norm(pooled_new, layer.indexer_kpool_norm, nullptr, LLM_NORM_RMS, il);
+ {
+ ggml_tensor * pos = ggml_view_1d(ctx0, inp_kpool->new_pool_pos, n_new, 0);
+ ggml_tensor * p3 = ggml_reshape_3d(ctx0, pooled_new, n_idx, 1, n_new);
+ pooled_new = ggml_reshape_2d(ctx0, build_indexer_rope(p3, pos, n_idx, freq_base, il), n_idx, n_new);
+ }
+ cb(pooled_new, "indexer_pool_k_new", il);
+
+ // write the fresh pooled keys into their rep rows, then read every pool back by cell: the
+ // older ones come from the rows earlier ubatches wrote
+ ggml_build_forward_expand(gf, kpool_cache.scatter_pooled(pooled_new, inp_kpool->new_pool_rep));
+
+ ggml_tensor * weights = ggml_mul_mat(ctx0, layer.indexer_proj, cur);
+ weights = ggml_scale(ctx0, weights, 1.0f / sqrtf(float(n_idx * n_idx_head)));
+ cb(weights, "indexer_weights", il);
+
+ ggml_tensor * score = nullptr;
+ if (cparams.fused_lid) {
+ // the fused indexer reads every pool's key straight from its cache cell, without gathering them first
+ // the list goes through a GPU copy: read straight from the host input buffer through src[4], the scores
+ // come out right but the decode diverges (root cause not found, bit-identical with the copy)
+ score = ggml_lightning_indexer_rows(ctx0, iq, kpool_cache.pooled_rows(), weights, inp_kpool->pool_mask,
+ ggml_cont(ctx0, inp_kpool->pool_cells));
+ res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
+ } else {
+ ggml_tensor * pooled = kpool_cache.gather_pooled(inp_kpool->pool_cells);
+ pooled = ggml_reshape_3d(ctx0, pooled, n_idx, 1, n_pool);
+ cb(pooled, "indexer_pool_k", il);
+
+ ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [n_idx, n_tokens, n_head]
+ ggml_tensor * k_p = ggml_permute(ctx0, pooled, 0, 2, 1, 3); // [n_idx, n_pool, 1]
+
+ ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_pool, n_tokens, n_head]
+ kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_head, n_tokens, n_pool]
+ score = ggml_relu(ctx0, kq);
+ score = ggml_mul(ctx0, score, weights);
+ score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_pool]
+ score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_pool, n_tokens, 1]
+ score = ggml_add(ctx0, score, inp_kpool->pool_mask);
+ }
+ cb(score, "indexer_score", il);
+
+ const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail);
+ ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered
+
+ // keep the cells of the picked pools and of the current group, -inf elsewhere; every
+ // retained cell is in the token's own sequence, so the kq_mask is not needed on top
+ ggml_tensor * tail = hparams.indexer_kpool_select_tail ? inp_kpool->tail_idxs : nullptr;
+ GGML_ASSERT(kpool*n_top_pool + (tail ? tail->ne[0] : 0) == (int64_t) inp_kpool->n_sel);
+
+ if (cparams.fused_top_k_mask) {
+ ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, type_mask, n_kv), 0.0f);
+ ggml_tensor * sel = ggml_top_k_mask(ctx0, keep, top_k, score, inp_kpool->pool_idxs, tail);
+ res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, sel, il});
+ cb(sel, "indexer_sel", il);
+
+ return sel;
+ }
+
+ // Backends without a fused top-k mask scatter the same selection by hand. Everything this arch
+ // changes about the indexer is upstream of score, so from here on it is glm5-next's path.
+ ggml_tensor * sel_idx = nullptr;
+ {
+ // sel_mask marks the first min(nv, n_top_pool) slots as the live pools, so put the picked
+ // set in descending score order to match it (ggml_top_k leaves it unordered)
+ ggml_tensor * sel_score = ggml_get_rows(ctx0,
+ ggml_reshape_3d(ctx0, score, 1, n_pool, n_tokens), top_k); // [1, n_top_pool, n_tokens]
+ ggml_tensor * sel_order = ggml_argsort(ctx0,
+ ggml_reshape_2d(ctx0, sel_score, n_top_pool, n_tokens), GGML_SORT_ORDER_DESC);
+
+ ggml_tensor * ordered = ggml_get_rows(ctx0,
+ ggml_reshape_3d(ctx0, ggml_cast(ctx0, top_k, GGML_TYPE_F32), 1, n_top_pool, n_tokens), sel_order);
+ ordered = ggml_cast(ctx0,
+ ggml_cont(ctx0, ggml_reshape_2d(ctx0, ordered, n_top_pool, n_tokens)), GGML_TYPE_I32);
+ cb(ordered, "indexer_top_k", il);
+
+ sel_idx = ggml_get_rows(ctx0, inp_kpool->pool_idxs,
+ ggml_reshape_1d(ctx0, ordered, n_top_pool*n_tokens)); // [kpool, n_top_pool*n_tokens]
+ sel_idx = ggml_reshape_2d(ctx0, sel_idx, kpool*n_top_pool, n_tokens);
+
+ if (tail) {
+ // the tail, with n_kv standing in for the cells it does not have yet
+ sel_idx = ggml_concat(ctx0, sel_idx, tail, 0);
+ }
+ }
+
+ const int64_t n_sel = sel_idx->ne[0];
+ GGML_ASSERT(n_sel == (int64_t) inp_kpool->n_sel);
+ GGML_ASSERT(inp_kpool->sel_mask && inp_kpool->sel_mask->ne[0] == n_sel &&
+ inp_kpool->sel_mask->ne[3] == n_tokens);
+
+ ggml_build_forward_expand(gf, sel_idx);
+
+ ggml_tensor * mask_all = ggml_new_tensor_4d(ctx0, type_mask, n_kv + n_sel, 1, 1, 1);
+ mask_all = ggml_fill(ctx0, mask_all, -INFINITY);
+ mask_all = ggml_repeat_4d(ctx0, mask_all, n_kv + n_sel, n_tokens, 1, 1);
+ mask_all = ggml_reshape_3d(ctx0, mask_all, 1, n_kv + n_sel, n_tokens);
+
+ ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, type_mask, n_sel, 1, 1, 1);
+ zeros = ggml_fill(ctx0, zeros, 0.0f);
+ zeros = ggml_repeat_4d(ctx0, zeros, n_sel, n_tokens, 1, 1);
+ zeros = ggml_reshape_3d(ctx0, zeros, 1, n_sel, n_tokens);
+
+ // Live slots (visible pools, real tail cells) address disjoint cells, and each dead slot writes
+ // its own dump row n_kv + slot, so a token's scatter indices stay unique:
+ // idx = dump + live*(idx - dump), live = exp(mask).
+ ggml_tensor * live = ggml_reshape_2d(ctx0, ggml_exp(ctx0, inp_kpool->sel_mask), n_sel, n_tokens);
+ ggml_tensor * dump = ggml_arange(ctx0, (float) n_kv, (float) (n_kv + n_sel), 1.0f);
+ ggml_tensor * idx_f = ggml_cast(ctx0, sel_idx, GGML_TYPE_F32);
+ idx_f = ggml_add(ctx0, ggml_mul(ctx0, ggml_sub(ctx0, idx_f, dump), live), dump);
+ sel_idx = ggml_cast(ctx0, idx_f, GGML_TYPE_I32);
+
+ ggml_tensor * sel = ggml_set_rows(ctx0, mask_all, zeros,
+ ggml_reshape_3d(ctx0, sel_idx, n_sel, n_tokens, 1));
+
+ sel = ggml_view_2d(ctx0, sel, n_kv, n_tokens, sel->nb[2], 0);
+ sel = ggml_cont(ctx0, sel);
+ cb(sel, "indexer_sel", il);
+
+ return sel;
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_attention(
+ const llama_model & model,
+ llm_graph_input_attn_k_iswa * inp_attn,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ const llama_memory_idx_iswa_context * mctx_idx,
+ llm_graph_input_kpool * inp_kpool,
+ int il) {
+ const auto & layer = model.layers[il];
+
+ const int64_t n_head_l = hparams.n_head(il);
+ const int64_t n_embd_head_k = hparams.n_embd_head_k_mla();
+ const int64_t n_embd_head_qk_rope = hparams.n_rot();
+ const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope;
+ const uint32_t kv_lora_rank = hparams.n_lora_kv;
+
+ const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k));
+ const float freq_base_l = model.get_rope_freq_base(cparams, il);
+
+ ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur);
+ qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ cb(qr, "q_resid", il);
+
+ ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr);
+
+ ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens,
+ ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l, 0);
+ ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens,
+ ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l,
+ ggml_row_size(q->type, n_embd_head_qk_nope));
+
+ ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
+ ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
+ ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+ ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
+
+ q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(q_pe, "q_pe", il);
+ k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(k_pe, "k_pe", il);
+
+ kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
+ cb(kv_cmpr, "kv_cmpr", il);
+
+ q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
+ ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
+ q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
+
+ // rope goes last so build_rope_shift() can shift the cache in place
+ ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
+
+ kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
+ ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
+ ggml_tensor * Vcur = kv_cmpr;
+
+ ggml_tensor * sel = nullptr;
+ // the draft block is a sparse layer and always owns an indexer, and is_indexer_full() only
+ // covers the trunk
+ if (inp_kpool != nullptr &&
+ (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il))) {
+ sel = build_kpool_select(cur, qr, inp_pos, freq_base_l, layer, mctx_idx, inp_kpool, il);
+ }
+
+ // wo is applied below so the gate can sit in front of it
+ ggml_tensor * attn;
+
+ // with a rotated K cache wv_b has to come after the de-rotation, so it is applied here and not in build_attn_mha
+ auto apply_wv_b = [&](ggml_tensor * a) {
+ a = ggml_reshape_3d(ctx0, a, kv_lora_rank, n_head_l, n_tokens);
+ a = ggml_permute(ctx0, a, 0, 2, 1, 3);
+ a = ggml_mul_mat(ctx0, layer.wv_b, a);
+ a = ggml_permute(ctx0, a, 0, 2, 1, 3);
+ a = ggml_cont(ctx0, a);
+ return ggml_reshape_2d(ctx0, a, a->ne[0]*a->ne[1], n_tokens);
+ };
+
+ if (sel != nullptr) {
+ const auto * mctx_base = mctx_idx->get_base();
+ const int64_t n_sel = inp_kpool->n_sel;
+
+ // this layer writes the cache itself, so it rotates q and k like build_attn does
+ ggml_tensor * k_rot = inp_attn->self_k_rot[il];
+ if (k_rot) {
+ Qcur = llama_mul_mat_hadamard(ctx0, Qcur, k_rot);
+ Kcur = llama_mul_mat_hadamard(ctx0, Kcur, k_rot);
+ }
+
+ // The indexer's selection replaces the kq_mask, so this layer cannot go through build_attn:
+ // write the latent row ourselves, then read the whole cache back and let the mask decide.
+ ggml_tensor * k = mctx_base->get_k(ctx0, il);
+ ggml_tensor * v = ggml_view_4d(ctx0, k, kv_lora_rank, k->ne[1], k->ne[2], k->ne[3],
+ k->nb[1], k->nb[2], k->nb[3], 0);
+
+ ggml_build_forward_expand(gf, Qcur);
+ ggml_build_forward_expand(gf, Kcur);
+ ggml_build_forward_expand(gf, mctx_base->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il));
+
+ // split the selection into the streams the K view carries
+ const int64_t n_stream = k->ne[3];
+ GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0);
+ sel = ggml_reshape_4d(ctx0, sel, sel->ne[0], sel->ne[1]/n_stream, 1, n_stream);
+
+ // The sparse hint makes Metal attend the selection with its vec kernels, one query row at a time. A
+ // prefill ubatch over a short cache view (n_kv) is faster through the dense matrix kernels with the same
+ // mask: pp2048 on M5 Max with dense everywhere, at depth 0/4k/8k/16k/32k: +5/+29/+12/0/-16%. Small batches
+ // (decode, MTP/draft verification) would get the dense vec kernel over all n_kv, so they keep the hint.
+ // Tuned for Metal only; CUDA/Vulkan/SYCL also read the hint.
+ const int64_t n_kv_hint = n_tokens >= 32 && sel->ne[0] <= HY_V4_FLASH_DENSE_FA_MAX_KV ? 0 : n_sel;
+ attn = build_attn_mha(Qcur, k, v, nullptr, sel, nullptr, layer.attn_sinks, k_rot ? nullptr : layer.wv_b,
+ n_kv_hint, kq_scale, il);
+
+ // A single-token selection is read by Metal straight from the indexer picks (the flash attention builds
+ // the cell list, see ggml_metal_op_hy4_mode4_ok): keep them alive until the attention runs, and make the
+ // read visible to the concurrency tracking. Other backends ignore these extra sources. Not for a batch:
+ // the [n_pool, n_tokens] scores would stay allocated through the attention for nothing.
+ if (n_tokens == 1) {
+ const ggml_tensor * tkm = sel;
+ while (tkm->view_src) {
+ tkm = tkm->view_src;
+ }
+ ggml_tensor * fa = res->get_fused_nodes().empty() ? nullptr : res->get_fused_nodes().back().tensor;
+ if (fa && fa->op == GGML_OP_FLASH_ATTN_EXT && tkm->op == GGML_OP_TOP_K_MASK && fa->src[6] == nullptr &&
+ fa->src[3] && fa->src[3]->view_src == tkm) {
+ for (int j = 1; j <= 4; ++j) {
+ fa->src[5 + j] = tkm->src[j];
+ }
+ }
+ }
+ if (k_rot) {
+ attn = llama_mul_mat_hadamard(ctx0, attn, k_rot);
+ attn = apply_wv_b(attn);
+ }
+ } else {
+ // null unless the K cache is quantized
+ ggml_tensor * k_rot = hparams.is_swa(il) ? inp_attn->self_k_rot_swa[il] : inp_attn->self_k_rot[il];
+
+ attn = build_attn(inp_attn,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, k_rot ? nullptr : layer.wv_b, kq_scale, il);
+ if (k_rot) {
+ attn = apply_wv_b(attn);
+ }
+ }
+ cb(attn, "attn_kqv", il);
+
+ ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur);
+ gate = ggml_sigmoid(ctx0, gate);
+ attn = ggml_mul(ctx0, attn, gate);
+ cb(attn, "attn_gated", il);
+
+ ggml_tensor * out = build_lora_mm(layer.wo, attn);
+ out = build_norm(out, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il);
+ cb(out, "attn_out", il);
+
+ return out;
+}
+
+ggml_tensor * llama_model_hy_v4_flash::graph::build_latent_moe(
+ const llama_model & model,
+ ggml_tensor * cur,
+ int il) const {
+ const auto & layer = model.layers[il];
+
+ // the router scores the full-width input, the experts take the latent one
+ ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, cur);
+ cb(logits, "ffn_moe_logits", il);
+
+ ggml_tensor * routed_in = layer.ffn_routed_down
+ ? ggml_mul_mat(ctx0, layer.ffn_routed_down, cur)
+ : cur;
+
+ ggml_tensor * moe_out = build_moe_ffn(routed_in,
+ nullptr,
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ layer.ffn_exp_probs_b,
+ n_expert, n_expert_used,
+ LLM_FFN_SILU, hparams.expert_weights_norm,
+ hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il,
+ logits);
+ cb(moe_out, "ffn_moe_out", il);
+
+ if (layer.ffn_routed_up) {
+ moe_out = ggml_mul_mat(ctx0, layer.ffn_routed_up, moe_out);
+ cb(moe_out, "ffn_moe_up", il);
+ }
+
+ ggml_tensor * shexp = build_ffn(cur,
+ layer.ffn_up_shexp, NULL, NULL,
+ layer.ffn_gate_shexp, NULL, NULL,
+ layer.ffn_down_shexp, NULL, NULL,
+ NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(shexp, "ffn_shexp", il);
+
+ return ggml_add(ctx0, moe_out, shexp);
+}
+
+llama_model_hy_v4_flash::graph::graph(const llama_model & model, const llm_graph_params & params) :
+ llm_graph_context(params) {
+ const int64_t hc = hparams.dsv4_hc_mult;
+
+ ggml_tensor * cur;
+
+ ggml_tensor * inp = build_inp_embd(model.tok_embd);
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ // With an indexer the memory is a composition, so the attention context has to be fetched out
+ // of it and the input told how to find it again on reuse.
+ const auto * mctx_idx = hparams.indexer_top_k > 0
+ ? static_cast<const llama_memory_idx_iswa_context *>(mctx) : nullptr;
+
+ llm_graph_input_attn_k_iswa * inp_attn = mctx_idx
+ ? build_attn_inp_k_iswa(
+ std::make_unique<llm_graph_input_attn_idx>(hparams, cparams, mctx_idx->get_attn()),
+ mctx_idx->get_attn())
+ : build_attn_inp_k_iswa();
+
+ llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr;
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ // Engram gathers go right after the token embeddings so they share the first graph split and
+ // the lazily read table stays on the host. An embedding-only ubatch has no token ids to hash,
+ // which matches the reference skipping Engram at its masked (image) positions.
+ auto inp_eng = std::make_unique<llm_graph_input_hy_v4_flash>(
+ static_cast<const llama_model_hy_v4_flash &>(model), inp_attn->mctx->get_base());
+ std::vector<ggml_tensor *> engram_rows;
+ std::vector<ggml_tensor *> engram_valid;
+ for (int il = 0; ubatch.token && hparams.ple_n_heads > 0 && il < n_layer; ++il) {
+ if (!hparams.is_ple(il)) {
+ continue;
+ }
+ const int64_t n_cols = hparams.ple_n_heads;
+
+ ggml_tensor * ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cols*n_tokens);
+ ggml_set_input(ids);
+ ggml_set_name(ids, ("engram_ids-" + std::to_string(il)).c_str());
+ inp_eng->engram_ids.push_back(ids);
+
+ ggml_tensor * valid = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cols*n_tokens);
+ ggml_set_input(valid);
+ ggml_set_name(valid, ("engram_valid-" + std::to_string(il)).c_str());
+ inp_eng->engram_valid.push_back(valid);
+ engram_valid.push_back(valid);
+
+ ggml_tensor * rows = ggml_get_rows(ctx0, model.per_layer_tok_embd, ids);
+ rows = ggml_reshape_3d(ctx0, rows, hparams.ple_head_dim, n_cols, n_tokens);
+ cb(rows, "engram_rows", il);
+ ggml_build_forward_expand(gf, rows);
+ engram_rows.push_back(rows);
+ }
+ res->add_input(std::move(inp_eng));
+
+ // one embedding feeds hc parallel residual streams
+ ggml_tensor * inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens);
+ inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
+ cb(inpL, "hc_init", -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ // Engram writes into the streams before the attention pre-mix reads them
+ const int e = static_cast<const llama_model_hy_v4_flash &>(model).engram_index(il);
+ if (e >= 0 && !engram_rows.empty()) {
+ inpL = build_engram(model, inpL, engram_rows[e], engram_valid[e], il);
+ cb(inpL, "engram_out", il);
+ }
+
+ ggml_tensor * residual = inpL;
+ hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
+ cur = build_norm(attn_hc.cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il);
+
+ inpL = build_hc_post(cur, residual, attn_hc.post, il);
+ cb(inpL, "hc_attn_out", il);
+
+ residual = inpL;
+ hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
+ cur = build_norm(ffn_hc.cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ if ((uint32_t) il < hparams.n_layer_dense_lead) {
+ cur = build_ffn(cur,
+ layer.ffn_up, NULL, NULL,
+ layer.ffn_gate, NULL, NULL,
+ layer.ffn_down, NULL, NULL,
+ NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ } else {
+ cur = build_latent_moe(model, cur, il);
+ }
+
+ cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_out", il);
+
+ inpL = build_hc_post(cur, residual, ffn_hc.post, il);
+ cb(inpL, "l_out", il);
+ }
+
+ // prune to the requested rows once, after every stream is done. Unmasked nextn embeddings want
+ // one row per token, so in that mode the narrowing waits until the state has been captured.
+ if (crop_before_nextn(inp_out_ids)) {
+ ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd * hc, n_tokens);
+ flat = ggml_get_rows(ctx0, flat, inp_out_ids);
+ inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs);
+ }
+
+ cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base);
+ cb(cur, "hc_head", -1);
+
+ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+
+ // the draft block consumes this exact state through its hnorm, so expose it for the caller to
+ // hand back via llama_batch_ext_set_embd_state()
+ cb(cur, "h_nextn", -1);
+ res->t_h_nextn = cur;
+
+ if (crop_after_nextn(inp_out_ids)) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
+
+llama_model_hy_v4_flash::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :
+ graph(params, no_build{}) {
+ GGML_ASSERT(hparams.n_layer_nextn > 0 && "hy_v4_flash MTP needs the draft block, convert without --no-mtp");
+ GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
+ cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
+ "nextn_layer_offset out of range [0, n_layer_nextn)");
+
+ const int il = hparams.n_layer() + cparams.nextn_layer_offset;
+ const auto & layer = model.layers[il];
+
+ GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm &&
+ "hy_v4_flash MTP block is missing");
+
+ const auto * mctx_idx = hparams.indexer_top_k > 0
+ ? static_cast<const llama_memory_idx_iswa_context *>(mctx) : nullptr;
+
+ llm_graph_input_attn_k_iswa * inp_attn = mctx_idx
+ ? build_attn_inp_k_iswa(
+ std::make_unique<llm_graph_input_attn_idx>(hparams, cparams, mctx_idx->get_attn()),
+ mctx_idx->get_attn())
+ : build_attn_inp_k_iswa();
+
+ llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr;
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ // the draft takes the next token's embedding alongside the trunk's final hidden state, which
+ // is the same tensor the trunk's own output head reads: already collapsed over the hc streams
+ // and already passed through output_norm
+ auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), n_embd);
+
+ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ ggml_set_input(inp->tokens);
+
+ inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+ ggml_set_input(inp->embd);
+
+ inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
+ ggml_set_input(inp->h);
+ ggml_set_name(inp->h, "mtp_h_input");
+
+ ggml_tensor * tok_embd = ubatch.token
+ ? ggml_get_rows(ctx0, model.tok_embd, inp->tokens)
+ : inp->embd;
+ ggml_tensor * prev_h = inp->h;
+ res->add_input(std::move(inp));
+
+ ggml_tensor * cur = ggml_concat(ctx0,
+ build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il),
+ build_norm(prev_h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il), 0);
+ cur = ggml_mul_mat(ctx0, layer.nextn.eh_proj, cur);
+ cb(cur, "mtp_eh_proj", il);
+
+ // from here it is an ordinary sparse layer, except that the residual is the plain hidden state
+ // rather than one of the trunk's hc streams
+ ggml_tensor * residual = cur;
+ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il);
+ cur = ggml_add(ctx0, cur, residual);
+
+ if (crop_before_nextn(inp_out_ids)) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ residual = cur;
+ cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cur = build_latent_moe(model, cur, il);
+ cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il);
+ cur = ggml_add(ctx0, cur, residual);
+
+ cur = build_norm(cur, layer.nextn.shared_head_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "mtp_result_norm", -1);
+
+ // the draft's own post-norm state, so a second draft step can be fed from this one
+ res->t_h_nextn = cur;
+
+ if (crop_after_nextn(inp_out_ids)) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ cur = build_lora_mm(model.output, cur);
+ cb(cur, "mtp_result_output", -1);
+
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
diff --git a/src/models/models.h b/src/models/models.h
index 1fc7cae1d..f8cdc66fc 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -9,6 +9,7 @@
#include <map>
class llama_memory_hybrid_idx_context;
+class llama_memory_idx_iswa_context;
// ref: https://github.com/ggml-org/llama.cpp/pull/28068
static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) {
@@ -2370,6 +2371,122 @@ struct llama_model_hy_v4 : public llama_model_base {
};
+// HY_V4_FLASH: hy_v4's iHC + gated MLA + sinks, with a hybrid attention stack (sparse full
+// layers alternating with sliding-window layers, each with its own head count and rope base),
+// sandwich norms and routed experts that run in a latent space.
+struct llama_model_hy_v4_flash : public llama_model_base {
+ llama_model_hy_v4_flash(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ // -1 when the layer has no Engram module, else its index among the Engram layers
+ int engram_index(int il) const;
+
+ // n-gram hash of module e. Writes one row id per (token, column) into ids, and alongside it the
+ // per-column validity: a short history is padded, as in deepseek41, so only a position with no
+ // n-gram at all (an embedding position) is marked invalid.
+ void engram_hash(int e, const llama_ubatch & ubatch, const std::vector<llama_token> & prev,
+ std::vector<int32_t> & ids, std::vector<float> & valid) const;
+
+ class llm_graph_input_kpool;
+
+ // the memory only composes an iSWA pair, so the standard input cannot find its context in
+ // params.mctx by itself
+ class llm_graph_input_attn_idx;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model & model, const llm_graph_params & params);
+
+ // the MTP graph reuses the helpers without running the trunk
+ struct no_build {};
+ graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {}
+
+ // iHC, same layout as hy_v4: pre reduces the hc streams to one and returns the per-stream
+ // post gates, post writes the sublayer output back, head collapses before the final norm.
+ struct hc_pre {
+ ggml_tensor * cur;
+ ggml_tensor * post;
+ };
+
+ // reduce the hc streams to one, weighted by the pre gates
+ ggml_tensor * build_hc_reduce(
+ ggml_tensor * x,
+ ggml_tensor * weights,
+ int il) const;
+
+ hc_pre build_hc_pre(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base,
+ int il) const;
+
+ ggml_tensor * build_hc_post(
+ ggml_tensor * x,
+ ggml_tensor * residual,
+ ggml_tensor * post,
+ int il) const;
+
+ ggml_tensor * build_hc_head(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base) const;
+
+ // mctx_idx and inp_kpool are null when the file carries no indexer, and the sparse layers
+ // then attend their whole context
+ ggml_tensor * build_attention(
+ const llama_model & model,
+ llm_graph_input_attn_k_iswa * inp_attn,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ const llama_memory_idx_iswa_context * mctx_idx,
+ llm_graph_input_kpool * inp_kpool,
+ int il);
+
+ llm_graph_input_kpool * build_inp_kpool(const llama_memory_idx_iswa_context * mctx_idx);
+
+ // rotate only the trailing n_rot of each indexer head
+ ggml_tensor * build_indexer_rope(ggml_tensor * x, ggml_tensor * pos, int64_t n_idx,
+ float freq_base, int il) const;
+
+ // score the pooled keys against the query and keep the best groups, plus the group the
+ // current token sits in
+ ggml_tensor * build_kpool_select(
+ ggml_tensor * cur,
+ ggml_tensor * qr,
+ ggml_tensor * inp_pos,
+ float freq_base,
+ const llama_layer & layer,
+ const llama_memory_idx_iswa_context * mctx_idx,
+ llm_graph_input_kpool * inp_kpool,
+ int il);
+
+ // routed experts run at n_expert_latent, the router and the shared expert at n_embd
+ ggml_tensor * build_latent_moe(
+ const llama_model & model,
+ ggml_tensor * cur,
+ int il) const;
+
+ // Engram writes its delta into the hc residual streams before the layer's attention pre-mix
+ ggml_tensor * build_engram(
+ const llama_model & model,
+ ggml_tensor * x,
+ ggml_tensor * rows,
+ ggml_tensor * valid,
+ int il) const;
+ };
+
+ // the draft block: one sparse layer fed the next token's embedding and the trunk's final
+ // hidden state, with no hyper-connections of its own
+ struct graph_mtp : public graph {
+ graph_mtp(const llama_model & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_hunyuan_vl : public llama_model_base {
llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 3ed418140..7f7b0904a 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
extern "C" {
// broadcast:
// ne3 % ne33 == 0
//
+ // same, over the K rows listed in rows (I32 [n_out]): k holds every row ([n_embd, 1, n_rows_k, n_stream]),
+ // the result and the mask have one entry per list element (the gather is fused into the scoring)
+ GGML_API struct ggml_tensor * ggml_lightning_indexer_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * weights,
+ struct ggml_tensor * mask,
+ struct ggml_tensor * rows);
+
GGML_API struct ggml_tensor * ggml_lightning_indexer(
struct ggml_context * ctx,
struct ggml_tensor * q,
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index dbdde6997..463f6801a 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
void ggml_compute_forward_lightning_indexer(
const ggml_tensor * k = dst->src[1];
const ggml_tensor * w = dst->src[2]; // weights
const ggml_tensor * m = dst->src[3]; // mask
+ const ggml_tensor * r = dst->src[4]; // optional list of the K rows to score
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT( q->type == GGML_TYPE_F32);
void ggml_compute_forward_lightning_indexer(
const int n_head = q->ne[1];
const int n_tokens = q->ne[2];
const int n_stream = q->ne[3];
- const int n_kv = k->ne[2];
+ const int n_kv = r ? (int) r->ne[0] : (int) k->ne[2];
+ const int32_t * rows = r ? (const int32_t *) r->data : nullptr;
ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float;
GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type");
void ggml_compute_forward_lightning_indexer(
const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3);
float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 );
for (int ik = ir0; ik < ir1; ++ik) {
- char * k_row = (char *) k->data + ik*nbk2 + s*nbk3;
+ const int64_t ir = rows ? rows[ik] : ik;
+ GGML_ASSERT(ir >= 0 && ir < k->ne[2]);
+ char * k_row = (char *) k->data + ir*nbk2 + s*nbk3;
if (k_to_float) {
k_to_float(k_row, k_row_f32, n_embd);
} else {
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 06956ce7a..bd2f5590b 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_SOLVE_TRI:
return true;
case GGML_OP_LIGHTNING_INDEXER:
- return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
+ // the row-list form (src[4]) is Metal/CPU only
+ return op->src[4] == nullptr && ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
case GGML_OP_TOP_K_MASK:
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index 597c126ca..073872b90 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
#import <Metal/Metal.h>
+#include <mach/mach_time.h>
+#include <math.h>
+
+// hy4 profiling knobs, read once:
+// GGML_METAL_N_MAIN nodes encoded+committed by the calling thread first: <1 = fraction of the graph, >=1 = node count
+// GGML_METAL_CB_TIMING=1 log host encode and GPU start/end times of every graph command buffer
+static float hy4_metal_n_main(void) {
+ static float v = -1.0f;
+ if (v < 0.0f) {
+ v = 0.0f;
+ const char * env = getenv("GGML_METAL_N_MAIN");
+ if (env != NULL && *env != '\0') {
+ char * end = NULL;
+ const float f = strtof(env, &end);
+ if (*end == '\0' && isfinite(f) && f >= 0.0f && f <= 1e6f) {
+ v = f;
+ } else {
+ GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_MAIN='%s'\n", __func__, env);
+ }
+ }
+ }
+ return v;
+}
+