Hy4-flash-GGUF / patch /hy4-flash-mac-minimal.patch
woodchen7's picture
Upload hy4-flash-mac-minimal.patch
125343c verified
Raw History Blame Contribute Delete
291 kB
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 3ed418140..7f7b0904a 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -2711,6 +2711,16 @@ extern "C" { // broadcast: // ne3 % ne33 == 0 // + // same, over the K rows listed in rows (I32 [n_out]): k holds every row ([n_embd, 1, n_rows_k, n_stream]), + // the result and the mask have one entry per list element (the gather is fused into the scoring) + GGML_API struct ggml_tensor * ggml_lightning_indexer_rows( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * weights, + struct ggml_tensor * mask, + struct ggml_tensor * rows); + GGML_API struct ggml_tensor * ggml_lightning_indexer( struct ggml_context * ctx, struct ggml_tensor * q, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index dbdde6997..463f6801a 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -12457,6 +12457,7 @@ void ggml_compute_forward_lightning_indexer( const ggml_tensor * k = dst->src[1]; const ggml_tensor * w = dst->src[2]; // weights const ggml_tensor * m = dst->src[3]; // mask + const ggml_tensor * r = dst->src[4]; // optional list of the K rows to score GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT( q->type == GGML_TYPE_F32); @@ -12484,7 +12485,8 @@ void ggml_compute_forward_lightning_indexer( const int n_head = q->ne[1]; const int n_tokens = q->ne[2]; const int n_stream = q->ne[3]; - const int n_kv = k->ne[2]; + const int n_kv = r ? (int) r->ne[0] : (int) k->ne[2]; + const int32_t * rows = r ? (const int32_t *) r->data : nullptr; ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float; GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type"); @@ -12509,7 +12511,9 @@ void ggml_compute_forward_lightning_indexer( const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3); float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 ); for (int ik = ir0; ik < ir1; ++ik) { - char * k_row = (char *) k->data + ik*nbk2 + s*nbk3; + const int64_t ir = rows ? rows[ik] : ik; + GGML_ASSERT(ir >= 0 && ir < k->ne[2]); + char * k_row = (char *) k->data + ir*nbk2 + s*nbk3; if (k_to_float) { k_to_float(k_row, k_row_f32, n_embd); } else { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 06956ce7a..bd2f5590b 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -6209,7 +6209,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_SOLVE_TRI: return true; case GGML_OP_LIGHTNING_INDEXER: - return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op); + // the row-list form (src[4]) is Metal/CPU only + return op->src[4] == nullptr && ggml_cuda_lightning_indexer_supported(dev_ctx->device, op); case GGML_OP_TOP_K_MASK: return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32; diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 597c126ca..073872b90 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -12,6 +12,57 @@ #import <Metal/Metal.h> +#include <mach/mach_time.h> +#include <math.h> + +// hy4 profiling knobs, read once: +// GGML_METAL_N_MAIN nodes encoded+committed by the calling thread first: <1 = fraction of the graph, >=1 = node count +// GGML_METAL_CB_TIMING=1 log host encode and GPU start/end times of every graph command buffer +static float hy4_metal_n_main(void) { + static float v = -1.0f; + if (v < 0.0f) { + v = 0.0f; + const char * env = getenv("GGML_METAL_N_MAIN"); + if (env != NULL && *env != '\0') { + char * end = NULL; + const float f = strtof(env, &end); + if (*end == '\0' && isfinite(f) && f >= 0.0f && f <= 1e6f) { + v = f; + } else { + GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_MAIN='%s'\n", __func__, env); + } + } + } + return v; +} + +static bool hy4_metal_cb_timing(void) { + static int v = -1; + if (v < 0) { + v = getenv("GGML_METAL_CB_TIMING") != NULL; + } + return v; +} + +// GGML_METAL_OP_PROFILE=1 encode every op (or fused group) in its own command buffer and log its GPU time; +// serializes the graph, so only relative costs are meaningful +static bool hy4_metal_op_profile(void) { + static int v = -1; + if (v < 0) { + v = getenv("GGML_METAL_OP_PROFILE") != NULL; + } + return v; +} + +// same timebase as MTLCommandBuffer GPUStartTime / GPUEndTime +static double hy4_metal_host_time(void) { + static mach_timebase_info_data_t tb; + if (tb.denom == 0) { + mach_timebase_info(&tb); + } + return (double) mach_absolute_time() * tb.numer / tb.denom * 1e-9; +} + #undef MIN #undef MAX #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -59,6 +110,8 @@ struct ggml_cgraph * gf; + uint64_t hy4_graph_id; // GGML_METAL_CB_TIMING: graph counter + // the callback given to the thread pool void (^encode_async)(size_t ith); @@ -482,7 +535,9 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * } // number of nodes encoded by the main thread (empirically determined) - const int n_main = MAX(64, 0.1*gf->n_nodes); + const float n_main_env = hy4_metal_n_main(); + const int n_main = n_main_env <= 0.0f ? MAX(64, 0.1*gf->n_nodes) : + n_main_env < 1.0f ? MAX(1, (int) (n_main_env*gf->n_nodes)) : (int) n_main_env; // number of threads in addition to the main thread const int n_cb = ctx->n_cb; @@ -499,6 +554,7 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; + ctx->hy4_graph_id++; if (ctx->n_cb == 0) { // single-threaded encoding: the whole graph is encoded by one command buffer @@ -724,7 +780,8 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { // when fusion stats are collected the graph must be encoded by a single thread so the // counters are race-free; override whatever the caller requested - if (ggml_metal_fusion_info_stats(ctx->finfo)) { + // GGML_METAL_OP_PROFILE commits one buffer per op, which only keeps dependency order from one thread + if (ggml_metal_fusion_info_stats(ctx->finfo) || hy4_metal_op_profile()) { n_cb = 0; } @@ -749,6 +806,8 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { const int n_nodes_per_cb = ctx->n_nodes_per_cb; + const double t_enc0 = hy4_metal_cb_timing() ? hy4_metal_host_time() : 0.0; + int idx_start = 0; int idx_end = n_nodes_0; @@ -759,6 +818,54 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { id<MTLCommandBuffer> cmd_buf = ctx->cmd_bufs[cb_idx].obj; + if (hy4_metal_op_profile()) { + // the assigned buffer stays empty; commit it first so the per-op buffers are not queued behind it + [cmd_buf commit]; + + id<MTLCommandQueue> queue = ggml_metal_device_get_queue(ctx->dev); + struct ggml_cgraph * gf = ctx->gf; + const unsigned long long gid = ctx->hy4_graph_id; + + int * idxs = (int *) malloc(sizeof(int) * (size_t) MAX(1, idx_end - idx_start)); + int n_idxs = 0; + for (int i = idx_start; i < idx_end; ++i) { + if (!ggml_op_is_empty(gf->nodes[i]->op) && !ggml_is_empty(gf->nodes[i])) { + idxs[n_idxs++] = i; + } + } + + id<MTLCommandBuffer> last = nil; + for (int k = 0; k < n_idxs; ) { + id<MTLCommandBuffer> cb = [queue commandBufferWithUnretainedReferences]; + + ggml_metal_op_t ctx_op = ggml_metal_op_init(ctx->dev, cb, gf, ctx->finfo, idxs[k], idx_end, false, false, 0); + const int res = MAX(1, ggml_metal_op_encode(ctx_op, 0)); + ggml_metal_op_free(ctx_op); + + const struct ggml_tensor * node = gf->nodes[idxs[k]]; + const struct ggml_tensor * last_node = gf->nodes[idxs[MIN(k + res, n_idxs) - 1]]; + char label[256]; + snprintf(label, sizeof(label), "%s|%s|%s|%s", ggml_op_desc(node), ggml_get_name(node), + node->src[0] ? ggml_get_name(node->src[0]) : "-", ggml_get_name(last_node)); + NSString * lbl = [NSString stringWithUTF8String:label]; + const int i0 = idxs[k]; + [cb addCompletedHandler:^(id<MTLCommandBuffer> b) { + if (b.status == MTLCommandBufferStatusError) { + GGML_LOG_ERROR("hy4_op g=%llu i=%d failed: %s\n", gid, i0, [[b.error localizedDescription] UTF8String]); + } + GGML_LOG_INFO("hy4_op g=%llu i=%d n=%d gpu0=%.7f gpu1=%.7f %s\n", gid, i0, res, b.GPUStartTime, b.GPUEndTime, [lbl UTF8String]); + }]; + [cb commit]; + last = cb; + k += res; + } + free(idxs); + if (last) { + [last waitUntilCompleted]; + } + return; + } + ggml_metal_op_t ctx_op = ggml_metal_op_init( ctx->dev, cmd_buf, @@ -781,6 +888,17 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ggml_metal_op_free(ctx_op); + if (hy4_metal_cb_timing()) { + const double t_enc1 = hy4_metal_host_time(); + const unsigned long long gid = ctx->hy4_graph_id; + const int is_main = cb_idx == n_cb_l; + const int n_nodes = idx_end - idx_start; + [cmd_buf addCompletedHandler:^(id<MTLCommandBuffer> cb) { + GGML_LOG_INFO("hy4_cbt g=%llu cb=%d main=%d nodes=%d enc0=%.6f enc1=%.6f gpu0=%.6f gpu1=%.6f\n", + gid, cb_idx, is_main, n_nodes, t_enc0, t_enc1, cb.GPUStartTime, cb.GPUEndTime); + }]; + } + if (cb_idx < 2 || ctx->abort_callback == NULL) { [cmd_buf commit]; } diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 463d57c7b..cfa020827 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -588,6 +588,60 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_met return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ggml_metal_library_t lib, const ggml_tensor * pre, bool with_mul, bool with_gate) { + char base[256]; + char name[256]; + + const bool gated = ggml_get_op_params_i32(pre, 1) != 0; + if (with_gate) { + GGML_ASSERT(!gated && with_mul); + snprintf(base, 256, "kernel_hy4_hc_gate_pre_rms_mul_f32"); + } else { + snprintf(base, 256, "kernel_hy4_hc_pre%s_rms%s_f32", gated ? "_gated" : "", with_mul ? "_mul" : ""); + } + snprintf(name, 256, "%s_n_hc=%d", base, (int) pre->src[0]->ne[1]); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, (int32_t) pre->src[0]->ne[1], FC_DSV4_HC + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + res.smem = 32*sizeof(float); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_add_rms(ggml_metal_library_t lib) { + const char * name = "kernel_hy4_add_rms_norm_mul_f32"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + res.smem = 32*sizeof(float); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_post_rms(ggml_metal_library_t lib) { + const char * name = "kernel_hy4_hc_post_rms_f32"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + res.smem = 32*sizeof(float); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv( ggml_metal_library_t lib, const ggml_tensor * op, int32_t nc, bool use_silu) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); @@ -1095,9 +1149,19 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta } break; case GGML_TYPE_Q8_0: { - nsg = N_SG_Q8_0; - nr0 = N_R0_Q8_0; - smem = 32*sizeof(float)*N_R0_Q8_0; + if (ne00 <= 512 && tsrc1 == GGML_TYPE_F32) { + // short rows (e.g. per-head MLA projections): whole rows per simdgroup, no cross-simdgroup + // reduce; 4 rows per simdgroup up to 256 columns, 2 above + nsg = N_SG_Q8_0_SHORT; + const int r = ne00 <= 256 ? 4 : 2; + nr0 = r*nsg; // rows per threadgroup + smem = 0; + suffix = r == 4 ? "_short_r4" : "_short_r2"; + } else { + nsg = N_SG_Q8_0; + nr0 = N_R0_Q8_0; + smem = 32*sizeof(float)*N_R0_Q8_0; + } } break; case GGML_TYPE_MXFP4: { @@ -2340,6 +2404,25 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale(ggml_ return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_rope_concat(ggml_metal_library_t lib) { + const char * base = "kernel_hy4_rope_concat_f32"; + const char * name = "kernel_hy4_rope_concat_f32_imrope=0_is_back=0"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + ggml_metal_cv_set_bool(cv, false, FC_ROPE + 0); + ggml_metal_cv_set_bool(cv, false, FC_ROPE + 1); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK); @@ -2458,6 +2541,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d(ggml_m return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hc_gate(ggml_metal_library_t lib) { + const char * name = "kernel_hc_gate_f32"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sigmoid_mul(ggml_metal_library_t lib) { + const char * name = "kernel_sigmoid_mul_f32"; + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake(ggml_metal_library_t lib, enum ggml_type type) { GGML_ASSERT(type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index e096d611b..2ba9e3fbc 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -59,6 +59,7 @@ struct ggml_metal_pipeline_with_params { }; int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline); +int ggml_metal_pipeline_simd_width (struct ggml_metal_pipeline_with_params pipeline); // // MTLCommandBuffer wrapper @@ -168,6 +169,12 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_tran struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hc_gate (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sigmoid_mul (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_pre_rms (ggml_metal_library_t lib, const struct ggml_tensor * pre, bool with_mul, bool with_gate); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_hc_post_rms (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_add_rms (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_hy4_rope_concat (ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 68a9addb4..9a3799de1 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -100,6 +100,10 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup; } +int ggml_metal_pipeline_simd_width(struct ggml_metal_pipeline_with_params pipeline) { + return pipeline.pipeline->obj.threadExecutionWidth; +} + // // MTLLibrary collection (one library per op-source, compiled separately) // @@ -1822,6 +1826,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te !ggml_is_contiguous_rows(op->src[3])) { return false; } + if (op->src[4] && (op->src[4]->type != GGML_TYPE_I32 || !ggml_is_contiguous(op->src[4]))) { + return false; + } switch (op->src[1]->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp index 020cad29d..8d022e417 100644 --- a/ggml/src/ggml-metal/ggml-metal-fusion.cpp +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -7,6 +7,7 @@ #include <algorithm> #include <cstddef> +#include <cstdint> #include <cstring> #include <set> #include <string> @@ -288,6 +289,363 @@ static bool ggml_metal_fusion_check_gdn_cache( return true; } +// UNARY (sigmoid) + MUL: dst = x * sigmoid(g) element-wise, x and g of the same contiguous F32 shape. +// The attention output gate of hy_v4_flash. +static bool ggml_metal_fusion_check_sigmoid_mul( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_UNUSED(mode); + + const ggml_tensor * sig = nodes[0]; + const ggml_tensor * mul = nodes[1]; + + if (ggml_get_unary_op(sig) != GGML_UNARY_OP_SIGMOID || mul->src[1] != sig || mul->src[0] == sig) { + return false; + } + + const ggml_tensor * g = sig->src[0]; + const ggml_tensor * x = mul->src[0]; + + if (g->type != GGML_TYPE_F32 || x->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || + !ggml_are_same_shape(g, x) || !ggml_are_same_shape(x, mul) || + !ggml_is_contiguous(g) || !ggml_is_contiguous(x) || !ggml_is_contiguous(mul) || + ggml_nelements(mul) >= INT32_MAX - 4) { + return false; + } + + // the kernel moves float4s + for (const ggml_tensor * t : { g, x, mul }) { + if (t->data && ((uintptr_t) t->data) % 16 != 0) { + return false; + } + } + + // each thread reads g[i], x[i] and writes dst[i]: storage may be shared element for element, but not shifted + // (leaf inputs are not covered by the generic range checks) + const ggml_tensor * ts[3] = { g, x, mul }; + for (int a = 0; a < 3; ++a) { + for (int b = a + 1; b < 3; ++b) { + const char * pa = (const char *) ts[a]->data; + const char * pb = (const char *) ts[b]->data; + const size_t n = ggml_nbytes(mul); + if (pa && pb && pa != pb && pa < pb + n && pb < pa + n) { + return false; + } + } + } + + return true; +} + +// ADD + RMS_NORM + MUL: an elementwise sum of two same-shape F32 tensors normed and scaled by a weight row. The +// sum has no other reader, so the fused kernel forms it in registers instead of writing it. +static bool ggml_metal_fusion_check_add_rms( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + + const ggml_tensor * add = nodes[0]; + const ggml_tensor * rms = nodes[1]; + const ggml_tensor * mul = nodes[2]; + + const ggml_tensor * a = add->src[0]; + const ggml_tensor * b = add->src[1]; + const ggml_tensor * w = mul->src[1]; + + if (rms->src[0] != add || mul->src[0] != rms || w == rms || w == add || + add->type != GGML_TYPE_F32 || a->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32 || + rms->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32) { + return false; + } + + // the kernel moves float4s over one row per threadgroup, with the weight shared by every row + if (add->ne[0] % 4 != 0 || add->ne[2] != 1 || add->ne[3] != 1 || + !ggml_are_same_shape(a, add) || !ggml_are_same_shape(b, add) || !ggml_are_same_shape(mul, add) || + !ggml_is_contiguous_rows(a) || !ggml_is_contiguous_rows(b) || !ggml_is_contiguous_rows(mul) || + w->ne[0] != add->ne[0] || ggml_nrows(w) != 1 || !ggml_is_contiguous(w)) { + return false; + } + + // RMS_NORM + MUL + ADD is its own fusion: matching here first would take the ADD that ends it out of its reach + // and change that chain's arithmetic (it adds inside the norm kernel), so leave the sequence to it + for (int i = node_idxs[idx + 2] + 1; i < gf->n_nodes; ++i) { + const ggml_tensor * next = gf->nodes[i]; + if (ggml_op_is_empty(next->op)) { + continue; + } + if (next->op == GGML_OP_ADD && (next->src[0] == mul || next->src[1] == mul)) { + return false; + } + break; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!a->data || !b->data || !w->data || !mul->data) { + return false; + } + + // the kernel moves float4s + for (const ggml_tensor * t : { a, b, w, (const ggml_tensor *) mul }) { + if (((uintptr_t) t->data) % 16 != 0 || (ggml_nrows(t) > 1 && t->nb[1] % 16 != 0)) { + return false; + } + } + // the output is written while both addends and the weight are still read (leaf inputs are not covered by + // the generic range checks). An addend may share its storage element for element: a thread writes only + // the elements it read itself, after the reduction barriers, and a threadgroup only touches its own row, + // so the row stride must match too. The weight is read by every threadgroup, so it must not overlap. + for (const ggml_tensor * t : { a, b }) { + if (ggml_metal_fusion_overlap(mul, t) && + (mul->data != t->data || (ggml_nrows(mul) > 1 && mul->nb[1] != t->nb[1]))) { + return false; + } + } + if (ggml_metal_fusion_overlap(mul, w)) { + return false; + } + } + + return true; +} + +// DSV4_HC_PRE + RMS_NORM (+ MUL by a weight row): the reduced residual streams normed for the next projection. +// The fused kernel keeps the float4 thread mapping of the rms_norm kernel, so the row must be a multiple of 4. +static bool ggml_metal_fusion_check_hc_pre_rms( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_UNUSED(mode); + + const ggml_tensor * pre = nodes[0]; + const ggml_tensor * rms = nodes[1]; + const ggml_tensor * mul = fusion->ops.size() > 2 ? nodes[2] : nullptr; + + if (rms->src[0] != pre || pre->type != GGML_TYPE_F32 || rms->type != GGML_TYPE_F32 || + pre->ne[0] % 4 != 0 || pre->ne[2] != 1 || pre->ne[3] != 1 || !ggml_is_contiguous(pre) || !ggml_is_contiguous(rms) || + pre->nb[0] != sizeof(float)) { + return false; + } + + if (mul) { + const ggml_tensor * w = mul->src[1]; + if (mul->src[0] != rms || w == rms || w == pre || mul->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || + w->ne[0] != pre->ne[0] || ggml_nrows(w) != 1 || !ggml_is_contiguous(w) || !ggml_is_contiguous(mul)) { + return false; + } + } + + // the output is written while the streams and the weights of every token are still read (leaf inputs are not + // covered by the generic range checks) + if (mode == GGML_METAL_FUSION_FULL) { + const ggml_tensor * dst = mul ? mul : rms; + if (ggml_metal_fusion_overlap(dst, pre->src[0]) || ggml_metal_fusion_overlap(dst, pre->src[1]) || + (mul && ggml_metal_fusion_overlap(dst, mul->src[1]))) { + return false; + } + } + + return true; +} + +// DSV4_HC_POST (no comb) + RESHAPE + RMS_NORM: the streams a sublayer leaves and the norm of their concatenation +// (the input of the next hc_mixes projection). Both are outputs; the norm must be the only reader of the reshape. +static bool ggml_metal_fusion_check_hc_post_rms( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(nodes); + + const std::vector<ggml_op> & ops_all = fusion->ops_all; + + const int raw_start = node_idxs[idx]; + const int raw_end = node_idxs[idx + (int) fusion->ops.size() - 1]; + if (raw_end - raw_start + 1 != (int) ops_all.size()) { + return false; + } + + int raw_idxs[GGML_METAL_FUSION_MAX]; + for (int i = 0; i < (int) ops_all.size(); ++i) { + raw_idxs[i] = raw_start + i; + if (gf->nodes[raw_start + i]->op != ops_all[i]) { + return false; + } + } + + const ggml_tensor * post = gf->nodes[raw_start + 0]; + const ggml_tensor * flat = gf->nodes[raw_start + 1]; + const ggml_tensor * rms = gf->nodes[raw_start + 2]; + const ggml_tensor * x = post->src[0]; + const ggml_tensor * res = post->src[1]; + const ggml_tensor * pw = post->src[2]; + + if (post->src[3] || flat->src[0] != post || rms->src[0] != flat || + post->type != GGML_TYPE_F32 || rms->type != GGML_TYPE_F32 || x->type != GGML_TYPE_F32 || + res->type != GGML_TYPE_F32 || pw->type != GGML_TYPE_F32) { + return false; + } + + const int64_t n_embd = post->ne[0]; + const int64_t hc = post->ne[1]; + + if (n_embd % 4 != 0 || hc != 4 || post->ne[3] != 1 || !ggml_is_contiguous(post) || !ggml_is_contiguous(rms) || + flat->ne[0] != n_embd*hc || flat->ne[1] != post->ne[2] || rms->ne[0] != flat->ne[0] || rms->ne[1] != flat->ne[1] || + rms->ne[2] != 1 || rms->ne[3] != 1) { + return false; + } + + const int outputs[2] = { raw_start, raw_end }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, (int) ops_all.size(), ops_all.data(), outputs, 2)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!post->data || !rms->data || !x->data || !res->data || !pw->data) { + return false; + } + // the norm is written while the streams are read back, and the streams while x/residual are read + if (ggml_metal_fusion_overlap(rms, post) || ggml_metal_fusion_overlap(rms, x) || ggml_metal_fusion_overlap(rms, res) || + ggml_metal_fusion_overlap(rms, pw) || ggml_metal_fusion_overlap(post, x) || ggml_metal_fusion_overlap(post, pw) || + (ggml_metal_fusion_overlap(post, res) && (post->data != res->data || post->nb[1] != res->nb[1] || post->nb[2] != res->nb[2]))) { + return false; + } + } + + return true; +} + +// ROPE (norm, F32) + CONCAT along dim 0 with the rotation as the second source: the kernel writes the rotation +// straight into the concatenation. The rotation must have no other reader. +static bool ggml_metal_fusion_check_rope_concat( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(nodes); + + const std::vector<ggml_op> & ops_all = fusion->ops_all; + + const int raw_start = node_idxs[idx]; + const int raw_end = node_idxs[idx + 1]; + if (raw_end != raw_start + 1) { + return false; + } + + const ggml_tensor * rope = gf->nodes[raw_start]; + const ggml_tensor * cat = gf->nodes[raw_end]; + const ggml_tensor * a = cat->src[0]; + + const int mode_rope = ((const int32_t *) rope->op_params)[2]; + + if (mode_rope != 0 || rope->type != GGML_TYPE_F32 || rope->src[0]->type != GGML_TYPE_F32 || + cat->type != GGML_TYPE_F32 || a->type != GGML_TYPE_F32 || cat->src[1] != rope || a == rope || + ggml_get_op_params_i32(cat, 0) != 0 || rope->src[0]->ne[0] % 2 != 0 || rope->src[0]->ne[0] > 1024 || + cat->ne[0] != a->ne[0] + rope->ne[0] || + a->ne[1] != rope->ne[1] || a->ne[2] != rope->ne[2] || a->ne[3] != rope->ne[3] || + cat->ne[1] != rope->ne[1] || cat->ne[2] != rope->ne[2] || cat->ne[3] != rope->ne[3] || + a->nb[0] != sizeof(float) || cat->nb[0] != sizeof(float)) { + return false; + } + + int raw_idxs[2] = { raw_start, raw_end }; + const int outputs[1] = { raw_end }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, 2, ops_all.data(), outputs, 1)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!cat->data || !a->data || !rope->src[0]->data) { + return false; + } + // the concatenation is written while both sources, the positions and the frequency factors are read (leaf + // inputs are not covered by the generic range checks) + if (ggml_metal_fusion_overlap(cat, a) || ggml_metal_fusion_overlap(cat, rope->src[0]) || + ggml_metal_fusion_overlap(cat, rope->src[1]) || (rope->src[2] && ggml_metal_fusion_overlap(cat, rope->src[2]))) { + return false; + } + } + + return true; +} + +// MUL + ADD + SIGMOID + SCALE (+ SCALE): s1*(s0*sigmoid(x*f + b) + c0) + c1 over a [n, rows] view, with a +// single factor f and a per-column bias b. The hyper-connection pre/post gates of hy_v4_flash. +static bool ggml_metal_fusion_check_hc_gate( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + const ggml_tensor * mul = nodes[0]; + const ggml_tensor * add = nodes[1]; + const ggml_tensor * sig = nodes[2]; + + if (ggml_get_unary_op(sig) != GGML_UNARY_OP_SIGMOID) { + return false; + } + + const ggml_tensor * x = mul->src[0]; + const ggml_tensor * f = mul->src[1]; + const ggml_tensor * b = add->src[0] == mul ? add->src[1] : add->src[0]; + const ggml_tensor * dst = nodes[fusion->ops.size() - 1]; + + // the bias must be an outside operand, and the int32 indexing of the kernel must cover dst + if (b == mul || ggml_nelements(dst) >= INT32_MAX) { + return false; + } + + // dst is written element-wise from x: overlapping storage is only safe with the same element mapping + if (x->data != nullptr && dst->data != nullptr) { + const char * x0 = (const char *) x->data; + const char * d0 = (const char *) dst->data; + const bool same = x0 == d0 && ggml_is_contiguous(x); + if (!same && x0 < d0 + ggml_nbytes(dst) && d0 < x0 + ggml_nbytes(x)) { + return false; + } + } + + bool ok = x->type == GGML_TYPE_F32 && f->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && ggml_are_same_shape(x, mul) && + ggml_nelements(f) == 1 && + ggml_nelements(b) == x->ne[0] && b->ne[0] == x->ne[0] && ggml_is_contiguous(b) && + x->ne[2] == 1 && x->ne[3] == 1 && x->nb[0] == sizeof(float) && + ggml_is_contiguous(dst); + + for (size_t i = 1; ok && i < fusion->ops.size(); ++i) { + ok = nodes[i]->type == GGML_TYPE_F32; + } + + return ok; +} + // MUL + SIN + SQR + MUL + ADD (snake activation) static bool ggml_metal_fusion_check_snake( const ggml_metal_fusion * fusion, @@ -500,7 +858,8 @@ static bool ggml_metal_fusion_check_topk_moe( return true; } -#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 8 +#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 16 +static_assert(2*GGML_METAL_MOE_REDUCE_MAX_EXPERTS <= GGML_METAL_FUSION_MAX, "the MUL + VIEWs + ADDs of a pattern must fit"); struct ggml_metal_moe_reduce_match { const ggml_tensor * experts; @@ -760,8 +1119,87 @@ static const std::vector<ggml_op> ops_moe_reduce_8 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +// MUL, n VIEWs, n - 1 ADDs (hy_v4_flash routes to 16 experts) +static std::vector<ggml_op> ggml_metal_ops_moe_reduce(int n) { + std::vector<ggml_op> ops(1, GGML_OP_MUL); + ops.insert(ops.end(), n, GGML_OP_VIEW); + ops.insert(ops.end(), n - 1, GGML_OP_ADD); + return ops; +} + static const std::vector<ggml_op> ops_mul_mat_add = { GGML_OP_MUL_MAT, GGML_OP_ADD }; +static const std::vector<ggml_op> ops_hc_gate_1 = { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE }; +static const std::vector<ggml_op> ops_hc_gate_2 = { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_SCALE }; + +static const std::vector<ggml_op> ops_sigmoid_mul = { GGML_OP_UNARY, GGML_OP_MUL }; + +static const std::vector<ggml_op> ops_add_rms = { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const std::vector<ggml_op> ops_hc_pre_rms = { GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM }; +static const std::vector<ggml_op> ops_hc_post_rms = { GGML_OP_DSV4_HC_POST, GGML_OP_RESHAPE, GGML_OP_RMS_NORM }; +static const std::vector<ggml_op> ops_rope_concat = { GGML_OP_ROPE, GGML_OP_CONCAT }; +static const std::vector<ggml_op> ops_hc_pre_rms_mul = { GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM, GGML_OP_MUL }; + +// the hc gate (MUL + ADD + SIGMOID + SCALE) computing the weights of DSV4_HC_PRE + RMS_NORM + MUL: the weights are +// computed in the HC_PRE kernel (four per token, as kernel_hc_gate_f32 does) instead of being stored +// (the raw sequence has the VIEW of the gate bias between MUL and ADD; it is listed so that the allocation +// dependencies of ggml_metal_fusion_add_alloc_deps, which match raw nodes, apply) +static const std::vector<ggml_op> ops_hc_gate_pre_rms = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_PRE, GGML_OP_RMS_NORM, GGML_OP_MUL +}; + +static bool ggml_metal_fusion_check_hc_gate_pre_rms( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + static const ggml_metal_fusion gate(GGML_METAL_FUSION_HC_GATE, ops_hc_gate_1, {}, false, nullptr); + static const ggml_metal_fusion prms(GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms_mul, {}, false, nullptr); + + // the gate as its own fusion would be checked: a chain of same-shape nodes + for (int j = 1; j < 4; ++j) { + if ((nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) || !ggml_are_same_shape(nodes[j], nodes[j - 1])) { + return false; + } + } + + // the gate nodes are not written: the norm weight must not be one of them + for (int j = 0; j < 4; ++j) { + if (nodes[6]->src[1] == nodes[j]) { + return false; + } + } + + const ggml_tensor * pre = nodes[4]; + if (pre->src[1] != nodes[3] || ggml_get_op_params_i32(pre, 1) != 0 || pre->src[0]->ne[1] != nodes[3]->ne[0] || pre->src[0]->ne[1] > 8 || + nodes[3]->ne[1] != pre->ne[1]) { + return false; + } + + if (!ggml_metal_fusion_check_hc_gate(&gate, nodes, gf, node_idxs, idx, mode) || + !ggml_metal_fusion_check_hc_pre_rms(&prms, nodes + 4, gf, node_idxs, idx + 4, mode)) { + return false; + } + + // the kernel also reads the mixes and the gate scale and bias + if (mode == GGML_METAL_FUSION_FULL) { + const ggml_tensor * dst = nodes[6]; + const ggml_tensor * gmul = nodes[0]; + const ggml_tensor * gadd = nodes[1]; + const ggml_tensor * gb = gadd->src[0] == gmul ? gadd->src[1] : gadd->src[0]; + if (ggml_metal_fusion_overlap(dst, gmul->src[0]) || ggml_metal_fusion_overlap(dst, gmul->src[1]) || + ggml_metal_fusion_overlap(dst, gb)) { + return false; + } + } + + const int n = (int) fusion->ops.size(); + const int outputs[1] = { node_idxs[idx + n - 1] }; + return ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, n, fusion->ops.data(), outputs, 1); +} + static const std::vector<ggml_metal_fusion> ggml_metal_fusions = { { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, @@ -788,8 +1226,18 @@ static const std::vector<ggml_metal_fusion> ggml_metal_fusions = { { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ggml_metal_ops_moe_reduce(16), {}, true, ggml_metal_fusion_check_moe_reduce }, { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, + { GGML_METAL_FUSION_HC_GATE, ops_hc_gate_1, {}, false, ggml_metal_fusion_check_hc_gate }, + { GGML_METAL_FUSION_HC_GATE, ops_hc_gate_2, {}, false, ggml_metal_fusion_check_hc_gate }, + { GGML_METAL_FUSION_SIGMOID_MUL, ops_sigmoid_mul, {}, false, ggml_metal_fusion_check_sigmoid_mul }, + { GGML_METAL_FUSION_ADD_RMS, ops_add_rms, {}, false, ggml_metal_fusion_check_add_rms }, + { GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms, {}, false, ggml_metal_fusion_check_hc_pre_rms }, + { GGML_METAL_FUSION_HC_PRE_RMS, ops_hc_pre_rms_mul, {}, false, ggml_metal_fusion_check_hc_pre_rms }, + { GGML_METAL_FUSION_HC_GATE_PRE_RMS, ops_hc_gate_pre_rms, {}, true, ggml_metal_fusion_check_hc_gate_pre_rms }, + { GGML_METAL_FUSION_HC_POST_RMS, ops_hc_post_rms, {0}, true, ggml_metal_fusion_check_hc_post_rms }, + { GGML_METAL_FUSION_ROPE_CONCAT, ops_rope_concat, {}, true, ggml_metal_fusion_check_rope_concat }, }; // ---- alloc deps ----------------------------------------------------------- @@ -1200,6 +1648,10 @@ int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { if (!fusion || total + len > GGML_METAL_FUSION_MAX) { break; } + // a pattern after the first must fit whole in the raw span, or the reorder could split it + if (i_f > 0 && idxs[i_f + len - 1] - idx + 1 > GGML_METAL_FUSION_MAX) { + break; + } total += len; i_f += len; diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h index 5d994c249..e7f59ce58 100644 --- a/ggml/src/ggml-metal/ggml-metal-fusion.h +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -20,7 +20,7 @@ struct ggml_metal_device_props; // the maximum number of nodes that can be fused in a single kernel // (also the maximum length of a packed fusion group during graph optimization) -#define GGML_METAL_FUSION_MAX 16 +#define GGML_METAL_FUSION_MAX 32 typedef enum ggml_metal_fusion_mode { // structural checks for the graph optimizer, which runs before the graph tensors are allocated (weights @@ -43,6 +43,13 @@ typedef enum ggml_metal_fusion_id { GGML_METAL_FUSION_MOE_REDUCE, // MUL + expert VIEWs + ADD chain (MoE output reduction) GGML_METAL_FUSION_SSM_CONV_SILU, // SSM_CONV + UNARY (silu) GGML_METAL_FUSION_MUL_MAT_ADD, // MUL_MAT + ADD (residual added in the few-row MMA store) + GGML_METAL_FUSION_HC_GATE, // MUL + ADD + SIGMOID + SCALE (+ SCALE): hyper-connection gate (hy_v4_flash) + GGML_METAL_FUSION_ADD_RMS, // ADD + RMS_NORM + MUL: the residual sum formed inside the norm (hy_v4_flash) + GGML_METAL_FUSION_HC_PRE_RMS, // DSV4_HC_PRE + RMS_NORM (+ MUL): the sublayer input of hy_v4_flash + GGML_METAL_FUSION_HC_GATE_PRE_RMS, // MUL + ADD + SIGMOID + SCALE (hc gate) + DSV4_HC_PRE + RMS_NORM + MUL + GGML_METAL_FUSION_HC_POST_RMS, // DSV4_HC_POST + RESHAPE + RMS_NORM: new streams and their norm (hy_v4_flash) + GGML_METAL_FUSION_ROPE_CONCAT, // ROPE + CONCAT(x, rope, 0): the [nope | rope] q and k of hy_v4_flash + GGML_METAL_FUSION_SIGMOID_MUL, // UNARY (sigmoid) + MUL: x * sigmoid(g), the attention output gate (hy_v4_flash) } ggml_metal_fusion_id; struct ggml_metal_fusion; // defined in ggml-metal-fusion.cpp diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 11c15feff..de4a7ab17 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -45,6 +45,9 @@ #define N_R0_Q8_0 2 #define N_SG_Q8_0 4 +// short-row Q8_0 mat-vec (ne00 <= 512): simdgroups per threadgroup +#define N_SG_Q8_0_SHORT 4 + #define N_R0_MXFP4 2 #define N_SG_MXFP4 2 @@ -514,6 +517,31 @@ typedef struct { int32_t nrows; } ggml_metal_kargs_flash_attn_ext_vec_reduce; +// hy4 MLA decode (kernel_hy4_mla_decode_f16): one query token, n_head heads over one latent KV head +typedef struct { + int32_t n_rows; // length of the row list (padded with -1) + int32_t rows_per_wg; // rows of the list per workgroup (multiple of the tile) + int32_t n_head; + int32_t nwg; + uint64_t nb_q_head; // bytes between the heads of q (f32) + uint64_t nb_k_row; // bytes between the K rows (f16) + float scale; + int32_t has_sinks; + int32_t has_mask; // 0: no mask; 2: list from kernel_flash_attn_ext_vec_idx (padded with -1); + // 3: the kernel compacts the mask row (n_cells long) itself. The mask is per cache cell + int32_t n_cells; + // mode 4: the rows come from the indexer picks (TOP_K_MASK inputs), the mask is its keep row + int32_t n_top; + int32_t n_blk; + int32_t n_pick; + int32_t n_extra; + uint64_t nbk0; // top_k + uint64_t nbs0; // score + uint64_t nbb0; // blocks + uint64_t nbb1; + uint64_t nbe0; // extra +} ggml_metal_kargs_hy4_mla; + typedef struct { int32_t ne00; int32_t ne02; @@ -705,6 +733,16 @@ typedef struct { int32_t C; } ggml_metal_kargs_snake; +typedef struct { + int32_t n; // columns per row + int32_t n_rows; + uint64_t nb_x1; // row stride of the input view + float s0; + float c0; + float s1; + float c1; +} ggml_metal_kargs_hc_gate; + typedef struct { int32_t IC; int32_t IH; @@ -1307,6 +1345,7 @@ typedef struct { uint64_t nbw3; uint64_t nbm1; uint64_t nbm3; + int32_t has_rows; // the K rows to score come from a list (buffer 6) } ggml_metal_kargs_lightning_indexer; typedef struct { @@ -1368,6 +1407,47 @@ typedef struct { float scale; } ggml_metal_kargs_dsv4_hc_pre; +// DSV4_HC_PRE + RMS_NORM (+ MUL), see kernel_hy4_hc_pre_rms_f32 +typedef struct { + ggml_metal_kargs_dsv4_hc_pre hc; + int32_t ne00_t; // n_embd/4 + float eps; + uint64_t nb_y1; // output row stride + // the weights from the mixes (kernel_hc_gate_f32): s1*(s0*sigmoid(mix*f + b) + c0) + c1 + uint64_t nb_m1; // mixes row stride + float s0; + float c0; + float s1; + float c1; +} ggml_metal_kargs_hy4_hc_pre_rms; + +// DSV4_HC_POST (no comb) + RMS_NORM of the flattened streams, see kernel_hy4_hc_post_rms_f32 +typedef struct { + int32_t n_embd; + int32_t ne00_t; // hc*n_embd/4 + uint64_t nb_x0; + uint64_t nb_x1; + uint64_t nb_r0; + uint64_t nb_r1; + uint64_t nb_r2; + uint64_t nb_p0; + uint64_t nb_p1; + uint64_t nb_d1; // stream stride of the HC_POST output (contiguous: n_embd floats) + uint64_t nb_d2; + uint64_t nb_y1; // row stride of the normed output + float eps; +} ggml_metal_kargs_hy4_hc_post_rms; + +// ADD + RMS_NORM + MUL, see kernel_hy4_add_rms_norm_mul_f32 +typedef struct { + int32_t ne00; + int32_t ne00_t; // ne00/4 + uint64_t nb_a1; // row strides of the two addends, the weight and the output + uint64_t nb_b1; + uint64_t nb_y1; + float eps; +} ggml_metal_kargs_hy4_add_rms; + typedef struct { int32_t n_embd; int32_t n_tokens; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index cace34bd4..a2b407ed3 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -81,6 +81,16 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } + // the whole graph and the graph index of node i (nodes outside this encoder's range included) + const ggml_cgraph * graph() const { + return gf; + } + + int graph_idx(int i) const { + assert(i >= 0 && i < (int) idxs.size()); + return idxs[i]; + } + // consult the fusion table for the longest pattern starting at i0 // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { @@ -821,6 +831,15 @@ int ggml_metal_op_acc(ggml_metal_op_t ctx, int idx) { int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + if (op->op == GGML_OP_UNARY && ggml_get_unary_op(op) == GGML_UNARY_OP_SIGMOID && ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SIGMOID_MUL) { + ctx->count_fusions(fusion); + return ggml_metal_op_sigmoid_mul_fused(ctx, idx); + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -1394,8 +1413,10 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK); + const ggml_tensor * r = op->src[4]; // optional list of the K rows + ggml_metal_kargs_lightning_indexer args = { - /*.n_kv =*/ (int32_t) k->ne[2], + /*.n_kv =*/ (int32_t) (r ? r->ne[0] : k->ne[2]), /*.n_batch =*/ (int32_t) q->ne[2], /*.mask_ne3 =*/ (int32_t) m->ne[3], /*.nb1 =*/ op->nb[1], @@ -1409,6 +1430,7 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { /*.nbw3 =*/ w->nb[3], /*.nbm1 =*/ m->nb[1], /*.nbm3 =*/ m->nb[3], + /*.has_rows =*/ r ? 1 : 0, }; ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q), 1); @@ -1416,6 +1438,7 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w), 3); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m), 4); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(r ? r : op), 6); const int nsg = OP_LIGHTNING_INDEXER_NSG; const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg; @@ -1425,16 +1448,97 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ggml_metal_encoder_dispatch_threadgroups(enc, - (k->ne[2] + nkptg - 1)/nkptg, + (args.n_kv + nkptg - 1)/nkptg, (q->ne[2] + nbptg - 1)/nbptg, q->ne[3], 32, nsg, 1); return 1; } +static bool ggml_metal_op_hy4_mla_ok(const ggml_tensor * op); +static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op); + +static const ggml_tensor * ggml_metal_op_view_base(const ggml_tensor * t) { + while (t && t->view_src) { + t = t->view_src; + } + return t; +} + +// hy4: the single-token indexer selection (named indexer_sel by the model) feeds one flash attention, which can build +// the list of selected cells from the top-k picks itself (kernel_hy4_mla_decode_v2 mode 4). True when the flash +// attention fa reads the mask tkm that way. +static bool ggml_metal_op_hy4_mode4_ok(const ggml_tensor * fa, const ggml_tensor * tkm) { + static const bool off = getenv("HY4_MLA_DISABLE") != nullptr; + if (off || fa->op != GGML_OP_FLASH_ATTN_EXT || tkm->op != GGML_OP_TOP_K_MASK || + strncmp(tkm->name, "indexer_sel", 11) != 0 || !fa->src[3] || + tkm->ne[1] != 1 || tkm->ne[2] != 1 || tkm->ne[3] != 1 || !tkm->src[2] || !tkm->src[3]) { + return false; + } + + // the mask must be the selection itself through views only (an op writing into its storage, e.g. a copy into + // it or an in-place add, also flattens to it as view_src, but changes what the attention must see) + for (const ggml_tensor * t = fa->src[3]; t != tkm; t = t->src[0]) { + if (!t || (t->op != GGML_OP_RESHAPE && t->op != GGML_OP_VIEW)) { + return false; + } + } + + // the picks must be extra sources of the attention (kept alive until it, see hy-v4-flash.cpp), the keep row a + // zero fill (it is not read), and the mask view must be the whole mask. The picked cells must be distinct (a + // repeated cell would count twice in the attention, once in the mask): hy4's pools and tail are disjoint and + // padding points past the cache. + const ggml_tensor * keep = tkm->src[0]; + const int64_t n_list = tkm->src[1]->ne[0]*tkm->src[3]->ne[0] + (tkm->src[4] ? tkm->src[4]->ne[0] : 0); + + return fa->src[6] == tkm->src[1] && fa->src[7] == tkm->src[2] && fa->src[8] == tkm->src[3] && fa->src[9] == tkm->src[4] && + keep->op == GGML_OP_FILL && ggml_get_op_params_f32(keep, 0) == 0.0f && + fa->src[3]->data == tkm->data && fa->src[3]->ne[0] == tkm->ne[0] && tkm->ne[0] <= fa->src[1]->ne[1] && n_list <= 4096 && + ggml_metal_op_flash_attn_ext_n_kv_max_sparse(fa) > 0 && ggml_metal_op_hy4_mla_ok(fa); +} + +static int ggml_metal_op_top_k_mask_impl(ggml_metal_op_t ctx, const ggml_tensor * op); + int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) { - ggml_tensor * op = ctx->node(idx); + const ggml_tensor * op = ctx->node(idx); + + // a selection that its flash attention reads through the picks is not built at all. Deferred only when the + // mask (through a chain of views) has that attention in this graph as its only reader, and is not an output + // (a graph split for an eval callback ends the graph at the mask, so the attention is not found). The use + // counts are those ggml_build_forward_expand computed: a graph edited afterwards (ggml_graph_add_node, sources + // retargeted) must not depend on this. + if (op->op == GGML_OP_TOP_K_MASK && strncmp(op->name, "indexer_sel", 11) == 0) { + const ggml_cgraph * gf = ctx->graph(); + const ggml_tensor * head = op; + int i_head = ctx->graph_idx(idx); + for (int i = i_head + 1; i < gf->n_nodes; ++i) { + const ggml_tensor * node = gf->nodes[i]; + bool reads = false; + for (int j = 0; j < GGML_MAX_SRC && !reads; ++j) { + reads = node->src[j] == head; + } + if (!reads) { + continue; + } + if ((head->flags & GGML_TENSOR_FLAG_OUTPUT) || ggml_node_get_use_count(gf, i_head) != 1) { + break; + } + if ((node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW) && node->src[0] == head) { + head = node; + i_head = i; + continue; + } + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) && ggml_metal_op_hy4_mode4_ok(node, op)) { + return 1; + } + break; + } + } + return ggml_metal_op_top_k_mask_impl(ctx, op); +} + +static int ggml_metal_op_top_k_mask_impl(ggml_metal_op_t ctx, const ggml_tensor * op) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -1550,9 +1654,191 @@ int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) { return 1; } +// a fused kernel that replaces a rms_norm reduces in its order only with its thread count; a pipeline that could +// not be created disables the fusion as well +static bool ggml_metal_pipeline_nth_eq( + const ggml_metal_pipeline_with_params & fused, const ggml_metal_pipeline_with_params & unfused) { + return fused.pipeline && unfused.pipeline && + std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(fused)) == + std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(unfused)); +} + +// DSV4_HC_PRE + RMS_NORM (+ MUL), see ggml_metal_fusion_check_hc_pre_rms +// n_gate == 4: the hc gate (MUL + ADD + SIGMOID + SCALE, see ggml_metal_op_hc_gate_fused) comes first and the +// weights are computed in the kernel (ggml_metal_fusion_check_hc_gate_pre_rms) +static int ggml_metal_op_hy4_hc_pre_rms(ggml_metal_op_t ctx, int idx, int n_fuse, int n_gate = 0) { + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_tensor * pre = ctx->node(idx + n_gate); + const ggml_tensor * rms = ctx->node(idx + n_gate + 1); + const ggml_tensor * mul = n_fuse - n_gate > 2 ? ctx->node(idx + n_gate + 2) : nullptr; + ggml_tensor * dst = ctx->node(idx + n_fuse - 1); + + const ggml_tensor * x = pre->src[0]; + const ggml_tensor * w = pre->src[1]; + + // the gate: s1*(s0*sigmoid(mix*f + b) + c0) + c1 with one SCALE + const ggml_tensor * gmix = nullptr; + const ggml_tensor * gf = nullptr; + const ggml_tensor * gb = nullptr; + float s0 = 1.0f, c0 = 0.0f; + if (n_gate) { + GGML_ASSERT(n_gate == 4 && mul); + const ggml_tensor * gmul = ctx->node(idx); + const ggml_tensor * gadd = ctx->node(idx + 1); + const ggml_tensor * gsc = ctx->node(idx + 3); + gmix = gmul->src[0]; + gf = gmul->src[1]; + gb = gadd->src[0] == gmul ? gadd->src[1] : gadd->src[0]; + s0 = ggml_get_op_params_f32(gsc, 0); + c0 = ggml_get_op_params_f32(gsc, 1); + } + + for (int i = 1; i < n_fuse; ++i) { + if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) { + ggml_metal_op_concurrency_reset(ctx); + break; + } + } + + float eps; + memcpy(&eps, rms->op_params, sizeof(float)); + + ggml_metal_kargs_hy4_hc_pre_rms args = { + /*.hc =*/ { + /*.n_embd =*/ (int32_t) x->ne[0], + /*.n_tokens =*/ (int32_t) x->ne[2], + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_x2 =*/ x->nb[2], + /*.nb_w0 =*/ w->nb[0], + /*.nb_w1 =*/ w->nb[1], + /*.nb_w2 =*/ w->nb[2], + /*.nb_d0 =*/ pre->nb[0], + /*.nb_d1 =*/ pre->nb[1], + /*.scale =*/ ggml_get_op_params_f32(pre, 0), + }, + /*.ne00_t =*/ (int32_t) x->ne[0]/4, + /*.eps =*/ eps, + /*.nb_y1 =*/ dst->nb[1], + /*.nb_m1 =*/ gmix ? gmix->nb[1] : 0, + /*.s0 =*/ s0, + /*.c0 =*/ c0, + /*.s1 =*/ 1.0f, + /*.c1 =*/ 0.0f, + }; + + auto pipeline = ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, pre, mul != nullptr, n_gate != 0); + + // the thread count of ggml_metal_op_norm, the sum depends on it + int nth = 32; + while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + nth *= 2; + } + nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + nth = std::min(nth, (args.ne00_t + 31)/32*32); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gmix ? gmix : w), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul ? mul->src[1] : x), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gf ? gf : x), 5); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(gb ? gb : x), 6); + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, args.hc.n_tokens, 1, 1, nth, 1, 1); + + return n_fuse; +} + +// DSV4_HC_POST + RESHAPE + RMS_NORM, see ggml_metal_fusion_check_hc_post_rms +static int ggml_metal_op_hy4_hc_post_rms(ggml_metal_op_t ctx, int idx, int n_fuse) { + ggml_metal_encoder_t enc = ctx->enc; + + GGML_ASSERT(n_fuse == 2); + + ggml_tensor * post = ctx->node(idx); + ggml_tensor * rms = ctx->node(idx + 1); + + const ggml_tensor * x = post->src[0]; + const ggml_tensor * r = post->src[1]; + const ggml_tensor * pw = post->src[2]; + + if (!ggml_metal_op_concurrency_check(ctx, rms)) { + ggml_metal_op_concurrency_reset(ctx); + } + + float eps; + memcpy(&eps, rms->op_params, sizeof(float)); + + ggml_metal_kargs_hy4_hc_post_rms args = { + /*.n_embd =*/ (int32_t) post->ne[0], + /*.ne00_t =*/ (int32_t) (rms->ne[0]/4), + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_r0 =*/ r->nb[0], + /*.nb_r1 =*/ r->nb[1], + /*.nb_r2 =*/ r->nb[2], + /*.nb_p0 =*/ pw->nb[0], + /*.nb_p1 =*/ pw->nb[1], + /*.nb_d1 =*/ post->nb[1], + /*.nb_d2 =*/ post->nb[2], + /*.nb_y1 =*/ rms->nb[1], + /*.eps =*/ eps, + }; + + auto pipeline = ggml_metal_library_get_pipeline_hy4_hc_post_rms(ctx->lib); + + // the thread count of ggml_metal_op_norm, the sum depends on it + int nth = 32; + while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + nth *= 2; + } + nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + nth = std::min(nth, (args.ne00_t + 31)/32*32); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(r), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(pw), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(post), 4); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(rms), 5); + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, rms->ne[1], 1, 1, nth, 1, 1); + + return n_fuse; +} + int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + // the fused kernels reduce in the order of the rms_norm kernel only with its thread count (capped at 1024) + auto same_nth = [&](const ggml_metal_pipeline_with_params & fused, int idx_rms, int n_norm) { + return ggml_metal_pipeline_nth_eq(fused, ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx_rms), n_norm)); + }; + + if (op->op == GGML_OP_DSV4_HC_POST && ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_POST_RMS && + same_nth(ggml_metal_library_get_pipeline_hy4_hc_post_rms(ctx->lib), idx + 1, 1)) { + ctx->count_fusions(fusion); + return ggml_metal_op_hy4_hc_post_rms(ctx, idx, n); + } + } + + if (op->op == GGML_OP_DSV4_HC_PRE && ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_PRE_RMS && + same_nth(ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, op, n > 2, false), idx + 1, n - 1)) { + ctx->count_fusions(fusion); + return ggml_metal_op_hy4_hc_pre_rms(ctx, idx, n); + } + } + ggml_metal_encoder_t enc = ctx->enc; auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op); @@ -3547,9 +3833,140 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const ggml_tensor * op) { return k_size + v_size; } +// hy4 MLA decode (kernel_hy4_mla_decode_v2): one query token over one latent KV head (DK 576, V = the first 512 +// dims of K) for the sparse path (n_kv_max hint), over the compacted list of the visible cells. Every K row is read +// once per 16 heads. Set HY4_MLA_DISABLE to fall back to the vec kernels. +static bool ggml_metal_op_hy4_mla_ok(const ggml_tensor * op) { + static const bool off = getenv("HY4_MLA_DISABLE") != nullptr; + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + float max_bias = 0.0f; + float softcap = 0.0f; + memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float)); + memcpy(&softcap, (const float *) op->op_params + 2, sizeof(float)); + const ggml_tensor * m = op->src[3]; + // the kernel reads q as f32 and K rows as f16 with 16-byte loads, and one mask row + return !off && q->ne[1] == 1 && q->ne[0] == 576 && op->src[2]->ne[0] == 512 && k->ne[2] == 1 && k->ne[3] == 1 && + q->ne[3] == 1 && q->ne[2] <= 128 && k->type == GGML_TYPE_F16 && max_bias == 0.0f && softcap == 0.0f && + ggml_metal_op_flash_attn_ext_v_is_view_of_k(op) && op->src[2]->data == k->data && op->src[2]->nb[1] == k->nb[1] && + q->nb[0] == 4 && k->nb[0] == 2 && + q->nb[2] % 16 == 0 && k->nb[1] % 16 == 0 && (!m || (m->ne[2] == 1 && m->ne[3] == 1 && m->ne[0] == k->ne[1])); +} + +static void ggml_metal_op_hy4_mla(ggml_metal_op_t ctx, const ggml_tensor * op, + ggml_metal_buffer_id bid_list, ggml_metal_buffer_id bid_mask, int32_t n_rows, int32_t n_valid_max, int32_t mask_mode, int32_t n_cells, + ggml_metal_buffer_id bid_tmp, ggml_metal_buffer_id bid_dst, float scale, const ggml_tensor * tkm = nullptr) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); + + const int32_t n_head = op->src[0]->ne[2]; + const bool has_sinks = op->src[4] != nullptr; + + constexpr int TR = 16; // K rows per tile + constexpr int NG = 2; // groups of 8 heads per threadgroup + + // measured on M5 Max: 64 heads x 512 window rows best at 16, 32 heads x 2052 sparse rows at 32 + // (the tmp buffer, ggml_metal_op_flash_attn_ext_extra_tmp, holds 32 partials per row) + const int nwg = n_head >= 64 ? 16 : 32; + GGML_ASSERT(nwg <= 32); + const int rows_per_wg = GGML_PAD((n_valid_max + nwg - 1)/nwg, TR); + + ggml_metal_kargs_hy4_mla args = { + /*.n_rows =*/ n_rows, + /*.rows_per_wg =*/ rows_per_wg, + /*.n_head =*/ n_head, + /*.nwg =*/ nwg, + /*.nb_q_head =*/ op->src[0]->nb[2], + /*.nb_k_row =*/ op->src[1]->nb[1], + /*.scale =*/ scale, + /*.has_sinks =*/ has_sinks ? 1 : 0, + /*.has_mask =*/ mask_mode, + /*.n_cells =*/ n_cells, + /*.n_top =*/ tkm ? (int32_t) tkm->src[1]->ne[0] : 0, + /*.n_blk =*/ tkm ? (int32_t) tkm->src[3]->ne[0] : 0, + /*.n_pick =*/ tkm ? (int32_t) tkm->src[3]->ne[1] : 0, + /*.n_extra =*/ tkm && tkm->src[4] ? (int32_t) tkm->src[4]->ne[0] : 0, + /*.nbk0 =*/ tkm ? tkm->src[1]->nb[0] : 0, + /*.nbs0 =*/ tkm ? tkm->src[2]->nb[0] : 0, + /*.nbb0 =*/ tkm ? tkm->src[3]->nb[0] : 0, + /*.nbb1 =*/ tkm ? tkm->src[3]->nb[1] : 0, + /*.nbe0 =*/ tkm && tkm->src[4] ? tkm->src[4]->nb[0] : 0, + }; + + const char * kname = "kernel_hy4_mla_decode_v2_ng2"; + ggml_metal_pipeline_with_params ppl = ggml_metal_library_get_pipeline(lib, kname); + if (!ppl.pipeline) { + ppl = ggml_metal_library_compile_pipeline(lib, kname, kname, nullptr); + } + GGML_ASSERT(32*4*NG <= ggml_metal_pipeline_max_theads_per_threadgroup(ppl)); + + // K tile [16][576] halfs, partial S [NG][4][8][16] floats, P [NG][8][16] halfs, diag [NG][64], M, S, row ids + // + the mode 3 list [1024] and simdgroup counts [32] + const size_t smem = GGML_PAD(2*(TR*576) + 4*(NG*4*8*TR) + 2*(NG*8*TR) + 4*(NG*64 + 2*8*NG) + 4*TR + 4*4 + 4*TR + 4*(1024 + 32), 16); + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + + ggml_metal_buffer_id bid_q = ggml_metal_get_buffer_id(op->src[0]); + ggml_metal_buffer_id bid_k = ggml_metal_get_buffer_id(op->src[1]); + ggml_metal_buffer_id bid_sinks = has_sinks ? ggml_metal_get_buffer_id(op->src[4]) : bid_q; + + ggml_metal_encoder_set_pipeline(enc, ppl); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_q, 1); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_list, 3); + ggml_metal_encoder_set_buffer (enc, bid_sinks, 4); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 5); + ggml_metal_encoder_set_buffer (enc, bid_mask, 6); + ggml_metal_encoder_set_buffer (enc, tkm ? ggml_metal_get_buffer_id(tkm->src[2]) : bid_mask, 7); + ggml_metal_encoder_set_buffer (enc, tkm ? ggml_metal_get_buffer_id(tkm->src[3]) : bid_mask, 8); + ggml_metal_encoder_set_buffer (enc, tkm && tkm->src[4] ? ggml_metal_get_buffer_id(tkm->src[4]) : bid_mask, 9); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, nwg, (n_head + 8*NG - 1)/(8*NG), 1, 32, 4*NG, 1); + + ggml_metal_op_concurrency_reset(ctx); + + const char * rname = "kernel_hy4_mla_reduce"; + ggml_metal_pipeline_with_params ppl_r = ggml_metal_library_get_pipeline(lib, rname); + if (!ppl_r.pipeline) { + ppl_r = ggml_metal_library_compile_pipeline(lib, rname, rname, nullptr); + } + + ggml_metal_encoder_set_pipeline(enc, ppl_r); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + ggml_metal_encoder_dispatch_threadgroups(enc, n_head, 1, 1, 128, 1, 1); +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + // hy4 sparse decode: the selected cells come from the indexer picks (see ggml_metal_op_hy4_mode4_ok). The mask + // was not built when this op is its reader; if it was (another reader came first), it is simply not used. + { + const ggml_tensor * base = ggml_metal_op_view_base(op->src[3]); + if (base && ggml_metal_op_hy4_mode4_ok(op, base)) { + // the picks are tracked as extra sources, the deferred mask's own inputs are not: serialize + ggml_metal_op_concurrency_reset(ctx); + { + const int n_list = (int) (base->src[1]->ne[0]*base->src[3]->ne[0] + (base->src[4] ? base->src[4]->ne[0] : 0)); + + ggml_metal_buffer_id bid_tmp = ggml_metal_get_buffer_id(op); + bid_tmp.offs += ggml_nbytes(op) + ggml_metal_op_flash_attn_ext_extra_pad(op) + ggml_metal_op_flash_attn_ext_extra_blk(op); + + float scale_mla; + memcpy(&scale_mla, (const float *) op->op_params + 0, sizeof(float)); + + ggml_metal_op_hy4_mla(ctx, op, ggml_metal_get_buffer_id(base->src[1]), ggml_metal_get_buffer_id(op->src[3]), + n_list, n_list, 4, (int32_t) base->ne[0], bid_tmp, ggml_metal_get_buffer_id(op), scale_mla, base); + ggml_metal_op_concurrency_reset(ctx); + return 1; + } + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -4105,6 +4522,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { const bool has_kvpad = !use_sparse && ne11_attn % ncpsg != 0; + // a short mask (e.g. the SWA cache) is compacted inside the hy4 MLA kernel, no separate pass + if (use_sparse && has_mask && ne30 <= 8192 && n_kv_max_sparse <= 1024 && ggml_metal_op_hy4_mla_ok(op)) { + ggml_metal_op_hy4_mla(ctx, op, bid_idx, bid_src3, n_kv_max_sparse, n_kv_max_sparse, 3, ne30, bid_tmp, bid_dst, scale); + return 1; + } + if (use_sparse) { assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0); @@ -4221,7 +4644,11 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } } else { - nwg = 32; + // a short KV view (e.g. a sliding window) is cheaper scanned by one workgroup that writes + // dst directly than split 32 ways with a reduce pass (hy_v4_flash decode +3%, 64 heads at + // <= 1k rows); with few heads one workgroup per head would leave most of the GPU idle + const int64_t n_tg = ne02*ne03*((ne01 + nqptg - 1)/nqptg); + nwg = ne11_attn <= 1024 && n_tg >= 32 ? 1 : 32; nsg = 1; while (2*nwg*nsg*ncpsg < ne11_attn && nsg < 4) { nsg *= 2; @@ -4273,6 +4700,11 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.n_kv_max_padded =*/ n_kv_max_padded, }; + if (use_sparse && ggml_metal_op_hy4_mla_ok(op)) { + ggml_metal_op_hy4_mla(ctx, op, bid_idx, bid_src3, n_kv_max_padded, n_kv_max_sparse, has_mask ? 2 : 0, ne30, bid_tmp, bid_dst, scale); + return 1; + } + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, has_rows, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); @@ -4338,6 +4770,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { return 1; } +static int ggml_metal_op_hy4_add_rms(ggml_metal_op_t ctx, int idx); + + int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { int n_fuse = 1; const ggml_metal_fusion * fusion = nullptr; @@ -4349,6 +4784,37 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); n_fuse = n; + // the residual sum computed inside RMS_NORM + MUL (same thread count as rms_norm) + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ADD_RMS) { + if (ggml_metal_pipeline_nth_eq(ggml_metal_library_get_pipeline_hy4_add_rms(ctx->lib), + ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx + 1), 2))) { + ctx->count_fusions(fusion); + return ggml_metal_op_hy4_add_rms(ctx, idx); + } + // encode this ADD alone; the following nodes match again on their own + fusion = nullptr; + n_fuse = 1; + } + + // the hyper-connection gate computed inside DSV4_HC_PRE + RMS_NORM + MUL (same thread count as rms_norm) + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_GATE_PRE_RMS) { + const ggml_tensor * pre = ctx->node(idx + 4); + if (ggml_metal_pipeline_nth_eq(ggml_metal_library_get_pipeline_hy4_hc_pre_rms(ctx->lib, pre, true, true), + ggml_metal_library_get_pipeline_norm(ctx->lib, ctx->node(idx + 5), 2))) { + ctx->count_fusions(fusion); + return ggml_metal_op_hy4_hc_pre_rms(ctx, idx, n_fuse, 4); + } + // encode this MUL alone; the following nodes match again on their own (the gate alone was not checked) + fusion = nullptr; + n_fuse = 1; + } + + // hyper-connection gate: mul -> add -> sigmoid -> scale (-> scale) + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_HC_GATE) { + ctx->count_fusions(fusion); + return ggml_metal_op_hc_gate_fused(ctx, idx, n_fuse); + } + // snake activation autofuse: mul -> sin -> sqr -> mul -> add if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SNAKE) { ctx->count_fusions(fusion); @@ -4628,6 +5094,59 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) { return 1; } +// ADD + RMS_NORM + MUL, see ggml_metal_fusion_check_add_rms +static int ggml_metal_op_hy4_add_rms(ggml_metal_op_t ctx, int idx) { + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_tensor * add = ctx->node(idx); + const ggml_tensor * rms = ctx->node(idx + 1); + ggml_tensor * dst = ctx->node(idx + 2); + + const ggml_tensor * a = add->src[0]; + const ggml_tensor * b = add->src[1]; + const ggml_tensor * w = dst->src[1]; + + for (int i = 1; i < 3; ++i) { + if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) { + ggml_metal_op_concurrency_reset(ctx); + break; + } + } + + float eps; + memcpy(&eps, rms->op_params, sizeof(float)); + + ggml_metal_kargs_hy4_add_rms args = { + /*.ne00 =*/ (int32_t) add->ne[0], + /*.ne00_t =*/ (int32_t) add->ne[0]/4, + /*.nb_a1 =*/ a->nb[1], + /*.nb_b1 =*/ b->nb[1], + /*.nb_y1 =*/ dst->nb[1], + /*.eps =*/ eps, + }; + + auto pipeline = ggml_metal_library_get_pipeline_hy4_add_rms(ctx->lib); + + // the thread count of ggml_metal_op_norm, the sum depends on it + int nth = 32; + while (nth < args.ne00_t && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + nth *= 2; + } + nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + nth = std::min(nth, (args.ne00_t + 31)/32*32); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(a), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(b), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(w), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4); + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, ggml_nrows(add), 1, 1, nth, 1, 1); + + return 3; +} + int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -4768,6 +5287,20 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + // ROPE + CONCAT: the rotation goes straight into the concatenation (see ggml_metal_fusion_check_rope_concat) + ggml_tensor * cat = nullptr; + if (ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ROPE_CONCAT) { + ctx->count_fusions(fusion); + cat = ctx->node(idx + 1); + if (!ggml_metal_op_concurrency_check(ctx, cat)) { + ggml_metal_op_concurrency_reset(ctx); + } + } + } + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); @@ -4810,7 +5343,7 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { const int n_offs = ((const int32_t *) op->op_params)[15]; // when dst aliases src0, the channels outside the rotated window already hold the correct data - const bool inplace = op->data == op->src[0]->data; + const bool inplace = !cat && op->data == op->src[0]->data; ggml_metal_kargs_rope args = { /*.ne00 =*/ ne00, @@ -4847,6 +5380,42 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { /* inplace =*/ inplace, }; + if (cat) { + const ggml_tensor * a = cat->src[0]; + + // the rotation is written with the strides of the concatenation + args.nb0 = cat->nb[0]; + args.nb1 = cat->nb[1]; + args.nb2 = cat->nb[2]; + args.nb3 = cat->nb[3]; + + ggml_metal_kargs_concat cc = {}; + cc.ne00 = (int32_t) a->ne[0]; + cc.nb00 = a->nb[0]; + cc.nb01 = a->nb[1]; + cc.nb02 = a->nb[2]; + cc.nb03 = a->nb[3]; + cc.nb0 = cat->nb[0]; + cc.nb1 = cat->nb[1]; + cc.nb2 = cat->nb[2]; + cc.nb3 = cat->nb[3]; + + auto ppl = ggml_metal_library_get_pipeline_hy4_rope_concat(lib); + + ggml_metal_encoder_set_pipeline(enc, ppl); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_bytes (enc, &cc, sizeof(cc), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2] ? op->src[2] : op->src[0]), 4); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(a), 5); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(cat), 6); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); + + return 2; + } + auto pipeline = ggml_metal_library_get_pipeline_rope(lib, op); ggml_metal_encoder_set_pipeline(enc, pipeline); @@ -5260,6 +5829,86 @@ int ggml_metal_op_col2im_1d(ggml_metal_op_t ctx, int idx) { return 1; } +// Dispatch the fused hyper-connection gate from the matched mul -> add -> sigmoid -> scale (-> scale) chain. +int ggml_metal_op_hc_gate_fused(ggml_metal_op_t ctx, int idx, int n_fuse) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + GGML_ASSERT(n_fuse == 4 || n_fuse == 5); + + const ggml_tensor * mul = ctx->node(idx + 0); + const ggml_tensor * add = ctx->node(idx + 1); + const ggml_tensor * sc0 = ctx->node(idx + 3); + const ggml_tensor * sc1 = n_fuse == 5 ? ctx->node(idx + 4) : nullptr; + ggml_tensor * dst = ctx->node(idx + n_fuse - 1); + + const ggml_tensor * x = mul->src[0]; + const ggml_tensor * f = mul->src[1]; + const ggml_tensor * b = add->src[0] == mul ? add->src[1] : add->src[0]; + + for (int i = 1; i < n_fuse; ++i) { + if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) { + ggml_metal_op_concurrency_reset(ctx); + + break; + } + } + + ggml_metal_kargs_hc_gate args = { + /*.n =*/ (int32_t) x->ne[0], + /*.n_rows =*/ (int32_t) x->ne[1], + /*.nb_x1 =*/ x->nb[1], + /*.s0 =*/ ggml_get_op_params_f32(sc0, 0), + /*.c0 =*/ ggml_get_op_params_f32(sc0, 1), + /*.s1 =*/ sc1 ? ggml_get_op_params_f32(sc1, 0) : 1.0f, + /*.c1 =*/ sc1 ? ggml_get_op_params_f32(sc1, 1) : 0.0f, + }; + + auto pipeline = ggml_metal_library_get_pipeline_hc_gate(lib); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(f), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(b), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 4); + + const int total = args.n * args.n_rows; + const int nth = std::min(256, std::max(32, total)); + ggml_metal_encoder_dispatch_threadgroups(enc, (total + nth - 1)/nth, 1, 1, nth, 1, 1); + + return n_fuse; +} + +// x * sigmoid(g): idx is the sigmoid, idx + 1 the mul. The caller has validated the pair. +int ggml_metal_op_sigmoid_mul_fused(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_tensor * sig = ctx->node(idx + 0); + ggml_tensor * mul = ctx->node(idx + 1); + + if (!ggml_metal_op_concurrency_check(ctx, mul)) { + ggml_metal_op_concurrency_reset(ctx); + } + + int32_t n = (int32_t) ggml_nelements(mul); + + auto pipeline = ggml_metal_library_get_pipeline_sigmoid_mul(lib); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &n, sizeof(n), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(sig->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul->src[0]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(mul), 3); + + const int nt = (n + 3)/4; + const int nth = std::min(256, std::max(32, nt)); + ggml_metal_encoder_dispatch_threadgroups(enc, (nt + nth - 1)/nth, 1, 1, nth, 1, 1); + + return 2; +} + // Dispatch the fused snake kernel from the matched mul -> sin -> sqr -> mul -> add chain. // idx points at the leading mul. The caller has validated the chain. int ggml_metal_op_snake_fused(ggml_metal_op_t ctx, int idx) { @@ -5978,6 +6627,50 @@ static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + // the 11-bit variant (3 histogram passes, parallel bucket select, single emit pass); HY4_TOPK_R8 for the old one + static const bool r8 = getenv("HY4_TOPK_R8") != nullptr; + if (!r8 && op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32) { + const char * kname = "kernel_top_k_f32_i32_r11"; + ggml_metal_pipeline_with_params ppl = ggml_metal_library_get_pipeline(lib, kname); + if (!ppl.pipeline) { + ppl = ggml_metal_library_compile_pipeline(lib, kname, kname, nullptr); + } + + // its prefix sums assume 32-wide SIMD groups (at most 32 of them) + if (!ppl.pipeline || ggml_metal_pipeline_simd_width(ppl) != 32) { + goto r8_fallback; + } + + int nth = 32; // power of two, the kernel splits the buckets evenly over the threads + while (2*nth <= std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(ppl))) { + nth *= 2; + } + + ggml_metal_kargs_top_k args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.top_k =*/ (int32_t) op->ne[0], + }; + + ggml_metal_encoder_set_pipeline(enc, ppl); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_set_threadgroup_memory_size(enc, 2048*sizeof(uint32_t), 0); + ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD(34*sizeof(uint32_t), 16), 1); + ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD( 2*sizeof(uint32_t), 16), 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); + return; + } +r8_fallback: + auto pipeline = ggml_metal_library_get_pipeline_top_k_radix(lib, op); // one workgroup per row; radix-select the k-th largest value @@ -6126,7 +6819,7 @@ int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) { const int k = op->ne[0]; const int nrows = ggml_nrows(op->src[0]); - const bool use_radix = + const bool use_radix = k > 0 && ncols > 2048 && (k > 64 || (nrows > 4 && ncols >= 8192)); if (use_radix) { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 47889fa84..4078cf77e 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -92,6 +92,8 @@ int ggml_metal_op_conv_transpose_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_transpose_2d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_col2im_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_snake_fused (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_hc_gate_fused (ggml_metal_op_t ctx, int idx, int n_fuse); +int ggml_metal_op_sigmoid_mul_fused (ggml_metal_op_t ctx, int idx); int ggml_metal_op_upscale (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad_reflect_1d (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 9bed6421b..4e247792b 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -9,6 +9,7 @@ #include "ggml-metal-ops.h" #include "ggml-metal-tuning.h" +#include <cstdlib> #include <mutex> #include <string> @@ -657,6 +658,23 @@ static void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) { ggml_metal_set_n_cb(ctx, n_cb); } +// hy4: number of extra command-buffer encoder threads, GGML_METAL_N_CB overrides the default of 1 +// capped below GGML_METAL_MAX_COMMAND_BUFFERS (the free path only releases that many); note that with an +// abort callback set, n_cb >= 2 leaves the main command buffer uncommitted (ggml-metal-context.m) +static int ggml_backend_metal_default_n_cb() { + const char * env = getenv("GGML_METAL_N_CB"); + if (env == nullptr || *env == '\0') { + return 1; + } + char * end = nullptr; + const long v = strtol(env, &end, 10); + if (*end != '\0' || v < 0 || v > 7) { // GGML_METAL_MAX_COMMAND_BUFFERS - 1 + GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_CB='%s'\n", __func__, env); + return 1; + } + return (int) v; +} + static ggml_backend_i ggml_backend_metal_i = { /* .get_name = */ ggml_backend_metal_name, /* .free = */ ggml_backend_metal_free, @@ -700,7 +718,7 @@ ggml_backend_t ggml_backend_metal_init(void) { /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend; } @@ -795,7 +813,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t /* .context = */ ctx, }; - ggml_backend_metal_set_n_cb(backend, 1); + ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb()); return backend; diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal index 5231b8395..bb275b88c 100644 --- a/ggml/src/ggml-metal/kernels/argsort.metal +++ b/ggml/src/ggml-metal/kernels/argsort.metal @@ -342,6 +342,125 @@ kernel void kernel_top_k_f32_i32( } } +// radix-select with 11-bit digits (3 histogram passes over the row instead of 4), the bucket holding the k-th value +// found by a threadgroup prefix sum instead of a serial scan, and the strict winners and the ties written in one +// pass. Same result contract as kernel_top_k_f32_i32 (unordered indices, ties in any order). +kernel void kernel_top_k_f32_i32_r11( + constant ggml_metal_kargs_top_k & args, + device const char * src0, + device int32_t * dst, + threadgroup atomic_uint * histo [[threadgroup(0)]], // 2048 + threadgroup uint * sh [[threadgroup(1)]], // 32 simdgroup sums + bucket + above + threadgroup atomic_uint * cnt [[threadgroup(2)]], // above / ties counters + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + const uint ncols = args.ne00; + const uint top_k = args.top_k; + const uint i01 = tgpig[0]; + const uint i02 = tgpig[1]; + const uint i03 = tgpig[2]; + + device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03); + + device int32_t * dst_row = dst + top_k*(i01 + args.ne01*i02 + args.ne01*args.ne02*i03); + + const uint tid = tpitg.x; + const uint nt = ntg.x; // power of two, <= 1024 + const uint nsg = (nt + 31)/32; + + uint prefix = 0; // fixed high bits of the threshold key + uint desired = top_k; // count still needed from the candidate range + + for (int p = 0; p < 3; ++p) { + const uint shift = p == 0 ? 21u : p == 1 ? 10u : 0u; + const uint nbits = p == 2 ? 10u : 11u; + const uint nb = 1u << nbits; + + for (uint i = tid; i < nb; i += nt) { + atomic_store_explicit(&histo[i], 0u, memory_order_relaxed); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint hi_mask = p == 0 ? 0u : (0xFFFFFFFFu << (shift + nbits)); + const uint prefix_hi = prefix & hi_mask; + + for (uint i = tid; i < ncols; i += nt) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + if ((key & hi_mask) == prefix_hi) { + atomic_fetch_add_explicit(&histo[(key >> shift) & (nb - 1)], 1u, memory_order_relaxed); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + // buckets in descending order: thread t owns [nb - (t + 1)*per, nb - t*per) + const uint per = nb >= nt ? nb/nt : 1u; + const uint hi = tid*per < nb ? nb - tid*per : 0u; // exclusive top bucket of this thread + uint c = 0; + for (uint j = 0; j < per && hi > j; ++j) { + c += atomic_load_explicit(&histo[hi - 1 - j], memory_order_relaxed); + } + const uint incl = simd_prefix_inclusive_sum(c); + if (tiisg == 31 || tid == nt - 1) { + sh[sgitg] = incl; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + uint base = 0; + for (uint w = 0; w < sgitg; ++w) { + base += sh[w]; + } + const uint my_incl = base + incl; + const uint my_excl = my_incl - c; + if (c > 0 && my_excl < desired && my_incl >= desired) { + uint acc = my_excl; + for (uint j = 0; j < per; ++j) { + const uint b = hi - 1 - j; + const uint h = atomic_load_explicit(&histo[b], memory_order_relaxed); + if (acc + h >= desired) { + sh[32] = b; + sh[33] = acc; + break; + } + acc += h; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + prefix |= sh[32] << shift; + desired -= sh[33]; + + // every thread has read sh before the next pass reuses it + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + // prefix is the full key of the k-th largest value; desired is how many of its ties are needed + const uint threshold = prefix; + const uint n_above = top_k - desired; + + if (tid == 0) { + atomic_store_explicit(&cnt[0], 0u, memory_order_relaxed); + atomic_store_explicit(&cnt[1], 0u, memory_order_relaxed); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint i = tid; i < ncols; i += nt) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + if (key > threshold) { + const uint pos = atomic_fetch_add_explicit(&cnt[0], 1u, memory_order_relaxed); + dst_row[pos] = (int32_t) i; + } else if (key == threshold) { + const uint t = atomic_fetch_add_explicit(&cnt[1], 1u, memory_order_relaxed); + if (t < desired) { + dst_row[n_above + t] = (int32_t) i; + } + } + } +} + // fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. // One SIMDgroup handles one token row; n_expert is limited to 1024 by the host. kernel void kernel_topk_moe_f32( @@ -470,10 +589,16 @@ kernel void kernel_moe_reduce_f32( const int n_expert_used = FC_moe_reduce_n_expert_used; - const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col; - float sum = 0.0f; - FOR_UNROLL (int e = 0; e < n_expert_used; ++e) { - sum += experts[base + e * args.ne00] * weights[token * n_expert_used + e]; + // the unfused MUL and ADD chain: every product rounded, then added in expert order (no fused multiply-add) + { +#pragma METAL fp math_mode(safe) +#pragma METAL fp contract(off) + const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col; + float sum = experts[base] * weights[token * n_expert_used]; + FOR_UNROLL (int e = 1; e < n_expert_used; ++e) { + const float p = experts[base + (int64_t) e * args.ne00] * weights[token * n_expert_used + e]; + sum = sum + p; + } + dst[token * args.ne00 + col] = sum; } - dst[token * args.ne00 + col] = sum; } diff --git a/ggml/src/ggml-metal/kernels/fa_aux.metal b/ggml/src/ggml-metal/kernels/fa_aux.metal index 6fe47470f..4b9081276 100644 --- a/ggml/src/ggml-metal/kernels/fa_aux.metal +++ b/ggml/src/ggml-metal/kernels/fa_aux.metal @@ -416,6 +416,7 @@ kernel void kernel_lightning_indexer( device const char * w, device const char * m, device char * dst, + device const int * rows, uint3 tgpig[[threadgroup_position_in_grid]], ushort tiitg[[thread_index_in_threadgroup]], ushort tiisg[[thread_index_in_simdgroup]], @@ -448,7 +449,8 @@ kernel void kernel_lightning_indexer( half4x4 tmp; if (i_kv_0 + ik < args.n_kv) { - device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3); + const int ir = args.has_rows ? rows[i_kv_0 + ik] : i_kv_0 + ik; + device const kd4x4_t * kr = (device const kd4x4_t *) (k + (int64_t) ir*args.nbk2 + i_stream*args.nbk3); deq_k(kr + i16/nl_k, i16%nl_k, tmp); } else { @@ -557,3 +559,367 @@ template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_ template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_0, 2, dequantize_q5_0>; template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_1, 2, dequantize_q5_1>; template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q8_0, 2, dequantize_q8_0>; + +// merge the nwg partial results of kernel_hy4_mla_decode_v2 (nwg <= 32): one threadgroup per head +kernel void kernel_hy4_mla_reduce( + constant ggml_metal_kargs_hy4_mla & args, + device const char * htmp, + device char * dst, + uint tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort ntg [[threads_per_threadgroup]]) { + constexpr int DV4 = 512/4; + const int NWG = args.nwg; + const int rid = tgpig; + + device const float * ss = (device const float *) htmp + (int64_t) args.n_head*512*NWG + rid*(2*NWG); + + float m = -FLT_MAX/2; + for (int w = 0; w < NWG; ++w) { + m = max(m, ss[2*w + 1]); + } + float S = 0.0f; + float ms[32]; + for (int w = 0; w < NWG; ++w) { + ms[w] = exp(ss[2*w + 1] - m); + S += ss[2*w + 0]*ms[w]; + } + const float iS = S == 0.0f ? 0.0f : 1.0f/S; + + // partials [head][workgroup][DV]: each workgroup writes its rows contiguously + device const float4 * h4 = (device const float4 *) htmp + (int64_t) rid*DV4*NWG; + device float4 * d4 = (device float4 *) dst + (int64_t) rid*DV4; + + for (int i = tiitg; i < DV4; i += ntg) { + float4 v = 0.0f; + for (int w = 0; w < NWG; ++w) { + v += h4[w*DV4 + i]*ms[w]; + } + d4[i] = v*iS; + } +} + +#define HY4_MLA_LMAX 1024 // longest list mode 3 compacts in threadgroup memory + +// hy4 MLA decode: one query token, n_head query heads over one latent KV head (DK 576, V = the first 512 dims of +// K). The rows come from the compacted list of the visible cells: built by kernel_flash_attn_ext_vec_idx (padded +// with -1), or, for a short mask, by every threadgroup itself. NG groups of 8 heads per threadgroup share every K tile, so a row is read +// once per NG*8 heads (the vec kernel reads it once per head), and both products run on 8x8 simdgroup matrices. +// Simdgroup (g, s) owns head group g and quarter s of DK (for S) and of DV (for O); its Q quarter stays in +// registers. The unnormalized O, S and M of every workgroup go to kernel_hy4_mla_reduce. +template<short NG> +kernel void kernel_hy4_mla_decode_v2( + constant ggml_metal_kargs_hy4_mla & args, + device const char * q, + device const char * k, + device const int * idx, + device const char * sinks, + device char * dst, + device const half * mask, + device const char * score, // mode 4 + device const char * blocks, // mode 4 + device const char * extra, // mode 4 + threadgroup half * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr int DK = 576; + constexpr int DV = 512; + constexpr int TR = 16; // K rows per tile + constexpr int NQ = 4; // quarters of DK/DV per head group + constexpr int NT = NG*NQ*32; + constexpr int DVS = DV/NQ; // 128 O columns per simdgroup + constexpr int KBS = DK/8/NQ; // 18 DK blocks per simdgroup + constexpr int HT = NG*8; // heads per threadgroup + + constexpr float M_NONE = -FLT_MAX/2; + + const int iwg = tgpig.x; + const int h0 = tgpig.y*HT; + const int NWG = args.nwg; + + const short g = sgitg/NQ; // head group of this simdgroup + const short qs = sgitg%NQ; // DK/DV quarter of this simdgroup + + threadgroup half * sk = shmem; // [TR][DK] (also Q staging and the O output) + threadgroup float * ss = (threadgroup float *) (sk + TR*DK); // [NG][NQ][8][TR] partial S + threadgroup half * sp = (threadgroup half *) (ss + NG*NQ*8*TR);// [NG][8][TR] P + threadgroup float * sd = (threadgroup float *) (sp + NG*8*TR); // [NG][8][8] diag(ms) + threadgroup float * sm = sd + NG*64; // M[HT] + threadgroup float * sS = sm + HT; // S[HT] + threadgroup int * sr = (threadgroup int *) (sS + HT); // row ids of the tile [TR], -1 = none + threadgroup int * sn = sr + TR; // valid rows in the tile + threadgroup float * sv = (threadgroup float *) (sn + 4); // mask value of each tile row [TR] + threadgroup int * sl = (threadgroup int *) (sv + TR); // mode 3: the compacted list [HY4_MLA_LMAX] + threadgroup int * sc = sl + HY4_MLA_LMAX; // mode 3: per-simdgroup counts [32] + + // Q -> registers, staged through sk in halves of 16 heads at most (16*576 halfs == TR*DK) + simdgroup_half8x8 mq[KBS]; + for (short gg = 0; gg < NG; gg += 2) { + threadgroup_barrier(mem_flags::mem_threadgroup); + const int nh = min(2, NG - gg)*8; + for (int i = tiitg; i < nh*DK/4; i += NT) { + const int h = h0 + gg*8 + i/(DK/4); + const int c = i%(DK/4); + const float4 v = h < args.n_head ? ((device const float4 *) (q + h*args.nb_q_head))[c] : float4(0.0f); + ((threadgroup half4 *) sk)[i] = (half4) v; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (g == gg || g == gg + 1) { + for (short b = 0; b < KBS; ++b) { + simdgroup_load(mq[b], sk + (g - gg)*8*DK + (qs*KBS + b)*8, DK); + } + } + } + if (tiitg < HT) { + sm[tiitg] = M_NONE; + sS[tiitg] = 0.0f; + } + + simdgroup_float8x8 mo[DVS/8]; + for (short i = 0; i < DVS/8; ++i) { + mo[i] = make_filled_simdgroup_matrix<float, 8>(0.0f); + } + + // mode 3: every threadgroup compacts the finite cells of the (short) mask row itself, in cell order, so all + // the workgroups see the same list and no separate compaction dispatch is needed + int n_rows = args.n_rows; + if (args.has_mask == 3) { + const int n = args.n_cells; + const int qn = n/NT; + const int rn = n%NT; + const int c0 = qn*tiitg + min((int) tiitg, rn); + const int c1 = c0 + qn + (tiitg < rn ? 1 : 0); + + int cnt = 0; + for (int c = c0; c < c1; ++c) { + cnt += isfinite((float) mask[c]) ? 1 : 0; + } + const int incl = simd_prefix_inclusive_sum(cnt); + if ((tiitg % 32) == 31) { + sc[sgitg] = incl; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + int base = incl - cnt; + int total = 0; + for (short w = 0; w < NT/32; ++w) { + base += w < sgitg ? sc[w] : 0; + total += sc[w]; + } + int j = base; + for (int c = c0; c < c1 && j < HY4_MLA_LMAX; ++c) { + if (isfinite((float) mask[c])) { + sl[j++] = c; + } + } + n_rows = min(total, HY4_MLA_LMAX); + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + // mode 3 splits the list it counted, so cells beyond the host's hint (e.g. repeated positions) are not dropped + const int rows_per_wg = args.has_mask == 3 ? ((n_rows + NWG*TR - 1)/(NWG*TR))*TR : args.rows_per_wg; + + const int r0 = iwg*rows_per_wg; + const int r1 = min(r0 + rows_per_wg, n_rows); + + // softmax layout: thread t -> group t/128, head (t/16)%8, row t%16 + const int tg = tiitg/128; + const int th = (tiitg/TR)%8; + const int tr = tiitg%TR; + + for (int rt = r0; rt < r1; rt += TR) { + threadgroup_barrier(mem_flags::mem_threadgroup); + if (tiitg < TR) { + const int r = rt + tiitg; + int row = -1; + if (r < r1) { + if (args.has_mask == 4) { + // the cells TOP_K_MASK would keep: the members of the picked pools with a finite score, then the tail + const int n_pool_rows = args.n_top*args.n_blk; + int c = -1; + if (r < n_pool_rows) { + const int p = *(device const int32_t *) ((device const char *) idx + (r/args.n_blk)*args.nbk0); + if (p >= 0 && p < args.n_pick && *(device const float *) (score + p*args.nbs0) > -INFINITY) { + c = *(device const int32_t *) (blocks + (r%args.n_blk)*args.nbb0 + p*args.nbb1); + } + } else { + c = *(device const int32_t *) (extra + (r - n_pool_rows)*args.nbe0); + } + row = c >= 0 && c < args.n_cells ? c : -1; + } else { + row = args.has_mask == 3 ? sl[r] : idx[r]; + } + } + // mode 4 keeps a zero-filled row: the value is 0 + float mv = 0.0f; + if (row >= 0 && args.has_mask && args.has_mask != 4) { + mv = (float) mask[row]; + if (!isfinite(mv)) { + row = -1; + } + } + sr[tiitg] = row; + sv[tiitg] = mv; + } + if (tiitg == 0) { + sn[0] = 0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (tiitg < TR && sr[tiitg] >= 0) { + atomic_fetch_add_explicit((threadgroup atomic_int *) sn, 1, memory_order_relaxed); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + // a compacted list ends in padding; the mode 4 list can have empty tiles anywhere + if (sn[0] == 0) { + if (args.has_mask == 4) { + continue; + } + break; + } + + for (int i = tiitg; i < TR*DK/8; i += NT) { + const int rr = i/(DK/8); + const int c = i%(DK/8); + const int row = sr[rr]; + ((threadgroup float4 *) sk)[i] = row >= 0 ? ((device const float4 *) (k + (uint64_t) row*args.nb_k_row))[c] : float4(0.0f); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + { + simdgroup_float8x8 ms0 = make_filled_simdgroup_matrix<float, 8>(0.0f); + simdgroup_float8x8 ms1 = make_filled_simdgroup_matrix<float, 8>(0.0f); + for (short b = 0; b < KBS; ++b) { + const short kb = qs*KBS + b; + simdgroup_half8x8 mk0; + simdgroup_half8x8 mk1; + simdgroup_load(mk0, sk + 0*DK + kb*8, DK, 0, true); + simdgroup_load(mk1, sk + 8*DK + kb*8, DK, 0, true); + simdgroup_multiply_accumulate(ms0, mq[b], mk0, ms0); + simdgroup_multiply_accumulate(ms1, mq[b], mk1, ms1); + } + simdgroup_store(ms0, ss + (g*NQ + qs)*8*TR + 0, TR); + simdgroup_store(ms1, ss + (g*NQ + qs)*8*TR + 8, TR); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tg < NG) { + const bool valid = sr[tr] >= 0; + float s = 0.0f; + for (short n = 0; n < NQ; ++n) { + s += ss[(tg*NQ + n)*8*TR + th*TR + tr]; + } + s = s*args.scale + sv[tr]; + + float mx = valid ? s : M_NONE; + mx = max(mx, simd_shuffle_xor(mx, 8)); + mx = max(mx, simd_shuffle_xor(mx, 4)); + mx = max(mx, simd_shuffle_xor(mx, 2)); + mx = max(mx, simd_shuffle_xor(mx, 1)); + + const int hh = tg*8 + th; + const float mold = sm[hh]; + const float mnew = max(mold, mx); + const float p = valid ? exp(s - mnew) : 0.0f; + const float msc = exp(mold - mnew); + + float ps = p; + ps += simd_shuffle_xor(ps, 8); + ps += simd_shuffle_xor(ps, 4); + ps += simd_shuffle_xor(ps, 2); + ps += simd_shuffle_xor(ps, 1); + + sp[(tg*8 + th)*TR + tr] = (half) p; + if (tr < 8) { + sd[tg*64 + th*8 + tr] = tr == th ? msc : 0.0f; + } + // every lane of the head read sm[hh] before lane 0 writes it: the 16 lanes are one simdgroup half + simdgroup_barrier(mem_flags::mem_threadgroup); + if (tr == 0) { + sm[hh] = mnew; + sS[hh] = sS[hh]*msc + ps; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + { + simdgroup_float8x8 md; + simdgroup_half8x8 mp0; + simdgroup_half8x8 mp1; + simdgroup_load(md, sd + g*64, 8); + simdgroup_load(mp0, sp + g*8*TR + 0, TR); + simdgroup_load(mp1, sp + g*8*TR + 8, TR); + for (short i = 0; i < DVS/8; ++i) { + const short d0 = qs*DVS + i*8; + simdgroup_half8x8 mv0; + simdgroup_half8x8 mv1; + simdgroup_load(mv0, sk + 0*DK + d0, DK); + simdgroup_load(mv1, sk + 8*DK + d0, DK); + simdgroup_multiply(mo[i], md, mo[i]); + simdgroup_multiply_accumulate(mo[i], mp0, mv0, mo[i]); + simdgroup_multiply_accumulate(mo[i], mp1, mv1, mo[i]); + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (args.has_sinks && iwg == 0) { + const int hh = tiitg/8; + const int c = tiitg%8; + float mn = 0.0f; + float msc = 0.0f; + float vs = 0.0f; + if (tiitg < HT*8) { + const float m = sm[hh]; + const float snk = h0 + hh < args.n_head ? ((device const float *) sinks)[h0 + hh] : M_NONE; + mn = max(m, snk); + msc = exp(m - mn); + vs = h0 + hh < args.n_head ? exp(snk - mn) : 0.0f; + sd[(hh/8)*64 + (hh%8)*8 + c] = c == hh%8 ? msc : 0.0f; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (tiitg < HT*8 && c == 0) { + sm[hh] = mn; + sS[hh] = sS[hh]*msc + vs; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + simdgroup_float8x8 md; + simdgroup_load(md, sd + g*64, 8); + for (short i = 0; i < DVS/8; ++i) { + simdgroup_multiply(mo[i], md, mo[i]); + } + } + + device float4 * dst4 = (device float4 *) dst; + device float * dst1 = (device float *) dst + (int64_t) args.n_head*DV*NWG; + + // O out through sk, one head group at a time ([8][DV] floats fit in the [TR][DK] halfs) + for (short gg = 0; gg < NG; ++gg) { + threadgroup_barrier(mem_flags::mem_threadgroup); + threadgroup float * so = (threadgroup float *) sk; + if (g == gg) { + for (short i = 0; i < DVS/8; ++i) { + simdgroup_store(mo[i], so + qs*DVS + i*8, DV); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (int i = tiitg; i < 8*DV/4; i += NT) { + const int h = h0 + gg*8 + i/(DV/4); + const int c = i%(DV/4); + if (h < args.n_head) { + dst4[((int64_t) h*NWG + iwg)*(DV/4) + c] = ((threadgroup float4 *) so)[i]; + } + } + } + if (tiitg < HT && h0 + tiitg < args.n_head) { + dst1[(h0 + tiitg)*(2*NWG) + 2*iwg + 0] = sS[tiitg]; + dst1[(h0 + tiitg)*(2*NWG) + 2*iwg + 1] = sm[tiitg]; + } +} + +typedef decltype(kernel_hy4_mla_decode_v2<4>) kernel_hy4_mla_decode_v2_t; + +template [[host_name("kernel_hy4_mla_decode_v2_ng2")]] kernel kernel_hy4_mla_decode_v2_t kernel_hy4_mla_decode_v2<2>; diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal index 5dfd4a60c..295fbe8b2 100644 --- a/ggml/src/ggml-metal/kernels/misc.metal +++ b/ggml/src/ggml-metal/kernels/misc.metal @@ -511,6 +511,52 @@ typedef decltype(kernel_count_equal<int32_t>) kernel_count_equal_t; template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal<int32_t>; +kernel void kernel_hc_gate_f32( + constant ggml_metal_kargs_hc_gate & args, + device const char * x, + device const float * f, + device const float * b, + device float * dst, + uint tgpig [[threadgroup_position_in_grid]], + uint tpitg [[thread_position_in_threadgroup]], + uint ntg [[threads_per_threadgroup]]) { + const int idx = tgpig * ntg + tpitg; + if (idx >= args.n * args.n_rows) { + return; + } + + const int i = idx % args.n; + const int r = idx / args.n; + + // same op order as the unfused chain: mul, add, sigmoid, then the scales + const float z = ((device const float *) (x + r*args.nb_x1))[i] * f[0] + b[i]; + const float s = 1.0f / (1.0f + exp(-z)); + dst[idx] = args.s1 * (args.s0 * s + args.c0) + args.c1; +} + +// x * sigmoid(g) over n contiguous floats, 4 per thread; the same sigmoid as kernel_unary (sigmoid op) +kernel void kernel_sigmoid_mul_f32( + constant int32_t & n, + device const float * g, + device const float * x, + device float * dst, + uint tpig [[thread_position_in_grid]]) { + const int i = tpig*4; + if (i >= n) { + return; + } + if (n - i >= 4) { + const float4 gv = ((device const float4 *) g)[tpig]; + const float4 s = 1.0f / (1.0f + exp(-gv)); + ((device float4 *) dst)[tpig] = ((device const float4 *) x)[tpig] * s; + } else { + for (int j = i; j < n; ++j) { + const float s = 1.0f / (1.0f + exp(-g[j])); + dst[j] = x[j] * s; + } + } +} + template <typename T> kernel void kernel_snake( constant ggml_metal_kargs_snake & args, @@ -810,6 +856,227 @@ kernel void kernel_dsv4_hc_pre_gated_f32( *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; } +// DSV4_HC_PRE + RMS_NORM (+ MUL, F == 2): one threadgroup per token with the float4 thread mapping and the +// reduction of kernel_rms_norm_fuse_impl<float4>, each element computed as kernel_dsv4_hc_pre(_gated) does, so +// the result is the unfused one bit for bit. WM: 0 the weights, 1 gated (sigmoid of a per-element gate), 2 the +// weights computed from the mixes as kernel_hc_gate_f32 does (the hc gate fused in front; the same bits as that +// kernel followed by these ops) +template <short WM, short F> +kernel void kernel_hy4_hc_pre_rms_f32( + constant ggml_metal_kargs_hy4_hc_pre_rms & args, + device const char * x, + device const char * w, + device const char * f0, + device char * dst, + device const float * gf, + device const float * gb, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constant ggml_metal_kargs_dsv4_hc_pre & hc = args.hc; + + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + const int it = tgpig.x; + + float wg[8]; // the host limits the fused gate to 8 streams + if (WM == 2) { + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + const float z = ((device const float *) (w + it*args.nb_m1))[ih] * gf[0] + gb[ih]; + const float s = 1.0f / (1.0f + exp(-z)); + wg[ih] = args.s1 * (args.s0 * s + args.c0) + args.c1; + } + } + + auto pre = [&](int i00) { + float4 v; + FOR_UNROLL (short c = 0; c < 4; ++c) { + const int i0 = 4*i00 + c; + device const char * xb = x + i0*hc.nb_x0 + it*hc.nb_x2; + float result = 0.0f; + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + float wv; + if (WM == 2) { + wv = wg[ih]; + } else if (WM == 1) { + wv = 1.0f/(1.0f + exp(-*(device const float *) (w + i0*hc.nb_w0 + ih*hc.nb_w1 + it*hc.nb_w2))); + } else { + wv = *(device const float *) (w + ih*hc.nb_w0 + it*hc.nb_w1); + } + const float xv = *(device const float *) (xb + ih*hc.nb_x1); + result = fma(xv, wv, result); + } + v[c] = hc.scale*result; + } + return v; + }; + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + const float4 v = pre(i00); + sumf += dot(v, v); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float mean = sumf/hc.n_embd; + const float scale = 1.0f/sqrt(mean + args.eps); + + device float4 * y = (device float4 *) (dst + it*args.nb_y1); + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + const float4 v = pre(i00); + if (F == 1) { + y[i00] = (v*scale); + } + if (F == 2) { + y[i00] = (v*scale)*((device const float4 *) f0)[i00]; + } + } +} + +typedef decltype(kernel_hy4_hc_pre_rms_f32<0, 1>) kernel_hy4_hc_pre_rms_t; + +template [[host_name("kernel_hy4_hc_pre_rms_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<0, 1>; +template [[host_name("kernel_hy4_hc_pre_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<0, 2>; +template [[host_name("kernel_hy4_hc_pre_gated_rms_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<1, 1>; +template [[host_name("kernel_hy4_hc_pre_gated_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<1, 2>; +template [[host_name("kernel_hy4_hc_gate_pre_rms_mul_f32")]] kernel kernel_hy4_hc_pre_rms_t kernel_hy4_hc_pre_rms_f32<2, 2>; + +// ADD + RMS_NORM + MUL: the sum is formed in registers in both passes instead of being stored and read back. +// Same float4 thread mapping and reduction as kernel_rms_norm_fuse_impl<float4, 2>, so the result is the unfused +// one bit for bit (the sum of two floats rounds the same way in registers as in memory). +kernel void kernel_hy4_add_rms_norm_mul_f32( + constant ggml_metal_kargs_hy4_add_rms & args, + device const char * a, + device const char * b, + device const char * w, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + const int i01 = tgpig.x; + + device const float4 * av = (device const float4 *) (a + i01*args.nb_a1); + device const float4 * bv = (device const float4 *) (b + i01*args.nb_b1); + device const float4 * wv = (device const float4 *) w; + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + const float4 v = av[i00] + bv[i00]; + sumf += dot(v, v); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float mean = sumf/args.ne00; + const float scale = 1.0f/sqrt(mean + args.eps); + + device float4 * y = (device float4 *) (dst + i01*args.nb_y1); + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + const float4 v = av[i00] + bv[i00]; + y[i00] = (v*scale)*wv[i00]; + } +} + +// DSV4_HC_POST (no comb) + RMS_NORM of the flattened streams: one threadgroup per token with the float4 mapping +// and the reduction of kernel_rms_norm_fuse_impl<float4>; the streams are computed as kernel_dsv4_hc_post_nocomb +// does, written out, and read back for the norm +kernel void kernel_hy4_hc_post_rms_f32( + constant ggml_metal_kargs_hy4_hc_post_rms & args, + device const char * x, + device const char * residual, + device const char * post, + device char * dst, + device char * dst_rms, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + const int it = tgpig.x; + const int ne4 = args.n_embd/4; + + device float4 * ys = (device float4 *) (dst + it*args.nb_d2); + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + const int idst = i00/ne4; + const int i0 = 4*(i00%ne4); + const float p = *(device const float *) (post + idst*args.nb_p0 + it*args.nb_p1); + float4 v; + FOR_UNROLL (short c = 0; c < 4; ++c) { + const float xv = *(device const float *) (x + (i0 + c)*args.nb_x0 + it*args.nb_x1); + const float rv = *(device const float *) (residual + (i0 + c)*args.nb_r0 + idst*args.nb_r1 + it*args.nb_r2); + v[c] = xv*p + rv; + } + ys[i00] = v; + sumf += dot(v, v); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float mean = sumf/(4*args.ne00_t); + const float scale = 1.0f/sqrt(mean + args.eps); + + device float4 * y = (device float4 *) (dst_rms + it*args.nb_y1); + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + y[i00] = (ys[i00]*scale); + } +} + kernel void kernel_dsv4_hc_post_nocomb_f32( constant ggml_metal_kargs_dsv4_hc_post & args, device const char * x, diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal index 5d012e484..41b6b6f4c 100644 --- a/ggml/src/ggml-metal/kernels/mul_mv.metal +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -597,6 +597,89 @@ kernel void kernel_mul_mv_q8_0_f32( kernel_mul_mv_q8_0_f32_impl<N_R0_Q8_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } +// Q8_0 mat-vec for short rows (ne00 <= 512, e.g. the per-head MLA projections wk_b/wv_b): each simdgroup owns +// R whole rows and reduces each with one simd_sum, so there is no cross-simdgroup reduction and no barrier. A +// lane keeps up to two 8-wide chunks of the activation in registers and reuses them for all R rows, and the +// loads of all R rows are issued before any reduction. +template<short R> +kernel void kernel_mul_mv_q8_0_f32_short( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_nsg; + + const int nq = args.ne00/8; // 8-wide chunks per row, at most 64 + + const int r0 = (tgpig.x*NSG + sgitg)*R; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + device const float * y = (device const float *) (src1 + r1*args.nb11 + i12*args.nb12 + i13*args.nb13); + + const int c0 = tiisg; + const int c1 = tiisg + 32; + + float y0[8]; + float y1[8]; + FOR_UNROLL (short i = 0; i < 8; ++i) { + y0[i] = c0 < nq ? y[c0*8 + i] : 0.0f; + y1[i] = c1 < nq ? y[c1*8 + i] : 0.0f; + } + + device float * dst_f32 = (device float *) dst + (uint64_t) im*args.ne0*args.ne1 + (uint64_t) r1*args.ne0; + + const uint64_t offset02 = (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + // issue the loads of all R rows before reducing, so they are in flight together + float sum[R]; + FOR_UNROLL (short row = 0; row < R; ++row) { + const int r = min(r0 + row, args.ne01 - 1); + + device const block_q8_0 * ax = (device const block_q8_0 *) (src0 + r*args.nb01 + offset02); + + float acc = 0.0f; + if (c0 < nq) { + device const int8_t * qs = ax[c0/4].qs + (c0%4)*8; + float s = 0.0f; + FOR_UNROLL (short i = 0; i < 8; ++i) { + s += qs[i]*y0[i]; + } + acc += s*(float) ax[c0/4].d; + } + if (c1 < nq) { + device const int8_t * qs = ax[c1/4].qs + (c1%4)*8; + float s = 0.0f; + FOR_UNROLL (short i = 0; i < 8; ++i) { + s += qs[i]*y1[i]; + } + acc += s*(float) ax[c1/4].d; + } + sum[row] = acc; + } + + FOR_UNROLL (short row = 0; row < R; ++row) { + const float tot = simd_sum(sum[row]); + if (tiisg == 0 && r0 + row < args.ne01) { + dst_f32[r0 + row] = tot; + } + } + + (void) shmem; +} + +typedef decltype(kernel_mul_mv_q8_0_f32_short<4>) kernel_mul_mv_q8_0_f32_short_t; + +template [[host_name("kernel_mul_mv_q8_0_f32_short_r4")]] kernel kernel_mul_mv_q8_0_f32_short_t kernel_mul_mv_q8_0_f32_short<4>; +template [[host_name("kernel_mul_mv_q8_0_f32_short_r2")]] kernel kernel_mul_mv_q8_0_f32_short_t kernel_mul_mv_q8_0_f32_short<2>; + // mat-vec kernel processing in chunks of float4 // chpb - chunks per quantization block template<short r1ptg, typename q_t, short chpb, void (*deq_t4)(device const q_t *, short, thread float4 &) > diff --git a/ggml/src/ggml-metal/kernels/rope.metal b/ggml/src/ggml-metal/kernels/rope.metal index 401ceacb0..7cae7dc49 100644 --- a/ggml/src/ggml-metal/kernels/rope.metal +++ b/ggml/src/ggml-metal/kernels/rope.metal @@ -44,20 +44,16 @@ static void rope_yarn_corr_dims( dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base))); } +// one row of kernel_rope_norm: dst is the row's base (the channels at i0*nb0 from it) template<typename T> -kernel void kernel_rope_norm( +static inline void rope_norm_row( constant ggml_metal_kargs_rope & args, device const char * src0, device const char * src1, device const char * src2, device char * dst, - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 tptg [[threads_per_threadgroup]], - uint3 tgpig[[threadgroup_position_in_grid]]) { - const int i3 = tgpig[2]; - const int i2 = tgpig[1]; - const int i1 = tgpig[0]; - + const int i1, const int i2, const int i3, + ushort tiitg, ushort3 tptg) { float corr_dims[2]; rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); @@ -81,7 +77,7 @@ kernel void kernel_rope_norm( rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, iw, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device T * dst_data = (device T *)( dst + i0*args.nb0); const float x0 = src[0]; const float x1 = src[1]; @@ -94,7 +90,7 @@ kernel void kernel_rope_norm( } device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device T * dst_data = (device T *)( dst + i0*args.nb0); dst_data[0] = src[0]; dst_data[1] = src[1]; @@ -102,6 +98,50 @@ kernel void kernel_rope_norm( } } +template<typename T> +kernel void kernel_rope_norm( + constant ggml_metal_kargs_rope & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + rope_norm_row<T>(args, src0, src1, src2, dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1, i1, i2, i3, tiitg, tptg); +} + +// ROPE (norm) + CONCAT along dim 0 with the rotated tensor second (hy_v4_flash: q and k as [nope | rope]): the +// row of the first source is copied, the rotation written right after it, as the two kernels would +kernel void kernel_hy4_rope_concat_f32( + constant ggml_metal_kargs_rope & args, + constant ggml_metal_kargs_concat & cc, + device const char * src0, + device const char * src1, + device const char * src2, + device const char * srcc, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + device char * row = dst + i3*cc.nb3 + i2*cc.nb2 + i1*cc.nb1; + + device const char * crow = srcc + i3*cc.nb03 + i2*cc.nb02 + i1*cc.nb01; + for (int j = tiitg; j < cc.ne00; j += tptg.x) { + *((device float *) (row + j*cc.nb0)) = *((device const float *) (crow + j*cc.nb00)); + } + + rope_norm_row<float>(args, src0, src1, src2, row + cc.ne00*cc.nb0, i1, i2, i3, tiitg, tptg); +} + template<typename T> kernel void kernel_rope_neox( constant ggml_metal_kargs_rope & args, diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 9999aac90..7de14d158 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -11815,7 +11815,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op) && backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_argmax_f32_i32) >= 64; case GGML_OP_LIGHTNING_INDEXER: - return op->src[0]->type == GGML_TYPE_F32 && + return op->src[4] == nullptr && op->src[0]->type == GGML_TYPE_F32 && (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_BF16) && op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F16 && op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type) && diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 9ee16423a..3ca421c3b 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -7394,7 +7394,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) && op->type == GGML_TYPE_F32; case GGML_OP_LIGHTNING_INDEXER: - return op->src[0]->type == GGML_TYPE_F32 && + return op->src[4] == nullptr && op->src[0]->type == GGML_TYPE_F32 && (op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_Q8_0 || op->src[1]->type == GGML_TYPE_Q5_1 || op->src[1]->type == GGML_TYPE_Q5_0 || diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ba8578716..afb0fedc9 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -16070,6 +16070,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && op->src[0]->ne[0] == 64; case GGML_OP_LIGHTNING_INDEXER: { + if (op->src[4] != nullptr) { + return false; // the row-list form is Metal/CPU only + } const ggml_tensor * q = op->src[0]; const ggml_tensor * k = op->src[1]; const ggml_tensor * w = op->src[2]; diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 3d60c44e5..970a043f5 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6553,6 +6553,44 @@ struct ggml_tensor * ggml_lightning_indexer( return result; } +struct ggml_tensor * ggml_lightning_indexer_rows( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * weights, + struct ggml_tensor * mask, + struct ggml_tensor * rows) { + + GGML_ASSERT( q->type == GGML_TYPE_F32); + GGML_ASSERT( weights->type == GGML_TYPE_F32); + GGML_ASSERT( mask->type == GGML_TYPE_F16); + GGML_ASSERT( rows->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_is_contiguous(rows) && ggml_nrows(rows) == 1); + GGML_ASSERT( q->ne[0] == k->ne[0]); + GGML_ASSERT( mask->ne[0] == rows->ne[0]); + GGML_ASSERT( q->ne[1] == weights->ne[0]); + GGML_ASSERT( k->ne[1] == 1); + GGML_ASSERT( mask->ne[1] == q->ne[2]); + GGML_ASSERT( q->ne[2] == weights->ne[1]); + GGML_ASSERT(weights->ne[2] == 1); + GGML_ASSERT( mask->ne[2] == 1); + GGML_ASSERT( q->ne[3] == k->ne[3]); + GGML_ASSERT( k->ne[3] == weights->ne[3]); + GGML_ASSERT(weights->ne[3] % mask->ne[3] == 0); + + int64_t ne[4] = { rows->ne[0], q->ne[2], 1, q->ne[3] }; + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + + result->op = GGML_OP_LIGHTNING_INDEXER; + result->src[0] = q; + result->src[1] = k; + result->src[2] = weights; + result->src[3] = mask; + result->src[4] = rows; + + return result; +} + // ggml_dsv4_hc_comb struct ggml_tensor * ggml_dsv4_hc_comb( diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97250c2e8..d15912560 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,6 +32,7 @@ set(LLAMA_CORE_SOURCES llama-memory-hybrid.cpp llama-memory-hybrid-iswa.cpp llama-memory-hybrid-idx.cpp + llama-memory-idx-iswa.cpp llama-memory-recurrent.cpp llama-mmap.cpp llama-model-loader.cpp diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 40f608371..b4df866e8 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -132,6 +132,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = { { LLM_ARCH_HUNYUAN_VL, "hunyuan_vl" }, { LLM_ARCH_HY_V3, "hy_v3" }, { LLM_ARCH_HY_V4, "hy_v4" }, + { LLM_ARCH_HY_V4_FLASH, "hy_v4_flash" }, { LLM_ARCH_SMOLLM3, "smollm3" }, { LLM_ARCH_OPENAI_MOE, "gpt-oss" }, { LLM_ARCH_LFM2, "lfm2" }, @@ -808,6 +809,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = { { LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "blk.%d.indexer_compressor_norm" }, { LLM_TENSOR_INDEXER_KPOOL_GATE, "blk.%d.indexer_compressor_gate" }, { LLM_TENSOR_INDEXER_KPOOL_APE, "blk.%d.indexer_compressor_ape" }, + { LLM_TENSOR_INDEXER_KPOOL_VALUE, "blk.%d.indexer_compressor_value" }, + { LLM_TENSOR_INDEXER_KPOOL_NORM, "blk.%d.indexer_compressor_norm" }, { LLM_TENSOR_FFN_GATE_TID2EID, "blk.%d.ffn_gate_tid2eid" }, { LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" }, { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, @@ -820,6 +823,10 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = { { LLM_TENSOR_DSPARK_MASK_EMBD, "mask_embd" }, { LLM_TENSOR_ENGRAM_KV, "blk.%d.engram_kv" }, { LLM_TENSOR_ENGRAM_GATE, "blk.%d.engram_gate" }, + { LLM_TENSOR_ENGRAM_KV_B, "blk.%d.engram_kv_b" }, + { LLM_TENSOR_ENGRAM_LOOKUP_NORM, "blk.%d.engram_lookup_norm" }, + { LLM_TENSOR_ENGRAM_PRE_GAMMA, "blk.%d.engram_pre_gamma" }, + { LLM_TENSOR_ENGRAM_HC_SCALE, "blk.%d.engram_hc_scale" }, { LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" }, { LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" }, { LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" }, @@ -1139,6 +1146,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = { {LLM_TENSOR_INDEXER_COMPRESSOR_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_INDEXER_KPOOL_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_INDEXER_KPOOL_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_INDEXER_KPOOL_VALUE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_INDEXER_KPOOL_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_GATE_TID2EID, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, @@ -1175,6 +1184,10 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = { {LLM_TENSOR_DSPARK_MASK_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_ENGRAM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ENGRAM_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_KV_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_ENGRAM_LOOKUP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_PRE_GAMMA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ENGRAM_HC_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, {LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1372,6 +1385,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK32: case LLM_ARCH_HY_V4: + case LLM_ARCH_HY_V4_FLASH: case LLM_ARCH_DOTS3NOTE: case LLM_ARCH_GLM_DSA: case LLM_ARCH_BITNET: diff --git a/src/llama-arch.h b/src/llama-arch.h index b98be40cf..afff4ca7e 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -136,6 +136,7 @@ enum llm_arch { LLM_ARCH_HUNYUAN_VL, LLM_ARCH_HY_V3, LLM_ARCH_HY_V4, + LLM_ARCH_HY_V4_FLASH, LLM_ARCH_SMOLLM3, LLM_ARCH_OPENAI_MOE, LLM_ARCH_LFM2, @@ -801,6 +802,8 @@ enum llm_tensor { LLM_TENSOR_INDEXER_COMPRESSOR_NORM, LLM_TENSOR_INDEXER_KPOOL_GATE, // glm5-next: k-pool gate scores LLM_TENSOR_INDEXER_KPOOL_APE, // glm5-next: k-pool position bias + LLM_TENSOR_INDEXER_KPOOL_VALUE, // hy_v4_flash: what the k-pool actually pools + LLM_TENSOR_INDEXER_KPOOL_NORM, // hy_v4_flash: norm of the pooled key LLM_TENSOR_FFN_GATE_TID2EID, LLM_TENSOR_NEXTN_PROJ_PRE, LLM_TENSOR_NEXTN_PROJ_POST, @@ -825,6 +828,10 @@ enum llm_tensor { LLM_TENSOR_DSPARK_MASK_EMBD, LLM_TENSOR_ENGRAM_KV, LLM_TENSOR_ENGRAM_GATE, + LLM_TENSOR_ENGRAM_KV_B, // hy_v4_flash: bias on the fused key/value projections + LLM_TENSOR_ENGRAM_LOOKUP_NORM, // hy_v4_flash: per-head RMSNorm of the looked-up rows + LLM_TENSOR_ENGRAM_PRE_GAMMA, // hy_v4_flash: scales the normed value before the gate + LLM_TENSOR_ENGRAM_HC_SCALE, // hy_v4_flash: per-hc per-channel additive gate offset LLM_TENSOR_DFLASH_ATTN_CONV_BASE, LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, LLM_TENSOR_DFLASH_FFN_CONV_BASE, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0cd795a56..0095311c6 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2715,7 +2715,8 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_01 || model.arch == LLM_ARCH_MINIMAX_M3 || - model.arch == LLM_ARCH_HY_V4) { + model.arch == LLM_ARCH_HY_V4 || + model.arch == LLM_ARCH_HY_V4_FLASH) { res = std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors()); } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) { // DFlash2's convolutions and selector are shape work rather than matmuls, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe8bdebe5..91955668a 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -834,8 +834,12 @@ void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) { mctx->get_swa()->set_input_attn_mask(self_attn_mask_swa, ubatch, cparams.causal_attn); } +const llama_kv_cache_iswa_context * llm_graph_input_attn_k_iswa::mctx_of(const llm_graph_params & params) const { + return static_cast<const llama_kv_cache_iswa_context *>(params.mctx); +} + bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) { - const auto * mctx = static_cast<const llama_kv_cache_iswa_context *>(params.mctx); + const auto * mctx = mctx_of(params); this->mctx = mctx; @@ -846,8 +850,10 @@ bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) { res &= self_k_idxs->ne[0] == params.ubatch.n_tokens; } + // build_attn_inp_k_iswa() builds plain masks without K/V row lists, so check them as such: + // can_reuse_attn_mask() expects row lists whenever fused_kv_rows yields a row count, and so never reused these if (self_attn_mask.raw && self_attn_mask.raw->buffer) { - res &= can_reuse_attn_mask(self_attn_mask, mctx->get_base(), params.ubatch, params.cparams); + res &= can_reuse_kq_mask(self_attn_mask.raw, mctx->get_base(), params.ubatch, params.cparams); } // swa tensors may not be allocated if there are no SWA attention layers @@ -856,7 +862,7 @@ bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) { } if (self_attn_mask_swa.raw && self_attn_mask_swa.raw->buffer) { - res &= can_reuse_attn_mask(self_attn_mask_swa, mctx->get_swa(), params.ubatch, params.cparams); + res &= can_reuse_kq_mask(self_attn_mask_swa.raw, mctx->get_swa(), params.ubatch, params.cparams); } return res; @@ -2452,7 +2458,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( const float limit = hparams.swiglu_clamp_exp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) { + if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4 || arch == LLM_ARCH_HY_V4_FLASH) { cur = ggml_swiglu_clamp(ctx0, cur, up, limit); } else { up = ggml_clamp(ctx0, up, -limit, limit); @@ -3043,7 +3049,10 @@ ggml_tensor * llm_graph_context::build_attn_mha( cur = ggml_mul_mat(ctx0, v_mla, cur); cb(cur, "fattn_mla", il); cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); - cur = ggml_cont(ctx0, cur); // Needed because ggml_reshape_2d expects contiguous inputs. + // ggml_reshape_2d needs contiguous data; a single token is already laid out that way after the permute + if (!ggml_is_contiguous(cur)) { + cur = ggml_cont(ctx0, cur); + } #endif } @@ -3668,7 +3677,17 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * k = mctx_cur->get_k(ctx0, il); ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0); - ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il); + // A standard sliding window shows each query at most n_swa cells, so a short decode (one token, or the block a + // speculative draft proposes) can attend over the compacted list of the cells it sees instead of scanning the + // whole SWA cache (Metal: the hy4 MLA decode kernel for one token, the per-row index lists above it). + // The bound assumes causal masking and one cell per position; the extra n_tokens - 1 entries cover a batch whose + // positions repeat, which a draft block does not produce. + const int64_t n_tok_max = 8; // the largest DFly block + const int64_t n_kv_max = is_swa && hparams.swa_type == LLAMA_SWA_TYPE_STANDARD && !kv_rows && + q_cur->ne[2] <= n_tok_max && hparams.n_swa > 0 && + hparams.n_swa + q_cur->ne[2] - 1 < (int64_t) k->ne[2] ? hparams.n_swa + q_cur->ne[2] - 1 : 0; + + ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, n_kv_max, kq_scale, il); cb(cur, "kqv_out", il); if (k_rot) { @@ -3868,7 +3887,13 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { const auto * mctx_cur = static_cast<const llama_kv_cache_iswa_context *>(mctx); - auto inp = std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur); + return build_attn_inp_k_iswa( + std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur), mctx_cur); +} + +llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa( + std::unique_ptr<llm_graph_input_attn_k_iswa> inp, + const llama_kv_cache_iswa_context * mctx_cur) const { { inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); diff --git a/src/llama-graph.h b/src/llama-graph.h index 37b47acbb..688fda7a0 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -562,12 +562,17 @@ public: cparams(cparams), mctx(mctx) { } - ~llm_graph_input_attn_k_iswa() = default; + virtual ~llm_graph_input_attn_k_iswa() = default; void set_input(const llama_ubatch * ubatch) override; bool can_reuse(const llm_graph_params & params) override; + // Where this input finds its context in the next batch's params. The default is that the + // memory is the iSWA cache itself; an arch whose memory merely composes one overrides this, + // because params.mctx is then that outer context and the cast would be wrong. + virtual const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const; + ggml_tensor * get_k_idxs() const { return self_k_idxs; } ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; } @@ -1370,6 +1375,12 @@ struct llm_graph_context { llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const; + // same, for a memory that composes an iSWA pair rather than being one: the arch supplies the + // input, so it can override mctx_of(), and the attention context out of its own + llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa( + std::unique_ptr<llm_graph_input_attn_k_iswa> inp, + const llama_kv_cache_iswa_context * mctx_cur) const; + // note: if k_cur is not provided, it will not be stored in the memory // note: the K cache is used as V (MLA-style attention) ggml_tensor * build_attn( diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4d2cb8931..19748815b 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -10,7 +10,10 @@ // bump if necessary #define LLAMA_MAX_LAYERS 512 #define LLAMA_MAX_EXPERTS 1024 // Kimi K3 -#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp, deepseek41 (all hash modules) +// multipliers are stored flat as [module][ngram] and indexed at stride ple_ngram_size +// (deepseek41.cpp:247), so this bounds ple_n_modules*ple_ngram_size, not ngram_size alone. +// hy_v4_flash needs 4 modules x 3 = 12. +#define LLAMA_MAX_PLE_NGRAM 16 // qwen4exp, deepseek41, hy_v4_flash (all hash modules) #define LLAMA_MAX_PLE_HEADS 64 // qwen4exp, deepseek41 (all hash modules) enum llama_expert_gating_func_type { diff --git a/src/llama-memory-idx-iswa.cpp b/src/llama-memory-idx-iswa.cpp new file mode 100644 index 000000000..dae3a814a --- /dev/null +++ b/src/llama-memory-idx-iswa.cpp @@ -0,0 +1,989 @@ +#include "llama-memory-idx-iswa.h" + +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-io.h" +#include "llama-model.h" + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <type_traits> + +// +// llama_memory_idx_iswa +// + +llama_memory_idx_iswa::llama_memory_idx_iswa( + const llama_model & model, + bool no_alloc, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + const layer_filter_cb & filter, + const layer_filter_cb & filter_idx) : + hparams_idx(model.hparams), + mem_attn(new llama_kv_cache_iswa( + model, no_alloc, type_k, type_v, v_trans, offload, swa_full, unified, + kv_size, n_seq_max, n_ubatch, + nullptr, filter, nullptr, nullptr)), + mem_idx([&] { + // a model without an indexer uses llama_kv_cache_iswa directly, so there is no null case here + GGML_ASSERT(filter_idx != nullptr); + + // the indexer mirrors the base child alone, so a SWA layer has no cells of its own to index + for (uint32_t il = 0; il < model.hparams.n_layer_all; ++il) { + GGML_ASSERT(!(filter_idx(il) && model.hparams.is_swa(il)) && "an indexer layer cannot be a SWA layer"); + } + + // MQA with a single key head of indexer_head_size, as llama_kv_cache_dsa shapes its own + std::fill(hparams_idx.n_head_kv_arr.begin(), hparams_idx.n_head_kv_arr.end(), 1); + // a k-pool indexer caches its per-token rows and the pooled key side by side + // (glm5-next: key | gate | pooled, qwen4exp: key | pooled) + hparams_idx.n_embd_head_k_full = model.hparams.indexer_head_size * (model.hparams.indexer_kpool > 0 ? model.hparams.indexer_kpool_row : 1); + + // the cached indexer keys are raw, rotation happens after pooling at read time, so a + // K-shift must not rotate them while the stream copies in the same update still apply + hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE; + + // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors + hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size; + hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size; + + LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size); + + // the indexer mirrors the base child, so it takes its size and its (absent) SWA as well + return new llama_kv_cache( + model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified, + kv_size, n_seq_max, 0, LLAMA_SWA_TYPE_NONE, + nullptr, filter_idx, nullptr, nullptr, "idx_"); + }()) {} + +llama_memory_context_ptr llama_memory_idx_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { + // note: repeats llama_kv_cache_iswa::init_batch, as the indexer needs the base slot infos that its context hides + do { + balloc.split_reset(); + + std::vector<llama_ubatch> ubatches; + + while (true) { + llama_ubatch ubatch; + + if (embd_all) { + // if all tokens are output, split by sequence + ubatch = balloc.split_seq(n_ubatch); + } else { + // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) + const bool unified = (mem_attn->get_base()->get_n_stream() == 1); + + ubatch = balloc.split_equal(n_ubatch, !unified, 0); + } + + if (ubatch.n_tokens == 0) { + break; + } + + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + // failed to find a suitable split + break; + } + + auto sinfos_base = mem_attn->get_base()->prepare(ubatches); + if (sinfos_base.empty()) { + LLAMA_LOG_ERROR("%s: failed to prepare non-SWA attention ubatches\n", __func__); + break; + } + + auto sinfos_swa = mem_attn->get_swa()->prepare(ubatches); + if (sinfos_swa.empty()) { + LLAMA_LOG_ERROR("%s: failed to prepare SWA attention ubatches\n", __func__); + break; + } + + assert(sinfos_base.size() == sinfos_swa.size()); + + // the indexer uses the base cache's slot layout; a separate one can drift from it + llama_kv_cache::slot_info_vec_t sinfos_idx = sinfos_base; + + return std::make_unique<llama_memory_idx_iswa_context>( + this, std::move(sinfos_base), std::move(sinfos_swa), std::move(sinfos_idx), std::move(ubatches)); + } while(false); + + return std::make_unique<llama_memory_idx_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE); +} + +llama_memory_context_ptr llama_memory_idx_iswa::init_full() { + return std::make_unique<llama_memory_idx_iswa_context>(this); +} + +llama_memory_context_ptr llama_memory_idx_iswa::init_update(llama_context * lctx, bool optimize) { + return std::make_unique<llama_memory_idx_iswa_context>(this, lctx, optimize); +} + +bool llama_memory_idx_iswa::get_can_shift() const { + // the indexer holds raw keys, so a shift never has to rotate them: only attention can refuse one + return mem_attn->get_can_shift(); +} + +void llama_memory_idx_iswa::clear(bool data) { + mem_attn->clear(data); + mem_idx ->clear(data); + + mem_idx_stale_set(-1, 0); +} + +// A pooled key is only valid while the grouping that produced it holds. Grouping is sequence relative, +// so an edit at p0 leaves every pool that ends before p0 alone. +void llama_memory_idx_iswa::mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0) { + p0 = std::max<llama_pos>(p0, 0); + + if (seq_id < 0) { + for (auto & p : mem_idx_stale) { + p = std::min(p, p0); + } + + return; + } + + GGML_ASSERT(seq_id < (llama_seq_id) LLAMA_MAX_SEQ); + + mem_idx_stale[seq_id] = std::min(mem_idx_stale[seq_id], p0); +} + +// An edit at or below the first position moves pos_min, which regroups the whole sequence. +llama_pos llama_memory_idx_iswa::mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const { + if (seq_id < 0 || p0 <= mem_idx->seq_pos_min(seq_id)) { + return 0; + } + + return p0; +} + +bool llama_memory_idx_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + // the stale position is relative to the first position the indexer still holds, so read it before the removal + const llama_pos stale = mem_idx_stale_pos(seq_id, p0); + + bool res = true; + + res = res & mem_attn->seq_rm(seq_id, p0, p1); + res = res & mem_idx ->seq_rm(seq_id, p0, p1); + + mem_idx_stale_set(seq_id, stale); + + return res; +} + +void llama_memory_idx_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + // only whole sequences are copied: a shared cell holds a single pool grouping + GGML_ASSERT(p0 <= 0 && p1 < 0 && "partial seq_cp is not supported"); + + mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1); + mem_idx ->seq_cp(seq_id_src, seq_id_dst, p0, p1); + + // a whole sequence copy gives the destination the source's pools, rep rows included: the source keeps its + // pooled keys, the destination rebuilds its layout and re-pools into the same rows + mem_idx_stale_set(seq_id_dst, 0); +} + +void llama_memory_idx_iswa::seq_keep(llama_seq_id seq_id) { + mem_attn->seq_keep(seq_id); + mem_idx ->seq_keep(seq_id); + + // every other sequence loses its cells, so their layouts must rebuild + mem_idx_stale_set(-1, 0); +} + +void llama_memory_idx_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + // a negative shift moves the cells below p0, so they regroup as well + const llama_pos stale = mem_idx_stale_pos(seq_id, shift < 0 ? p0 + shift : p0); + + mem_attn->seq_add(seq_id, p0, p1, shift); + mem_idx ->seq_add(seq_id, p0, p1, shift); + + mem_idx_stale_set(seq_id, stale); +} + +void llama_memory_idx_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + mem_attn->seq_div(seq_id, p0, p1, d); + mem_idx ->seq_div(seq_id, p0, p1, d); + + mem_idx_stale_set(seq_id, 0); +} + +llama_pos llama_memory_idx_iswa::seq_pos_min(llama_seq_id seq_id) const { + return mem_attn->seq_pos_min(seq_id); +} + +llama_pos llama_memory_idx_iswa::seq_pos_max(llama_seq_id seq_id) const { + return mem_attn->seq_pos_max(seq_id); +} + +std::map<ggml_backend_buffer_type_t, size_t> llama_memory_idx_iswa::memory_breakdown() const { + std::map<ggml_backend_buffer_type_t, size_t> mb = mem_attn->memory_breakdown(); + + for (const auto & buft_size : mem_idx->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + + return mb; +} + +void llama_memory_idx_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + mem_attn->state_write(io, seq_id, flags); + + // the indexer section goes last, so it is a pure suffix: an iSWA-only reader stops early instead of misparsing it + mem_idx->state_write(io, seq_id, flags); +} + +void llama_memory_idx_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + // note: repeats llama_kv_cache_iswa::state_read + // the indexer needs the base cache's cells, and a half-failed restore must leave both caches alike + + // the indexer restore adopts the base cache's layout instead of searching for cells of its own + // two find_slot calls agree only while both caches see the same occupancy, which a restore cannot promise + llama_kv_cache::slot_info_vec_t sinfos_base; + + try { + mem_attn->get_base()->state_read_sinfo(io, seq_id, flags, &sinfos_base, nullptr); + mem_attn->get_swa ()->state_read(io, seq_id, flags); + + // must mirror the write order in state_write + // the indexer has the SWA type of the base child, so both read the same partial state + mem_idx->state_read_sinfo(io, seq_id, flags, nullptr, &sinfos_base); + + // a restore that rewrites cells does it behind the pool layout's back + if (!sinfos_base.empty()) { + mem_idx_stale_set(seq_id, 0); + } + } catch (...) { + // a half-restored context is the one state the indexer cannot fix by itself: attention holds new cells, the indexer old ones + // drop what was being restored from both of them, which is a state they do agree on. + state_drop(seq_id); + + throw; + } +} + +void llama_memory_idx_iswa::state_drop(llama_seq_id seq_id) { + if (seq_id < 0) { + clear(true); + + return; + } + + mem_attn->get_base()->state_clear(seq_id); + mem_attn->get_swa ()->state_clear(seq_id); + + mem_idx->state_clear(seq_id); + mem_idx_stale_set(seq_id, 0); +} + +llama_kv_cache_iswa * llama_memory_idx_iswa::get_mem_attn() const { + return mem_attn.get(); +} + +llama_kv_cache * llama_memory_idx_iswa::get_mem_idx() const { + return mem_idx.get(); +} + +// +// llama_memory_idx_iswa_context +// + +// streams in each ubatch's slot info, matching get_k/get_v's `ns` +static std::vector<uint32_t> llama_memory_idx_iswa_ns(const llama_kv_cache::slot_info_vec_t & sinfos) { + std::vector<uint32_t> res; + res.reserve(sinfos.size()); + + for (const auto & sinfo : sinfos) { + res.push_back(sinfo.s1 - sinfo.s0 + 1); + } + + return res; +} + +// Which cells of a sequence make up which pool, for the whole cache. +struct llama_memory_idx_iswa::kpool_layout { + struct seq { + llama_pos pos_min = 0; + uint32_t strm = 0; // Stream holding this sequence's cells + std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position. + std::vector<uint32_t> pools; + + // Where the pool scan stopped, so an append resumes instead of starting over. + size_t j_next = 0; + }; + + std::array<seq, LLAMA_MAX_SEQ> seqs; + + uint32_t n_pool_real = 0; +}; + +// Which pools of the layout the current ubatch must re-pool, in the layout's pool order. +struct llama_memory_idx_iswa_context::kpool_state { + std::vector<uint32_t> is_new; + std::vector<uint32_t> rep_gen; // per global cell, the generation that last marked a pool with that rep + uint32_t generation = 0; + + uint32_t n_pool_real = 0; + uint32_t n_new = 0; + uint32_t n_new_g = 1; // graph size of the new pool list, stable across decode steps +}; + +namespace { + +// The last padded pool is always unused. +uint32_t kpool_pad(uint32_t n_pool) { + return std::max<uint32_t>(64u, GGML_PAD(n_pool + 1, 64u)); +} + +// Rank of (pos, cell) in a sequence's cells sorted by position then cell, or -1 when absent. +// In order mode the rank alone places a token: cells sharing a position (M-RoPE images) have distinct ranks. +int64_t kpool_rank(const std::vector<std::pair<llama_pos, uint32_t>> & cells, llama_pos pos, uint32_t cell) { + auto it = std::lower_bound(cells.begin(), cells.end(), std::make_pair(pos, cell)); + return it != cells.end() && it->second == cell && it->first == pos ? it - cells.begin() : -1; +} + +} + +llama_memory_idx_iswa::~llama_memory_idx_iswa() = default; + +const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_get() const { + GGML_ASSERT(kpool_lay != nullptr); + + return *kpool_lay; +} + +// Pools are fixed by the positions relative to the sequence's first one, so the layout survives a plain +// append. A sequence edit can regroup them, and mem_idx_stale tells us it happened. +const llama_memory_idx_iswa::kpool_layout & llama_memory_idx_iswa::kpool_layout_update() { + if (!kpool_lay) { + kpool_lay = std::make_unique<kpool_layout>(); + } + + auto & lay = *kpool_lay; + + const uint32_t kpool = get_kpool(); + const uint32_t n_stream_kv = mem_idx->get_n_stream(); + const bool unified = n_stream_kv == 1; + + lay.n_pool_real = 0; + + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + auto & sq = lay.seqs[s]; + + // a non unified cache gives each sequence its own stream, with stream local cell indices + if (!unified && s >= (llama_seq_id) n_stream_kv) { + sq = kpool_layout::seq(); + continue; + } + + const auto & cells = mem_idx->get_cells(unified ? 0 : s); + const auto & sp = cells.seq_pos_get(s); + + sq.strm = unified ? 0 : mem_idx->get_stream(s); + + if (mem_idx_stale[s] == POS_CLEAN && !sq.cells.empty() && !sp.empty() && + sq.pos_min == sp.begin()->first) { + for (auto it = sp.upper_bound(sq.cells.back()); it != sp.end(); ++it) { + sq.cells.push_back(*it); + } + } + + // the appended tail accounts for every cell only if nothing before it was dropped, but an edit can + // regroup a sequence without changing its cell count, so a stale sequence must rebuild regardless + if (sq.cells.size() != sp.size() || mem_idx_stale[s] != POS_CLEAN) { + sq.cells.assign(sp.begin(), sp.end()); + sq.pools.clear(); + sq.j_next = 0; + sq.pos_min = sp.empty() ? 0 : sp.begin()->first; + } + + // Pools start at the first valid token + size_t j = sq.j_next; + if (hparams_idx.indexer_kpool_by_order) { + // consecutive cells in sequence order, whatever their positions + for (; j + kpool <= sq.cells.size(); j += kpool) { + sq.pools.push_back((uint32_t) j); + } + } else { + while (j + kpool <= sq.cells.size()) { + const llama_pos p0 = sq.cells[j].first; + if ((p0 - sq.pos_min) % (llama_pos) kpool != 0) { + ++j; + continue; + } + bool ok = true; + for (uint32_t k = 1; k < kpool; ++k) { + if (sq.cells[j + k].first != p0 + (llama_pos) k) { + ok = false; + break; + } + } + if (ok) { + sq.pools.push_back((uint32_t) j); + j += kpool; + } else { + ++j; + } + } + } + sq.j_next = j; + + lay.n_pool_real += (uint32_t) sq.pools.size(); + } + + return lay; +} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_status status) : status(status) {} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem) : + mem(mem), + // graph reservation walks a full context, and the sparse attention is built only when this is set + // without it the reserved worst case is the dense graph, so ggml-alloc must grow the buffer on the first decode + ns_ubatch(std::vector<uint32_t>{ mem->get_mem_idx()->get_n_stream() }), + ctx_attn(mem->get_mem_attn()->init_full()), + ctx_idx (mem->get_mem_idx ()->init_full()), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) { + if (kpool_track()) { + mem->kpool_layout_update(); + auto st = kpool_build_sizes(); + const auto * idx = mem->get_mem_idx(); + const uint64_t n_pool_max = uint64_t(idx->get_size() / mem->get_kpool()) * idx->get_n_seq_max(); + GGML_ASSERT(n_pool_max <= UINT32_MAX - 64); + st.n_pool_real = std::max(st.n_pool_real, uint32_t(n_pool_max)); + st.n_new = st.n_pool_real; + st.n_new_g = std::max(st.n_new, 1u); + kpool_st = std::make_unique<kpool_state>(std::move(st)); + i_kpool = 0; + } +} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + llama_context * lctx, + bool optimize) : + mem(mem), + ctx_attn(mem->get_mem_attn()->init_update(lctx, optimize)), + // update() applies a pending cross-stream seq_cp, else the copy keeps stale indexer keys + ctx_idx (mem->get_mem_idx ()->init_update(lctx, optimize)), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) {} + +llama_memory_idx_iswa_context::llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_swa, + slot_info_vec_t sinfos_idx, + std::vector<llama_ubatch> ubatches) : + mem(mem), + ubatches(std::move(ubatches)), + ns_ubatch(llama_memory_idx_iswa_ns(sinfos_idx)), + sinfos_kpool(mem->get_kpool() > 0 && mem->get_kpool_by_order() ? sinfos_idx : slot_info_vec_t()), + // note: the children copy the ubatches; ctx_idx gets a copy of its own + ctx_attn(new llama_kv_cache_iswa_context(mem->get_mem_attn(), std::move(sinfos_base), std::move(sinfos_swa), this->ubatches)), + ctx_idx (new llama_kv_cache_context(mem->get_mem_idx(), std::move(sinfos_idx), this->ubatches)), + status(llama_memory_status_combine(ctx_attn->get_status(), ctx_idx->get_status())) { + // Sequence edits force the touched positions to re-pool. + mem_idx_stale_batch = mem->mem_idx_stale_get(); +} + +llama_memory_idx_iswa_context::~llama_memory_idx_iswa_context() = default; + +bool llama_memory_idx_iswa_context::next() { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + // Clear only after a successful ubatch. + if (i_next == 0 && mem != nullptr) { + mem->mem_idx_stale_clear(); + } + + ctx_attn->next(); + ctx_idx ->next(); + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_memory_idx_iswa_context::apply() { + assert(!llama_memory_status_is_fail(status)); + + bool res = true; + + res = res & ctx_attn->apply(); + res = res & ctx_idx ->apply(); + + // Extend the pool layout with this ubatch's cells, then pick what it must re-pool. + if (res && kpool_track()) { + mem->kpool_layout_update(); + if (!kpool_st) { + kpool_st = std::make_unique<kpool_state>(); + } + kpool_build_state(get_ubatch()); + i_kpool = i_next; + } + + return res; +} + +llama_memory_status llama_memory_idx_iswa_context::get_status() const { + return status; +} + +const llama_ubatch & llama_memory_idx_iswa_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +bool llama_memory_idx_iswa_context::kpool_track() const { + // Derived from mem instead of being cached. + return mem != nullptr && mem->get_kpool() > 0 && !ns_ubatch.empty(); +} + +const llama_kv_cache_iswa_context * llama_memory_idx_iswa_context::get_attn() const { + return static_cast<const llama_kv_cache_iswa_context *>(ctx_attn.get()); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_base() const { + return get_attn()->get_base(); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_swa() const { + return get_attn()->get_swa(); +} + +const llama_kv_cache_context * llama_memory_idx_iswa_context::get_idx() const { + return static_cast<const llama_kv_cache_context *>(ctx_idx.get()); +} + +uint32_t llama_memory_idx_iswa_context::get_n_stream() const { + GGML_ASSERT(i_next < ns_ubatch.size()); + + return ns_ubatch[i_next]; +} + +llama_memory_idx_iswa_context::kpool_access::kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd) : ctx(ctx) { + // rows are the per-token part (glm5-next: key | gate, qwen4exp: key), then the pooled key + const int64_t n_tok = k->ne[0] - n_embd; + GGML_ASSERT(n_tok > 0 && n_tok % n_embd == 0); + + const int64_t n_cells = k->ne[1]*k->ne[2]; + + // Pool indices can refer to other streams. Revisit these full-storage views if that changes: + // https://github.com/ggml-org/llama.cpp/pull/27773#discussion_r4130905603 + key_gate = ggml_view_2d(ctx, k, n_tok, n_cells, k->nb[1], 0); + pooled = ggml_view_2d(ctx, k, n_embd, n_cells, k->nb[1], ggml_row_size(k->type, n_tok)); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_key_gate(ggml_tensor * idxs) const { + return ggml_get_rows(ctx, key_gate, idxs); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const { + return ggml_set_rows(ctx, pooled, values, idxs); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::gather_pooled(ggml_tensor * idxs) const { + return ggml_get_rows(ctx, pooled, idxs); +} + +ggml_tensor * llama_memory_idx_iswa_context::kpool_access::pooled_rows() const { + return ggml_view_4d(ctx, pooled, pooled->ne[0], 1, pooled->ne[1], 1, + pooled->nb[1], pooled->nb[1], pooled->nb[1]*pooled->ne[1], 0); +} + +llama_memory_idx_iswa_context::kpool_access llama_memory_idx_iswa_context::get_kpool_access( + ggml_context * ctx, int32_t il, int64_t n_embd) const { + GGML_ASSERT(mem != nullptr); + + return kpool_access(ctx, mem->get_mem_idx()->get_k_storage(il), n_embd); +} + +// k-pool DSA indexer (hy_v4_flash, glm5-next, qwen4exp QSA) + +// Sizes only, used by the full cache context so get_n_kpool() works during graph reserve. +llama_memory_idx_iswa_context::kpool_state llama_memory_idx_iswa_context::kpool_build_sizes() const { + const auto & lay = mem->kpool_layout_get(); + + kpool_state st; + st.n_pool_real = lay.n_pool_real; + + return st; +} + +// Which pools this ubatch must re-pool. +// Pool cache lifecycle: +// 1. cpy_k writes each token's key | gate into its idx cache row, pooled slot are zeroed. +// 2. This marks the pools the ubatch touches or completes as new, during decode that's one pool every kpool tokens, zero elsewise. +// 3. The graph pools only the new pools and set_rows each result into the pooled slot of the pool's last member row. +// 4. All pools are gathered in one get_rows via pool_cells, fresh ones just written, older ones from whatever batch last wrote them. +// A seq_* edit regroups the pools from the edited position on, so it stales them and the first ubatch of the next batch +// rebuilds them from the still-valid key | gate rows, rewriting the (possibly different) rep rows. +// Orphaned pooled slots are never cleared, a slot is only ever read through pool_cells, which follows the current grouping. +void llama_memory_idx_iswa_context::kpool_build_state(const llama_ubatch & ubatch) { + const auto & lay = mem->kpool_layout_get(); + auto & st = *kpool_st; + + const auto * idx = mem->get_mem_idx(); + const uint32_t kv_size = idx->get_size(); + const uint32_t kpool = mem->get_kpool(); + + st.n_pool_real = lay.n_pool_real; + st.n_new = 0; + if (++st.generation == 0) { + std::fill(st.is_new.begin(), st.is_new.end(), 0); + std::fill(st.rep_gen.begin(), st.rep_gen.end(), 0); + st.generation = 1; + } + st.is_new.resize(lay.n_pool_real, 0); + st.rep_gen.resize((size_t) kv_size*idx->get_n_stream(), 0); + + std::array<uint32_t, LLAMA_MAX_SEQ> pool_start; + + // a pool is marked once per rep: sequences sharing cells (a seq_cp, or tokens decoded for several sequences) + // share their pools, whose single pooled row they all read through pool_cells, so the scatter rows stay unique + auto mark = [&](llama_seq_id s, size_t k) { + const auto & sq = lay.seqs[s]; + const size_t rep = (size_t) sq.strm*kv_size + sq.cells[sq.pools[k] + kpool - 1].second; + if (st.rep_gen[rep] != st.generation) { + st.rep_gen[rep] = st.generation; + st.is_new[pool_start[s] + k] = st.generation; + ++st.n_new; + } + }; + + uint32_t ip = 0; + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + const auto & sq = lay.seqs[s]; + pool_start[s] = ip; + ip += (uint32_t) sq.pools.size(); + + // A sequence edit invalidates only pools ending after the edited position. + const llama_pos stale_from = i_next == 0 ? + mem_idx_stale_batch[s] : llama_memory_idx_iswa::POS_CLEAN; + if (stale_from == llama_memory_idx_iswa::POS_CLEAN) { + continue; + } + + auto first = std::lower_bound(sq.pools.begin(), sq.pools.end(), stale_from, + [&](uint32_t j, llama_pos p) { return sq.cells[j + kpool - 1].first < p; }); + for (auto it = first; it != sq.pools.end(); ++it) { + mark(s, it - sq.pools.begin()); + } + } + GGML_ASSERT(ip == st.is_new.size()); + + // in order mode a token's cell gives its rank, and the rank its pool: positions cannot, as an image shares one + const bool by_order = mem->get_kpool_by_order(); + const auto * sinfo = by_order ? &sinfos_kpool[i_next] : nullptr; + const uint32_t n_tps = by_order ? (uint32_t) sinfo->size() : 0; + + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + const llama_pos p = ubatch.pos[i]; + for (int32_t k = 0; k < ubatch.n_seq_id[i]; ++k) { + const llama_seq_id s = ubatch.seq_id[i][k]; + const auto & sq = lay.seqs[s]; + if (by_order) { + const int64_t r = kpool_rank(sq.cells, p, sinfo->idxs[i / n_tps][i % n_tps]); + GGML_ASSERT(r >= 0); + if ((size_t) r / kpool < sq.pools.size()) { + mark(s, (size_t) r / kpool); + } + continue; + } + auto it = std::upper_bound(sq.pools.begin(), sq.pools.end(), p, + [&](llama_pos pos, uint32_t j) { return pos < sq.cells[j].first; }); + if (it == sq.pools.begin()) { + continue; + } + --it; + if (p <= sq.cells[*it + kpool - 1].first) { + mark(s, it - sq.pools.begin()); + } + } + } + + // a ubatch touches at most t_s/kpool + 1 pools per sequence, pad to that bound so the graph keeps its shape + // as the count moves; reserve sizes the list for every pool the cache can hold, so never pad past n_pool_max + const uint32_t n_pool_max = kv_size / kpool * idx->get_n_seq_max(); + const uint32_t bound = ubatch.n_tokens/kpool + ubatch.n_seqs_unq; + st.n_new_g = std::max({st.n_new, 1u, std::min({bound, kpool_pad(st.n_pool_real) - 1, n_pool_max})}); +} + +const llama_memory_idx_iswa_context::kpool_state & llama_memory_idx_iswa_context::kpool_cur() const { + GGML_ASSERT(kpool_st != nullptr && i_kpool == i_next && "k-pool state read before apply()"); + + return *kpool_st; +} + +uint32_t llama_memory_idx_iswa_context::get_n_kpool() const { + return kpool_pad(kpool_cur().n_pool_real); +} + +uint32_t llama_memory_idx_iswa_context::get_n_kpool_new() const { + return kpool_cur().n_new_g; +} + +void llama_memory_idx_iswa_context::set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs, + ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep, + const llama_ubatch * ubatch, ggml_tensor * new_pool_pos) const { + GGML_ASSERT(mem != nullptr); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_cells->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_idxs->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(pool_mask->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(tail_idxs->buffer)); + + const uint32_t kpool = mem->get_kpool(); + const uint32_t n_kv = get_idx()->get_n_kv(); + + const auto & st = kpool_cur(); + const auto & lay = mem->kpool_layout_get(); + + const uint32_t n_tokens = ubatch->n_tokens; + const uint32_t n_pool = (uint32_t) pool_cells->ne[0]; + const uint32_t n_new = st.n_new; + // the graph always pools at least one entry, padded to a stable bound, see kpool_build_state + const uint32_t n_new_g = st.n_new_g; + + const bool by_order = mem->get_kpool_by_order(); + + const uint32_t n_tail_slots = (uint32_t) tail_idxs->ne[0]; + const bool tail_full = n_tail_slots == kpool; + + GGML_ASSERT(n_pool == kpool_pad(st.n_pool_real)); + GGML_ASSERT(st.is_new.size() == st.n_pool_real); + GGML_ASSERT(pool_mask->ne[0] == (int64_t) n_pool && pool_mask->ne[1] == (int64_t) n_tokens); + GGML_ASSERT((n_tail_slots == kpool - 1 || tail_full) && tail_idxs->ne[1] == (int64_t) n_tokens); + GGML_ASSERT(pool_idxs->ne[0] == (int64_t) kpool && pool_idxs->ne[1] == (int64_t) n_pool); + GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_idxs->buffer)); + GGML_ASSERT(ggml_nelements(new_pool_idxs) == (int64_t) kpool*n_new_g); + // the graph always scatters the fresh pooled keys back into the cache + GGML_ASSERT(new_pool_rep != nullptr && ggml_backend_buffer_is_host(new_pool_rep->buffer)); + GGML_ASSERT(new_pool_rep->ne[0] == (int64_t) n_new_g); + if (new_pool_pos != nullptr) { + GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_pos->buffer)); + GGML_ASSERT(new_pool_pos->ne[0] == 4*(int64_t) n_new_g); + } + + const uint32_t kv_size = mem->get_mem_idx()->get_size(); + const uint32_t n_stream_kv = mem->get_mem_idx()->get_n_stream(); + + auto gcell = [&](const llama_memory_idx_iswa::kpool_layout::seq & sq, uint32_t cell) { + return (int64_t) sq.strm*kv_size + cell; + }; + + // Sequences present in this ubatch, pools of absent sequences must fall on the scatter sentinel row. + std::vector<uint8_t> seq_in_ub(LLAMA_MAX_SEQ, 0); + for (uint32_t i = 0; i < n_tokens; ++i) { + for (int32_t k = 0; k < ubatch->n_seq_id[i]; ++k) { + seq_in_ub[ubatch->seq_id[i][k]] = 1; + } + } + + // a cell of this ubatch, written before any read, so the padded pools read a finite K row + int64_t dummy_cell = 0; + { + const llama_seq_id s = ubatch->seq_id[0][0]; + const auto & sq = lay.seqs[s]; + auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(ubatch->pos[0], 0u)); + GGML_ASSERT(it != sq.cells.end() && it->first == ubatch->pos[0]); + dummy_cell = gcell(sq, it->second); + } + + // in order mode a token sees the pools and the tail up to its own rank in the sequence, which its cell pins down + std::vector<int64_t> rank; + if (by_order) { + const auto & sinfo = sinfos_kpool[i_next]; + const uint32_t n_tps = (uint32_t) sinfo.size(); + + rank.resize(n_tokens); + for (uint32_t i = 0; i < n_tokens; ++i) { + rank[i] = kpool_rank(lay.seqs[ubatch->seq_id[i][0]].cells, ubatch->pos[i], sinfo.idxs[i / n_tps][i % n_tps]); + GGML_ASSERT(rank[i] >= 0); + } + } + + // padding and absent cells point at the n_kv sentinel row, one past the live cells + const int32_t sentinel = (int32_t) n_kv; + + float * gm = nullptr; + uint32_t n_sel = 0; + uint32_t n_top = 0; // Pools per token in the selection. + uint32_t n_tail_sel = 0; + if (sel_mask != nullptr) { + GGML_ASSERT(ggml_backend_buffer_is_host(sel_mask->buffer)); + GGML_ASSERT(sel_mask->type == GGML_TYPE_F32); + GGML_ASSERT(sel_mask->ne[3] == (int64_t) n_tokens && sel_mask->ne[1] == 1 && sel_mask->ne[2] == 1); + n_sel = (uint32_t) sel_mask->ne[0]; + n_tail_sel = tail_full ? kpool : n_sel % kpool; + GGML_ASSERT(n_sel >= n_tail_sel && (n_sel - n_tail_sel) % kpool == 0); + GGML_ASSERT(n_tail_sel == 0 || n_tail_sel == n_tail_slots); + n_top = (n_sel - n_tail_sel) / kpool; + gm = (float *) sel_mask->data; + } + + // pools are laid out per sequence + std::vector<uint32_t> seq_pool_start(LLAMA_MAX_SEQ, 0); + std::vector<llama_pos> pool_end; + pool_end.reserve(n_pool); + + int32_t * pcell = (int32_t *) pool_cells->data; + int32_t * pidx = (int32_t *) pool_idxs->data; + int32_t * nidx = (int32_t *) new_pool_idxs->data; + int64_t * nrep = (int64_t *) new_pool_rep->data; + int32_t * npos = new_pool_pos != nullptr ? (int32_t *) new_pool_pos->data : nullptr; + + if (npos != nullptr) { + std::fill(npos, npos + 4*n_new_g, 0); + } + + uint32_t i_new = 0; + for (llama_seq_id s = 0; s < LLAMA_MAX_SEQ; ++s) { + const auto & sq = lay.seqs[s]; + seq_pool_start[s] = (uint32_t) pool_end.size(); + + const bool inert = n_stream_kv > 1 && !seq_in_ub[s]; + + for (size_t pi = 0; pi < sq.pools.size(); ++pi) { + const uint32_t j = sq.pools[pi]; + const uint32_t ip = (uint32_t) pool_end.size(); + GGML_ASSERT(ip + 1 < n_pool); + + // The pooled key lives in the last member's row. + const uint32_t rep = sq.cells[j + kpool - 1].second; + pcell[ip] = (int32_t) gcell(sq, rep); + + for (uint32_t k = 0; k < kpool; ++k) { + pidx[(size_t) ip*kpool + k] = inert ? sentinel : (int32_t) sq.cells[j + k].second; + } + + if (st.is_new[ip] == st.generation) { + GGML_ASSERT(i_new < n_new); + for (uint32_t k = 0; k < kpool; ++k) { + nidx[(size_t) i_new*kpool + k] = (int32_t) gcell(sq, sq.cells[j + k].second); + } + nrep[i_new] = gcell(sq, rep); + if (npos != nullptr) { + // a pooled key is rotated to the M-RoPE position of its first member + const uint32_t c = sq.cells[j].second; + const auto & e = mem->get_mem_idx()->get_cells(s).ext_get(c); + npos[0*n_new_g + i_new] = sq.cells[j].first; + npos[1*n_new_g + i_new] = e.y; + npos[2*n_new_g + i_new] = e.x; + npos[3*n_new_g + i_new] = sq.cells[j].first; + } + ++i_new; + } + + pool_end.push_back(sq.cells[j + kpool - 1].first); + } + } + GGML_ASSERT(i_new == n_new); + + // Padded entries re-pool cells whose pooled slot is never read: only the reps of complete pools are read. + // Each entry takes its own cell, entries sharing one would write it from several threads in the scatter. + if (n_new_g > n_new) { + std::vector<int64_t> reps(pcell, pcell + pool_end.size()); + std::sort(reps.begin(), reps.end()); + + int64_t pad_cell = 0; + for (uint32_t i = n_new; i < n_new_g; ++i, ++pad_cell) { + while (std::binary_search(reps.begin(), reps.end(), pad_cell)) { + ++pad_cell; + } + GGML_ASSERT(pad_cell < (int64_t) kv_size*n_stream_kv); + for (uint32_t k = 0; k < kpool; ++k) { + nidx[(size_t) i*kpool + k] = (int32_t) pad_cell; + } + nrep[i] = pad_cell; + } + } + + const uint32_t n_pool_real = (uint32_t) pool_end.size(); + for (uint32_t ip = n_pool_real; ip < n_pool; ++ip) { + pcell[ip] = (int32_t) dummy_cell; // pool_cells always addresses the K storage + for (uint32_t k = 0; k < kpool; ++k) { + pidx[(size_t) ip*kpool + k] = sentinel; + } + } + + // a pool is visible when it belongs to the token's sequence and ends at or before it, unless reserved by a full-width tail + auto fill_mask = [&](auto * data) { + using T = std::remove_pointer_t<decltype(data)>; + const T keep = llama_cast<T>(0.0f); + const T drop = llama_cast<T>(-INFINITY); + + for (uint32_t i = 0; i < n_tokens; ++i) { + const llama_seq_id s = ubatch->seq_id[i][0]; + const llama_pos p = ubatch->pos[i]; + + T * row = data + (size_t) i*n_pool; + std::fill(row, row + n_pool, drop); + + const uint32_t p0 = seq_pool_start[s]; + const uint32_t p1 = p0 + (uint32_t) lay.seqs[s].pools.size(); + const uint32_t nv = by_order ? std::min(p1 - p0, (uint32_t) ((rank[i] + !tail_full)/kpool)) : + (uint32_t) ((tail_full ? std::lower_bound(pool_end.begin() + p0, pool_end.begin() + p1, p) : + std::upper_bound(pool_end.begin() + p0, pool_end.begin() + p1, p)) - (pool_end.begin() + p0)); + std::fill(row + p0, row + p0 + nv, keep); + + // Finite visible pools occupy the first min(nv, n_top) ranked slots. + if (gm != nullptr) { + const uint32_t nvc = std::min(nv, n_top); + float * grow = gm + (size_t) i*n_sel; + std::fill(grow, grow + (size_t) nvc*kpool, 0.0f); + std::fill(grow + (size_t) nvc*kpool, grow + (size_t) n_top*kpool, -INFINITY); + } + } + }; + if (pool_mask->type == GGML_TYPE_F16) { + fill_mask((ggml_fp16_t *) pool_mask->data); + } else { + fill_mask((float *) pool_mask->data); + } + + int32_t * tidx = (int32_t *) tail_idxs->data; + for (uint32_t i = 0; i < n_tokens; ++i) { + const llama_seq_id s = ubatch->seq_id[i][0]; + const llama_pos p = ubatch->pos[i]; + const auto & sq = lay.seqs[s]; + + const int64_t pos = by_order ? rank[i] : p - sq.pos_min; + const uint32_t n_tail = tail_full ? (uint32_t) (pos % kpool) + 1 : (uint32_t) ((pos + 1) % kpool); + + for (uint32_t k = 0; k < n_tail_slots; ++k) { + int32_t cell = sentinel; + bool real = false; + if (k < n_tail && by_order) { + const uint32_t c = sq.cells[rank[i] - k].second; + cell = (int32_t) c; + real = true; + } else if (k < n_tail) { + const llama_pos pt = p - (llama_pos) k; + auto it = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(pt, 0u)); + if (it != sq.cells.end() && it->first == pt) { + cell = (int32_t) it->second; + real = true; + } + } + tidx[(size_t) i*n_tail_slots + k] = cell; + + if (gm != nullptr && n_tail_sel != 0) { + gm[(size_t) i*n_sel + (size_t) n_top*kpool + k] = real ? 0.0f : -INFINITY; + } + } + } +} diff --git a/src/llama-memory-idx-iswa.h b/src/llama-memory-idx-iswa.h new file mode 100644 index 000000000..8827773d7 --- /dev/null +++ b/src/llama-memory-idx-iswa.h @@ -0,0 +1,252 @@ +#pragma once + +#include "llama-kv-cache-iswa.h" + +#include <array> +#include <limits> +#include <memory> +#include <vector> + +// +// llama_memory_idx_iswa +// + +// llama_kv_cache_iswa plus a second cache with one indexer key per token, for block-sparse attention (hy_v4_flash) +// only the non-SWA layers own an indexer, so it mirrors the base child alone: same size, padding, streams and +// slots, so cell j is one token in both + +class llama_memory_idx_iswa : public llama_memory_i { +public: + llama_memory_idx_iswa( + const llama_model & model, + bool no_alloc, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + /* layer filters */ + const layer_filter_cb & filter, + /* the SWA layers never own an indexer, so this must select non-SWA layers only */ + const layer_filter_cb & filter_idx); + + // Defined out of line because kpool_layout is incomplete here. + ~llama_memory_idx_iswa(); + + // + // llama_memory_i + // + + llama_memory_context_ptr init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) override; + + llama_memory_context_ptr init_full() override; + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; + + void clear(bool data) override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + + std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override; + + // state write/load + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; + + // + // llama_memory_idx_iswa specific API + // + + llama_kv_cache_iswa * get_mem_attn() const; + llama_kv_cache * get_mem_idx () const; + + // The model's indexer pool size. + uint32_t get_kpool() const { return hparams_idx.indexer_kpool; } + + // Whether pools are kpool consecutive cells in sequence order (qwen4exp) instead of kpool consecutive positions. + bool get_kpool_by_order() const { return hparams_idx.indexer_kpool_by_order; } + + // Which cells of a sequence make up which pool of kpool consecutive positions (or cells, in order mode). + // It is kept here because it outlives the batch: pools are fixed by the positions relative to the + // sequence's first one, so a ubatch only ever appends to it. Sequence edits drop it, see mem_idx_stale. + struct kpool_layout; + + const kpool_layout & kpool_layout_update(); + const kpool_layout & kpool_layout_get() const; + + // The pooled keys persist in the idx cache across batches. A sequence edit can regroup the pools + // from some position on, which stales every pooled key at or after it. POS_CLEAN means none. + using stale_pos_t = std::array<llama_pos, LLAMA_MAX_SEQ>; + + static constexpr llama_pos POS_CLEAN = std::numeric_limits<llama_pos>::max(); + + static stale_pos_t stale_pos_clean() { + stale_pos_t res; + res.fill(POS_CLEAN); + return res; + } + + const stale_pos_t & mem_idx_stale_get() const { return mem_idx_stale; } + void mem_idx_stale_clear() { mem_idx_stale.fill(POS_CLEAN); } + +private: + // forget seq_id (all of it if seq_id < 0) in both caches at once, so a failed restore cannot leave them out of step + // seq_id < 0 drops the whole context, as the caches themselves do on a failed restore + void state_drop(llama_seq_id seq_id); + + // the indexer cache holds one key head per layer, so it needs its own hparams: + // llama_kv_cache keeps a reference to what it is given + llama_hparams hparams_idx; + + const std::unique_ptr<llama_kv_cache_iswa> mem_attn; + const std::unique_ptr<llama_kv_cache> mem_idx; + + // unique_ptr because kpool_layout is incomplete here + std::unique_ptr<kpool_layout> kpool_lay; + + // seq_id < 0 stales every sequence, p0 < 0 stales the sequence from its first position + void mem_idx_stale_set(llama_seq_id seq_id, llama_pos p0); + + // the position an edit at p0 stales the sequence from + llama_pos mem_idx_stale_pos(llama_seq_id seq_id, llama_pos p0) const; + + stale_pos_t mem_idx_stale = stale_pos_clean(); +}; + +class llama_memory_idx_iswa_context : public llama_memory_context_i { +public: + class kpool_access { + public: + ggml_tensor * gather_key_gate(ggml_tensor * idxs) const; + ggml_tensor * scatter_pooled(ggml_tensor * values, ggml_tensor * idxs) const; + ggml_tensor * gather_pooled(ggml_tensor * idxs) const; + + // the pooled key of every cell as [n_embd, 1, n_cells, 1], for ops that read the rows by index + ggml_tensor * pooled_rows() const; + + private: + friend class llama_memory_idx_iswa_context; + + kpool_access(ggml_context * ctx, ggml_tensor * k, int64_t n_embd); + + ggml_context * ctx; + ggml_tensor * key_gate; + ggml_tensor * pooled; + }; + + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + // used for errors + explicit llama_memory_idx_iswa_context(llama_memory_status status); + + // used to create a full-cache context + explicit llama_memory_idx_iswa_context(llama_memory_idx_iswa * mem); + + // used to create an update context + llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + llama_context * lctx, + bool optimize); + + // used to create a batch processing context from a batch + llama_memory_idx_iswa_context( + llama_memory_idx_iswa * mem, + slot_info_vec_t sinfos_base, + slot_info_vec_t sinfos_swa, + slot_info_vec_t sinfos_idx, + std::vector<llama_ubatch> ubatches); + + ~llama_memory_idx_iswa_context(); // Defined out of line because kpool_state is incomplete here. + + // + // llama_memory_context_i + // + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + // + // llama_memory_idx_iswa_context specific API + // + + const llama_kv_cache_iswa_context * get_attn() const; + + // the per-layer attention context: base for the non-SWA layers, swa for the rest + const llama_kv_cache_context * get_base() const; + const llama_kv_cache_context * get_swa () const; + + const llama_kv_cache_context * get_idx () const; + + // streams in the current slot info, the `ns` of get_k/get_v; 1 if unified + uint32_t get_n_stream() const; + + // hy_v4_flash, complete pools of kpool cells per sequence, scored as whole pools. + uint32_t get_n_kpool () const; // Padded pool count, where the last pool is always unused. + uint32_t get_n_kpool_new() const; // Pools to re-pool this ubatch, padded to a stable bound, never below 1. + kpool_access get_kpool_access(ggml_context * ctx, int32_t il, int64_t n_embd) const; + // sel_mask (F32 [n_sel, 1, 1, n_tokens], can be null): 0 for the live selection slots, -inf for the dead ones + // new_pool_pos (I32 [4*n_new]): M-RoPE position of each new pool's first member, for pooled keys rotated at pooling time + // tail_idxs width: kpool reserves the current group; kpool - 1 keeps only the incomplete tail. + void set_input_kpool(ggml_tensor * pool_cells, ggml_tensor * pool_idxs, ggml_tensor * pool_mask, ggml_tensor * tail_idxs, + ggml_tensor * sel_mask, ggml_tensor * new_pool_idxs, ggml_tensor * new_pool_rep, + const llama_ubatch * ubatch, ggml_tensor * new_pool_pos = nullptr) const; + +private: + llama_memory_idx_iswa * mem = nullptr; + + // the index of the next ubatch to process + size_t i_next = 0; + + std::vector<llama_ubatch> ubatches; + + // streams per ubatch, read from the slot infos before ctx_idx takes them + // declared before sinfos_kpool and ctx_idx, so it is initialised while sinfos_idx is still intact + const std::vector<uint32_t> ns_ubatch; + + // the indexer cells of each ubatch, kept for pools in cache order (qwen4exp): token s*n + i of ubatch u + // sits in cell idxs[s][i] of stream strm[s] of sinfos_kpool[u], and several cells can share a position + const slot_info_vec_t sinfos_kpool; + + const llama_memory_context_ptr ctx_attn; + const llama_memory_context_ptr ctx_idx; + + const llama_memory_status status; + + // Which pools of the layout this ubatch must re-pool. The layout itself belongs to the memory. + struct kpool_state; + kpool_state kpool_build_sizes() const; + void kpool_build_state(const llama_ubatch & ubatch); + const kpool_state & kpool_cur() const; + + // unique_ptr because kpool_state is incomplete here. + std::unique_ptr<kpool_state> kpool_st; + + // The ubatch kpool_st was built for, guards against reads before apply. + size_t i_kpool = SIZE_MAX; + + // Whether this context tracks k-pool states. + bool kpool_track() const; + + // Positions each sequence must re-pool from, cleared only after the first ubatch succeeds + llama_memory_idx_iswa::stale_pos_t mem_idx_stale_batch = llama_memory_idx_iswa::stale_pos_clean(); +}; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 36f3850d5..1287e8051 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -357,10 +357,14 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks); add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size); } - if (hparams.ple_n_heads > 0) { - add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); - add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map); - } + } + + // any arch with an Engram module needs these, not just deepseek41. The token map is what says + // the arch has one at all: qwen4exp sets ple_n_heads for a plain per-layer embedding and has no + // hashed n-gram table, so it must not pick up a stray pad id. + if (hparams.ple_n_heads > 0 && !model->engram_token_map.empty()) { + add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); + add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map); } // the PLE group only means anything whole: write all of it or none diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7529f55c2..a07ab1226 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -17,6 +17,7 @@ #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" #include "llama-memory-hybrid-idx.h" +#include "llama-memory-idx-iswa.h" #include "llama-memory-recurrent.h" #include "llama.h" @@ -306,6 +307,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_hy_v3(params); case LLM_ARCH_HY_V4: return new llama_model_hy_v4(params); + case LLM_ARCH_HY_V4_FLASH: + return new llama_model_hy_v4_flash(params); case LLM_ARCH_SMOLLM3: return new llama_model_smollm3(params); case LLM_ARCH_OPENAI_MOE: @@ -2638,7 +2641,8 @@ void llama_model::print_info() const { if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK2OCR || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_MISTRAL4 || - arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4) { + arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4 || + arch == LLM_ARCH_HY_V4_FLASH) { LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead); LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q); LLAMA_LOG_INFO("%s: n_lora_kv = %d\n", __func__, hparams.n_lora_kv); @@ -3029,6 +3033,75 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr); } } break; + case LLM_ARCH_HY_V4_FLASH: + { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + if (hparams.indexer_top_k == 0) { + // no indexer, so the sparse layers attend their whole context: a plain iSWA pair is enough + llama_kv_cache_iswa::layer_filter_cb filter = [&](uint32_t il) { + return il < hparams.n_layer(); + }; + + // the draft context holds the MTP block alone + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + if (hparams.n_layer_nextn == 0) { + throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp"); + } + filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; + } + + res = new llama_kv_cache_iswa( + *this, + params.no_alloc, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + nullptr, + filter, + nullptr, + nullptr); + } else { + // the indexer lives on the sparse layers, which are the non-SWA ones + llama_memory_idx_iswa::layer_filter_cb filter = [&](uint32_t il) { + return il < hparams.n_layer(); + }; + llama_memory_idx_iswa::layer_filter_cb filter_idx = [&](uint32_t il) { + return il < hparams.n_layer() && !hparams.is_swa(il) && hparams.is_indexer_full(il); + }; + + // the draft context holds the MTP block alone, which is a sparse layer and so + // owns an indexer of its own + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + if (hparams.n_layer_nextn == 0) { + throw std::runtime_error("hy_v4_flash MTP requires the draft block, convert without --no-mtp"); + } + filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; + filter_idx = [&](uint32_t il) { return il >= hparams.n_layer(); }; + } + + res = new llama_memory_idx_iswa( + *this, + params.no_alloc, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + std::move(filter), + std::move(filter_idx)); + } + } break; case LLM_ARCH_DOTS3NOTE: { GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); @@ -3616,6 +3689,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM; // HY_V4 rotates consecutive pairs, matching the reference implementation case LLM_ARCH_HY_V4: + case LLM_ARCH_HY_V4_FLASH: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 diff --git a/src/llama-model.h b/src/llama-model.h index da4964c98..9d11ce517 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -612,10 +612,20 @@ struct llama_layer { // glm5-next k-pool indexer struct ggml_tensor * indexer_kpool_gate = nullptr; struct ggml_tensor * indexer_kpool_ape = nullptr; + // hy_v4_flash: the pool runs over a projection of the normed key, and the pooled result is + // normed again before it is scored + struct ggml_tensor * indexer_kpool_value = nullptr; + struct ggml_tensor * indexer_kpool_norm = nullptr; // deepseek41 struct ggml_tensor * engram_kv = nullptr; struct ggml_tensor * engram_gate = nullptr; // q_weight*k_weight folded + // hy_v4_flash only: biases on engram_kv, the per-head norm of the looked-up rows, and the + // scale/offset of the delta the module writes back + struct ggml_tensor * engram_kv_b = nullptr; + struct ggml_tensor * engram_lookup_norm = nullptr; + struct ggml_tensor * engram_pre_gamma = nullptr; + struct ggml_tensor * engram_hc_scale = nullptr; // MSA struct ggml_tensor * index_q_proj = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c247858..3f1b58e00 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -326,6 +326,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // Engram gate is an elementwise weight (ggml_mul), not a matmul operand quantize &= name.find("engram_gate.weight") == std::string::npos; + // same for these: they are the second operand of a ggml_add / ggml_mul, which has no + // quantized kernel, so quantizing them aborts at run time rather than degrading quality + quantize &= name.find("engram_kv_b.weight") == std::string::npos; + quantize &= name.find("engram_hc_scale.weight") == std::string::npos; + quantize &= name.find("indexer_compressor_ape.weight") == std::string::npos; + // these are very small (e.g. 4x4) quantize &= name.find("altup") == std::string::npos; quantize &= name.find("laurel") == std::string::npos; diff --git a/src/models/hy-v4-flash.cpp b/src/models/hy-v4-flash.cpp new file mode 100644 index 000000000..18deb364f --- /dev/null +++ b/src/models/hy-v4-flash.cpp @@ -0,0 +1,1376 @@ +#include "models.h" + +#include "llama-kv-cache.h" +#include "llama-kv-cache-iswa.h" +#include "llama-memory-idx-iswa.h" + +#include <cmath> + +// HY_V4_FLASH reuses hy_v4's iHC, gated MLA and learnable sink. What it adds: +// - a hybrid stack: "sparse" full-attention layers and sliding-window layers, with their own +// head count and rope base, over one shared MLA latent geometry +// - sandwich norms after the attention output and after the FFN +// - routed experts that run at n_expert_latent instead of n_embd + +// a token that cannot be hashed: outside the map, or an embedding (image) position +static constexpr int64_t HY_V4_FLASH_ENGRAM_DEAD = -1; + +// The reference builds every Engram norm as an RMSNorm with eps left unset, which resolves to the +// fp32 machine epsilon rather than to the model's rms_norm_eps. +static constexpr float HY_V4_FLASH_ENGRAM_EPS = 1.1920929e-07f; + +// cache view length (n_kv) up to which a prefill ubatch attends the indexer selection through dense attention kernels +static constexpr int64_t HY_V4_FLASH_DENSE_FA_MAX_KV = 12288; + +static size_t hy_v4_flash_elem_offset(const ggml_tensor * t, int64_t i) { + return ggml_row_size(t->type, i); +} + +static ggml_tensor * hy_v4_flash_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) { + return ggml_view_1d(ctx, t, ne0, hy_v4_flash_elem_offset(t, i0)); +} + +static ggml_tensor * hy_v4_flash_view_2d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t ne1, int64_t i0) { + return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], hy_v4_flash_elem_offset(t, i0)); +} + +void llama_model_hy_v4_flash::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent, false); + + // routed experts only; the shared and dense FFNs are not clamped + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false); + + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + ml.get_key(LLM_KV_HYPER_CONNECTION_MAGNITUDE, hparams.hc_magnitude); + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + // Compressed indexer, optional: present on the sparse layers only. A file without it falls + // back to those layers attending their whole context, which is what phase 1 shipped, so the + // keys are read only when indexer.top_k says there is an indexer at all. + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k, false); + if (hparams.indexer_top_k > 0) { + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL, hparams.indexer_kpool); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail, false); + ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); + + // indexer.types is what says which layers own one. Without it nothing does, and a file that + // carries the other indexer keys anyway must not select a cache this graph cannot drive. + bool any = false; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + any = any || hparams.is_indexer_full(il); + } + + if (!any) { + hparams.indexer_top_k = 0; + } else { + if (hparams.indexer_kpool < 2 || hparams.indexer_top_k % hparams.indexer_kpool != 0) { + throw std::runtime_error(format( + "hy_v4_flash: indexer top_k %u must be a multiple of the pool size %u", + hparams.indexer_top_k, hparams.indexer_kpool)); + } + if (hparams.indexer_kpool_select_tail && hparams.indexer_top_k / hparams.indexer_kpool < 2) { + throw std::runtime_error("hy_v4_flash: indexer top_k must reserve at least one history group and one tail group"); + } + // the indexer cache mirrors the full-attention child, so a window layer cannot own one + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (!hparams.is_indexer_full(il)) { + continue; + } + if (hparams.is_swa(il)) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u is a sliding-window layer and cannot own an indexer", il)); + } + // the indexer splits each head into a nope front and a rope tail, so the tail must fit + if (hparams.n_rot(il) >= hparams.indexer_head_size) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u indexer key length %u must be larger than the rope dimension %u", + il, hparams.indexer_head_size, hparams.n_rot(il))); + } + } + } + } + + GGML_ASSERT(hparams.is_mla()); + + if (!hparams.is_swa_any()) { + throw std::runtime_error("hy_v4_flash: no sliding-window layer, expected a hybrid stack"); + } + + // One latent geometry for both layer types: only the head count differs, so the graph keeps a + // single latent cache and reads one set of MLA dimensions. dots3note is the arch that varies + // these per layer, and a file written for that shape would be read here as if it did not, so + // refuse it instead of building mis-sized wk_b / wv_b / wo views. + uint32_t n_embd_head_k_mla_swa = 0; + uint32_t n_embd_head_v_mla_swa = 0; + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, n_embd_head_k_mla_swa, false); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, n_embd_head_v_mla_swa, false); + + if ((n_embd_head_k_mla_swa && n_embd_head_k_mla_swa != hparams.n_embd_head_k_mla()) || + (n_embd_head_v_mla_swa && n_embd_head_v_mla_swa != hparams.n_embd_head_v_mla())) { + throw std::runtime_error(format( + "hy_v4_flash: needs one MLA geometry for both layer types, but the " + "sliding-window layers declare key/value %u/%u against %u/%u", + n_embd_head_k_mla_swa, n_embd_head_v_mla_swa, + hparams.n_embd_head_k_mla(), hparams.n_embd_head_v_mla())); + } + + // MLA is driven as MQA here: one shared latent row per token, so a file claiming more + // key/value heads would silently disagree with the cache this graph builds + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.n_head_kv(il) != 1) { + throw std::runtime_error(format( + "hy_v4_flash: layer %u has n_head_kv %u, expected 1 for MLA", + il, hparams.n_head_kv(il))); + } + } + + // Engram, optional: the same n-gram hash tables deepseek41 carries, with one module per Engram + // layer. Multipliers, primes and offsets are all stored flat as [module][...] and indexed at + // module stride, so their lengths pin down the module count. + std::vector<uint32_t> e_layers; + ml.get_arr(LLM_KV_PLE_LAYERS, e_layers, false); + if (!e_layers.empty()) { + std::vector<uint64_t> mult; + std::vector<uint64_t> offsets; + std::vector<uint64_t> primes; + ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size); + ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram); + ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim); + ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult); + ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets); + ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes); + ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id); + ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map); + + const size_t n_e = e_layers.size(); + const size_t n_gram = hparams.ple_ngram_size; + const size_t n_cols = n_gram >= 2 ? (n_gram - 1)*hparams.ple_heads_per_ngram : 0; + if (n_cols == 0 || hparams.ple_head_dim == 0 || + mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols || + mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS || + hparams.engram_pad_id >= engram_token_map.size()) { + throw std::runtime_error("hy_v4_flash engram metadata is inconsistent"); + } + for (size_t e = 0; e < n_e; ++e) { + if (e_layers[e] >= hparams.n_layer() || (e > 0 && e_layers[e] <= e_layers[e - 1])) { + throw std::runtime_error(format("hy_v4_flash engram layer %u is out of range or out of order", e_layers[e])); + } + hparams.is_ple_impl.set(e_layers[e]); + } + for (size_t c = 0; c < n_e*n_cols; ++c) { + // a bucket count of 0 traps in the hash; row ids are int32 + if (primes[c] < 2 || offsets[c] + primes[c] > (uint64_t) INT32_MAX) { + throw std::runtime_error(format("hy_v4_flash engram head %zu: bucket range out of range", c)); + } + hparams.ple_head_offsets[c] = (uint32_t) offsets[c]; + hparams.ple_head_vocab_sizes[c] = (uint32_t) primes[c]; + } + std::copy(mult.begin(), mult.end(), hparams.ple_layer_multipliers.begin()); + hparams.ple_n_heads = (uint32_t) n_cols; + hparams.ple_n_modules = (uint32_t) n_e; + } + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_hy_v4_flash::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + GGML_ASSERT(n_embd_head_qk_nope >= 1); + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_ff_exp = hparams.n_ff_exp(); + const int64_t n_expert_shared = hparams.n_expert_shared; + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t n_embd_latent = hparams.n_expert_latent ? hparams.n_expert_latent : n_embd; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc * n_embd, hc}, 0); + hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc}, 0); + hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + + const int64_t ple_n_heads = hparams.ple_n_heads; + const int64_t ple_head_dim = hparams.ple_head_dim; + if (ple_n_heads > 0) { + // every module's buckets live in one table, with the converter rebasing each module's head + // offsets into it, so the row count is the end of the last bucket + int64_t n_rows = 0; + for (size_t c = 0; c < (size_t) hparams.ple_n_modules*ple_n_heads; ++c) { + n_rows = std::max(n_rows, (int64_t) hparams.ple_head_offsets[c] + hparams.ple_head_vocab_sizes[c]); + } + // 118 GiB at the full size: read lazily, so only the rows a batch hashes to are faulted in + per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {ple_head_dim, n_rows}, TENSOR_READ_LAZY); + } + + // the MTP block sits after the trunk and is a sparse layer in its own right, minus the + // hyper-connections: it is not part of the hc stack the trunk runs + for (int i = 0; i < (int) hparams.n_layer_all; ++i) { + auto & layer = layers[i]; + const bool is_nextn = i >= n_layer; + + // the sliding-window layers carry more heads over the same latent geometry + const int64_t n_head_l = hparams.n_head(i); + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head_l}, 0); + + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head_l * n_embd_head_k_mla}, 0); + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0); + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0); + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_qk_nope, kv_lora_rank, n_head_l}, 0); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head_l}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head_l * n_embd_head_v_mla, n_embd}, 0); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_l * n_embd_head_v_mla}, 0); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + + if (!is_nextn) { + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {2 * hc}, 0); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {2}, 0); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc * n_embd, 2 * hc}, 0); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {2 * hc}, 0); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {2}, 0); + } else { + // the draft block folds the next token's embedding into the trunk's final hidden state + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, 0); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, 0); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, 0); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, 0); + } + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + + // is_indexer_full() only covers the trunk, and the draft block always owns an indexer + if (hparams.indexer_top_k > 0 && (is_nextn || hparams.is_indexer_full(i))) { + const int64_t n_idx = hparams.indexer_head_size; + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t kpool = hparams.indexer_kpool; + + layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd, n_idx}, 0); + layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx}, 0); + // the pooled quantity is a projection of the normed key, not the key itself, and the + // gate that weights the pooling is another one: both square at the indexer width + layer.indexer_kpool_value = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_VALUE, "weight", i), {n_idx, n_idx}, 0); + layer.indexer_kpool_gate = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_GATE, "weight", i), {n_idx, n_idx}, 0); + layer.indexer_kpool_ape = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_APE, "weight", i), {n_idx, kpool}, 0); + layer.indexer_kpool_norm = create_tensor(tn(LLM_TENSOR_INDEXER_KPOOL_NORM, "weight", i), {n_idx}, 0); + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx*n_idx_head}, 0); + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0); + } + + if (!is_nextn && engram_index(i) >= 0) { + // the hc key projections and the single value projection are fused into one matmul, as + // deepseek41 does; unlike deepseek41 they carry a bias + layer.engram_kv = create_tensor(tn(LLM_TENSOR_ENGRAM_KV, "weight", i), {ple_n_heads * ple_head_dim, n_embd * (hc + 1)}, 0); + layer.engram_kv_b = create_tensor(tn(LLM_TENSOR_ENGRAM_KV_B, "weight", i), {n_embd, hc + 1}, 0); + layer.engram_gate = create_tensor(tn(LLM_TENSOR_ENGRAM_GATE, "weight", i), {n_embd, hc}, 0); + layer.engram_lookup_norm = create_tensor(tn(LLM_TENSOR_ENGRAM_LOOKUP_NORM, "weight", i), {ple_head_dim, ple_n_heads}, 0); + layer.engram_pre_gamma = create_tensor(tn(LLM_TENSOR_ENGRAM_PRE_GAMMA, "weight", i), {n_embd}, 0); + layer.engram_hc_scale = create_tensor(tn(LLM_TENSOR_ENGRAM_HC_SCALE, "weight", i), {n_embd, hc}, 0); + } + + if (!is_nextn && i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + if (n_expert == 0 || n_expert_used == 0) { + throw std::runtime_error("hy_v4_flash: n_expert and n_expert_used must be > 0"); + } + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + + if (hparams.n_expert_latent) { + layer.ffn_routed_down = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_DOWN, "weight", i), {n_embd, n_embd_latent}, 0); + layer.ffn_routed_up = create_tensor(tn(LLM_TENSOR_FFN_ROUTED_UP, "weight", i), {n_embd_latent, n_embd}, 0); + } + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd_latent, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd_latent, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + } + } +} + +int llama_model_hy_v4_flash::engram_index(int il) const { + if (hparams.ple_n_heads == 0 || !hparams.is_ple(il)) { + return -1; + } + int e = 0; + for (int j = 0; j < il; ++j) { + e += hparams.is_ple(j); + } + return e; +} + +void llama_model_hy_v4_flash::engram_hash(int e, const llama_ubatch & ubatch, + const std::vector<llama_token> & prev, + std::vector<int32_t> & ids, std::vector<float> & valid) const { + const int64_t n_gram = hparams.ple_ngram_size; + const int64_t n_prev = n_gram - 1; + const int64_t n_head = hparams.ple_heads_per_ngram; + const int64_t n_cols = hparams.ple_n_heads; + + const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram; + const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols; + const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols; + const int64_t pad = engram_token_map[hparams.engram_pad_id]; + + const auto cid = [&](llama_token t) -> int64_t { + return t < 0 || t >= (llama_token) engram_token_map.size() ? HY_V4_FLASH_ENGRAM_DEAD : engram_token_map[t]; + }; + + ids.resize(ubatch.n_tokens*n_cols); + valid.resize(ubatch.n_tokens*n_cols); + + std::vector<uint64_t> toks(n_gram); + + for (int64_t i = 0; i < ubatch.n_tokens; ++i) { + // A history shorter than the n-gram is padded, not dropped: the reference shifts the token + // window and fills the overhang with the pad id, so a real row is looked up even at the + // start of a sequence. Only an embedding (image) position has no n-gram at all, and there + // the reference marks every column invalid and masks the whole contribution. + const bool is_embd = !ubatch.token || (ubatch.is_mixed() && ubatch.type[i]); + int64_t c = is_embd ? HY_V4_FLASH_ENGRAM_DEAD : cid(ubatch.token[i]); + const bool dead = c == HY_V4_FLASH_ENGRAM_DEAD; + toks[0] = dead ? pad : c; + for (int64_t s = 1; s < n_gram; ++s) { + c = ubatch.pos[i] - s < 0 ? HY_V4_FLASH_ENGRAM_DEAD : cid(prev[i*n_prev + (n_prev - s)]); + toks[s] = c == HY_V4_FLASH_ENGRAM_DEAD ? pad : c; + } + + uint64_t rolling = toks[0]*mult[0]; + for (int64_t o = 1; o < n_gram; ++o) { + rolling ^= toks[o]*mult[o]; + for (int64_t h = 0; h < n_head; ++h) { + const int64_t col = (o - 1)*n_head + h; + ids [i*n_cols + col] = (int32_t) (rolling % primes[col] + offs[col]); + valid[i*n_cols + col] = dead ? 0.0f : 1.0f; + } + } + } +} + +// the row ids and their validity are host-side, so they are graph inputs rather than computed +class llm_graph_input_hy_v4_flash : public llm_graph_input_i { +public: + llm_graph_input_hy_v4_flash(const llama_model_hy_v4_flash & model, + const llama_kv_cache_context * mctx) : model(model), mctx(mctx) {} + + void set_input(const llama_ubatch * ubatch) override; + bool can_reuse(const llm_graph_params & params) override; + + std::vector<ggml_tensor *> engram_ids; + std::vector<ggml_tensor *> engram_valid; + + const llama_model_hy_v4_flash & model; + const llama_kv_cache_context * mctx; +}; + +void llm_graph_input_hy_v4_flash::set_input(const llama_ubatch * ubatch) { + if (engram_ids.empty()) { + return; + } + + // a token shared by several sequences has the same history in each, so its first one resolves it + std::vector<llama_token> prev; + mctx->get_prev_tokens(*ubatch, model.hparams.ple_ngram_size - 1, prev); + + // hash every module before the first gather, so a lazily read table gets one batched readahead + std::vector<std::vector<int32_t>> ids(engram_ids.size()); + std::vector<std::vector<float>> valid(engram_ids.size()); + std::vector<int32_t> all; + for (size_t e = 0; e < engram_ids.size(); ++e) { + model.engram_hash((int) e, *ubatch, prev, ids[e], valid[e]); + all.insert(all.end(), ids[e].begin(), ids[e].end()); + } + if (model.can_prefetch.count(model.per_layer_tok_embd)) { + llama_prefetch_rows(model.per_layer_tok_embd, all.data(), all.size()); + } + for (size_t e = 0; e < engram_ids.size(); ++e) { + if (engram_ids[e]->buffer) { + ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t)); + } + if (engram_valid[e]->buffer) { + ggml_backend_tensor_set(engram_valid[e], valid[e].data(), 0, valid[e].size()*sizeof(float)); + } + } +} + +bool llm_graph_input_hy_v4_flash::can_reuse(const llm_graph_params & params) { + // The memory context is rebuilt for every batch, so the one captured at graph build time is + // dangling by now and set_input would read freed memory. Re-fetch it, exactly as deepseek41 + // does for its own Engram input. + const auto * iswa = model.hparams.indexer_top_k > 0 + ? static_cast<const llama_memory_idx_iswa_context *>(params.mctx)->get_attn() + : static_cast<const llama_kv_cache_iswa_context *>(params.mctx); + mctx = iswa->get_base(); + + bool res = true; + for (auto * t : engram_ids) { + res &= t->ne[0] == (int64_t) (model.hparams.ple_n_heads*params.ubatch.n_tokens); + } + return res; +} + +std::unique_ptr<llm_graph_context> llama_model_hy_v4_flash::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique<graph_mtp>(*this, params); + } + return std::make_unique<graph>(*this, params); +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_engram( + const llama_model & model, + ggml_tensor * x, + ggml_tensor * rows, + ggml_tensor * valid, + int il) const { + const auto & layer = model.layers[il]; + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + const int64_t n_cols = hparams.ple_n_heads; + const int64_t head_dim = hparams.ple_head_dim; + + GGML_ASSERT(rows->ne[0] == head_dim && rows->ne[1] == n_cols && rows->ne[2] == nt); + + // delta 1: each looked-up row is RMS-normed on its own and scaled by its head's weight. + // The per-column scale by valid is belt and braces: valid is uniform across a token's columns + // (a short history is padded, not invalidated), and the whole-token mask below is what actually + // suppresses an embedding position. + ggml_tensor * r = ggml_rms_norm(ctx0, rows, HY_V4_FLASH_ENGRAM_EPS); + r = ggml_mul(ctx0, r, layer.engram_lookup_norm); + r = ggml_mul(ctx0, r, ggml_reshape_3d(ctx0, valid, 1, n_cols, nt)); + r = ggml_reshape_2d(ctx0, r, head_dim*n_cols, nt); + cb(r, "engram_rows_normed", il); + + // delta 2: the fused key/value projection carries a bias + ggml_tensor * kv = build_lora_mm(layer.engram_kv, r); + kv = ggml_add(ctx0, ggml_reshape_3d(ctx0, kv, n_embd, hc + 1, nt), layer.engram_kv_b); + cb(kv, "engram_kv", il); + + ggml_tensor * key = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, hc, nt, kv->nb[1], kv->nb[2], 0)); + ggml_tensor * value = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, 1, nt, kv->nb[1], kv->nb[2], hc*kv->nb[1])); + + // the gate is deepseek41's: engram_gate folds norm_key.w * norm_query.w + ggml_tensor * egate = ggml_is_quantized(layer.engram_gate->type) ? ggml_cast(ctx0, layer.engram_gate, GGML_TYPE_F32) : layer.engram_gate; + ggml_tensor * dot = ggml_mul(ctx0, ggml_mul(ctx0, ggml_rms_norm(ctx0, x, HY_V4_FLASH_ENGRAM_EPS), ggml_rms_norm(ctx0, key, HY_V4_FLASH_ENGRAM_EPS)), egate); + dot = ggml_scale(ctx0, ggml_sum_rows(ctx0, dot), 1.0f/sqrtf((float) n_embd)); + cb(dot, "engram_dot", il); + + // sigmoid(copysign(sqrt(max(|dot|, 1e-6)), dot)); sign(0) = +1 as in copysign + ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY)); + ggml_tensor * sign = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_neg(ctx0, dot)), -2.0f, 1.0f); + ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, sign)); + cb(gate, "engram_gate", il); + + // delta 3: the delta is the normed value scaled by a learned gamma, not the raw value + ggml_tensor * signal = ggml_mul(ctx0, ggml_rms_norm(ctx0, value, HY_V4_FLASH_ENGRAM_EPS), layer.engram_pre_gamma); + // delta 4: the per-hc offset is additive and per channel, so it broadcasts over tokens only + ggml_tensor * scale = ggml_add(ctx0, ggml_repeat_4d(ctx0, gate, n_embd, hc, nt, 1), layer.engram_hc_scale); + cb(scale, "engram_scale", il); + + ggml_tensor * delta = ggml_mul(ctx0, ggml_repeat_4d(ctx0, signal, n_embd, hc, nt, 1), scale); + + // A token with no valid n-gram contributes nothing. Zeroing its columns is not enough: the + // projection bias would still put a constant through the value path. This is the first token + // of every sequence, and any embedding (image) position. + ggml_tensor * any = ggml_sum_rows(ctx0, ggml_reshape_2d(ctx0, valid, n_cols, nt)); + any = ggml_step(ctx0, any); + delta = ggml_mul(ctx0, delta, ggml_reshape_3d(ctx0, any, 1, 1, nt)); + cb(delta, "engram_delta", il); + + return ggml_add(ctx0, x, delta); +} + +// reduce hc streams x[:,i,:] weighted by w[i,:] -> [n_embd, n_tokens] +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_reduce( + ggml_tensor * x, + ggml_tensor * weights, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(x->ne[1] == hc); + + // the fused node is registered per layer, so the head reduce (il < 0) keeps the loop + if (cparams.fused_dsv4_hc_pre && il >= 0) { + ggml_tensor * result = ggml_dsv4_hc_pre(ctx0, x, weights); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, result, il}); + return result; + } + + ggml_tensor * result = nullptr; + for (int64_t ih = 0; ih < hc; ++ih) { + ggml_tensor * xh = ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih * x->nb[1]); + ggml_tensor * wh = ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih * weights->nb[0]); + ggml_tensor * cur = ggml_mul(ctx0, xh, wh); + result = result ? ggml_add(ctx0, result, cur) : cur; + } + return result; +} + +llama_model_hy_v4_flash::graph::hc_pre +llama_model_hy_v4_flash::graph::build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + GGML_ASSERT(x->ne[0] == n_embd && x->ne[1] == hc); + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_mixes", il); + + ggml_tensor * scale_pre = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 0); + ggml_tensor * scale_post = hy_v4_flash_view_1d(ctx0, hc_scale, 1, 1); + ggml_tensor * base_pre = hy_v4_flash_view_1d(ctx0, hc_base, hc, 0); + ggml_tensor * base_post = hy_v4_flash_view_1d(ctx0, hc_base, hc, hc); + + ggml_tensor * pre = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, 0); + pre = ggml_mul(ctx0, pre, scale_pre); + pre = ggml_add(ctx0, pre, base_pre); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_pre", il); + + ggml_tensor * post = hy_v4_flash_view_2d(ctx0, mixes, hc, nt, hc); + post = ggml_mul(ctx0, post, scale_post); + post = ggml_add(ctx0, post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, hparams.hc_magnitude); + post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps); + cb(post, "hc_post_gate", il); + + return { build_hc_reduce(x, pre, il), post }; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[1]; + GGML_ASSERT(x->ne[0] == n_embd); + GGML_ASSERT(residual->ne[1] == hc); + + // comb is the identity here: each stream keeps its own residual, hy_v4 has no cross-stream mix + if (cparams.fused_dsv4_hc_post) { + ggml_tensor * result = ggml_dsv4_hc_post(ctx0, x, residual, post, nullptr); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, result, il}); + return result; + } + + ggml_tensor * out = nullptr; + for (int64_t i = 0; i < hc; ++i) { + ggml_tensor * res_i = ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], i * residual->nb[1]); + ggml_tensor * post_i = ggml_view_2d(ctx0, post, 1, nt, post->nb[1], i * post->nb[0]); + ggml_tensor * cur = ggml_add(ctx0, res_i, ggml_mul(ctx0, x, post_i)); + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, nt); + out = out ? ggml_concat(ctx0, out, cur, 1) : cur; + } + + return out; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc * n_embd, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, hparams.f_norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_head_mixes", -1); + + ggml_tensor * pre = ggml_mul(ctx0, mixes, hc_scale); + pre = ggml_add(ctx0, pre, hc_base); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_head_pre", -1); + + return build_hc_reduce(x, pre, -1); +} + +// The memory composes an iSWA pair rather than being one, so the standard input cannot recover its +// context from params.mctx on its own. +class llama_model_hy_v4_flash::llm_graph_input_attn_idx : public llm_graph_input_attn_k_iswa { +public: + using llm_graph_input_attn_k_iswa::llm_graph_input_attn_k_iswa; + + const llama_kv_cache_iswa_context * mctx_of(const llm_graph_params & params) const override { + return static_cast<const llama_memory_idx_iswa_context *>(params.mctx)->get_attn(); + } +}; + +class llama_model_hy_v4_flash::llm_graph_input_kpool : public llm_graph_input_i { +public: + llm_graph_input_kpool(const llama_memory_idx_iswa_context * mctx, uint32_t kpool) : mctx(mctx), kpool(kpool) {} + virtual ~llm_graph_input_kpool() = default; + + void set_input(const llama_ubatch * ubatch) override { + mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch); + mctx->set_input_kpool(pool_cells, pool_idxs, pool_mask, tail_idxs, sel_mask, + new_pool_idxs, new_pool_rep, ubatch, new_pool_pos); + } + + bool can_reuse(const llm_graph_params & params) override { + mctx = static_cast<const llama_memory_idx_iswa_context *>(params.mctx); + + bool res = true; + + res &= k_idxs->ne[0] == params.ubatch.n_tokens; + res &= pool_cells->ne[0] == mctx->get_n_kpool(); + res &= pool_mask->ne[1] == params.ubatch.n_tokens; + res &= tail_idxs->ne[1] == params.ubatch.n_tokens; + res &= n_kv == mctx->get_idx()->get_n_kv(); + res &= n_new == std::max(mctx->get_n_kpool_new(), 1u); + + return res; + } + + ggml_tensor * k_idxs = nullptr; // I64 [n_tokens] + ggml_tensor * pool_cells = nullptr; // I32 [n_pool] cell caching each pool's pooled key + ggml_tensor * pool_idxs = nullptr; // I32 [kpool, n_pool] member cells per pool + ggml_tensor * pool_mask = nullptr; // F32/F16 [n_pool, n_tokens] + ggml_tensor * tail_idxs = nullptr; // I32 [kpool, n_tokens] when the current group is forced + ggml_tensor * sel_mask = nullptr; // F32 [n_sel, 1, 1, n_tokens] + ggml_tensor * new_pool_idxs = nullptr; // I32 [kpool*n_new] + ggml_tensor * new_pool_rep = nullptr; // I64 [n_new] + ggml_tensor * new_pool_pos = nullptr; // I32 [4*n_new] first member's position per new pool + + const llama_memory_idx_iswa_context * mctx; + const uint32_t kpool; + uint32_t n_new = 0; + uint32_t n_sel = 0; + uint32_t n_kv = 0; +}; + +llama_model_hy_v4_flash::llm_graph_input_kpool * llama_model_hy_v4_flash::graph::build_inp_kpool( + const llama_memory_idx_iswa_context * mctx_idx) { + const auto * mctx_lid = mctx_idx->get_idx(); + + const uint32_t kpool = hparams.indexer_kpool; + const uint32_t n_pool = mctx_idx->get_n_kpool(); + // a ubatch that completes no pool still builds one dummy entry, so the graph does not change + // shape every kpool tokens + const uint32_t n_new = std::max(mctx_idx->get_n_kpool_new(), 1u); + + const auto type_mask = cparams.fused_lid ? GGML_TYPE_F16 : GGML_TYPE_F32; + + auto inp = std::make_unique<llm_graph_input_kpool>(mctx_idx, kpool); + + inp->k_idxs = mctx_lid->build_input_k_idxs(ctx0, ubatch); + inp->pool_cells = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pool); + inp->pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_pool); + inp->pool_mask = ggml_new_tensor_2d(ctx0, type_mask, n_pool, n_tokens); + inp->tail_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, hparams.indexer_kpool_select_tail ? kpool : kpool - 1, n_tokens); + ggml_set_input(inp->pool_cells); + ggml_set_input(inp->pool_idxs); + ggml_set_input(inp->pool_mask); + ggml_set_input(inp->tail_idxs); + + ggml_build_forward_expand(gf, inp->pool_cells); + ggml_build_forward_expand(gf, inp->pool_idxs); + ggml_build_forward_expand(gf, inp->pool_mask); + ggml_build_forward_expand(gf, inp->tail_idxs); + + inp->n_kv = mctx_lid->get_n_kv(); + + { + // The reference keeps group_topk - 1 scored groups and forces the one the current token + // sits in, which the tail covers even when that group is complete. + const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail); + const int64_t n_sel = kpool*n_top_pool + (hparams.indexer_kpool_select_tail ? kpool : 0); + inp->n_sel = (uint32_t) n_sel; + + if (!cparams.fused_top_k_mask) { + inp->sel_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_sel, 1, 1, n_tokens); + ggml_set_input(inp->sel_mask); + ggml_build_forward_expand(gf, inp->sel_mask); + } + } + + inp->n_new = n_new; + + inp->new_pool_idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kpool*n_new); + ggml_set_input(inp->new_pool_idxs); + inp->new_pool_rep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_new); + ggml_set_input(inp->new_pool_rep); + // 4 planes so the vision case can rotate pooled keys with mrope; text uses plane 0 + inp->new_pool_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 4*n_new); + ggml_set_input(inp->new_pool_pos); + ggml_build_forward_expand(gf, inp->new_pool_pos); + + return (llm_graph_input_kpool *) res->add_input(std::move(inp)); +} + +// rotate only the trailing n_rot of each indexer head, leaving the head's front half untouched +ggml_tensor * llama_model_hy_v4_flash::graph::build_indexer_rope( + ggml_tensor * x, ggml_tensor * pos, int64_t n_idx, float freq_base, int il) const { + const int64_t n_nope = n_idx - n_rot; + GGML_ASSERT(n_nope > 0); + + ggml_tensor * front = ggml_view_3d(ctx0, x, n_nope, x->ne[1], x->ne[2], + x->nb[1], x->nb[2], 0); + ggml_tensor * back = ggml_view_3d(ctx0, x, n_rot, x->ne[1], x->ne[2], + x->nb[1], x->nb[2], ggml_row_size(x->type, n_nope)); + + // the same base the layer's own attention uses, so a cparams override reaches the indexer too + back = ggml_rope_ext(ctx0, ggml_cont(ctx0, back), pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + cb(back, "indexer_rope", il); + + return ggml_concat(ctx0, ggml_cont(ctx0, front), back, 0); +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_kpool_select( + ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * inp_pos, float freq_base, + const llama_layer & layer, + const llama_memory_idx_iswa_context * mctx_idx, llm_graph_input_kpool * inp_kpool, int il) { + const auto * mctx_lid = mctx_idx->get_idx(); + + const int64_t n_idx_head = hparams.indexer_n_head; + const int64_t n_idx = hparams.indexer_head_size; + const int64_t kpool = hparams.indexer_kpool; + const int64_t n_pool = inp_kpool->pool_cells->ne[0]; + const int64_t n_new = inp_kpool->n_new; + const int64_t n_kv = mctx_lid->get_n_kv(); + + // the query comes off the same lora residual the main attention uses, and is rotated at the + // token's own position + ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr); + iq = ggml_reshape_3d(ctx0, iq, n_idx, n_idx_head, n_tokens); + iq = build_indexer_rope(iq, inp_pos, n_idx, freq_base, il); + cb(iq, "indexer_q", il); + + // key side: weight-only RMSNorm, unlike glm5-next's LayerNorm with a bias this model lacks + ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur); + ik = build_norm(ik, layer.indexer_k_norm, nullptr, LLM_NORM_RMS, il); + cb(ik, "indexer_k", il); + + // what the pool averages is a projection of the normed key, and the weights come from another + // projection of it: glm5-next pools the key itself and gates off the hidden state + ggml_tensor * iv = ggml_mul_mat(ctx0, layer.indexer_kpool_value, ik); + ggml_tensor * ig = ggml_mul_mat(ctx0, layer.indexer_kpool_gate, ik); + cb(ig, "indexer_gate", il); + + // cache rows hold value | gate | pooled + ggml_tensor * pzero = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_idx, n_tokens), 0.0f); + ggml_tensor * packed = ggml_concat(ctx0, ggml_concat(ctx0, iv, ig, 0), pzero, 0); + packed = ggml_reshape_3d(ctx0, packed, 3*n_idx, 1, n_tokens); + ggml_build_forward_expand(gf, mctx_lid->cpy_k(ctx0, packed, inp_kpool->k_idxs, il)); + + auto kpool_cache = mctx_idx->get_kpool_access(ctx0, il, n_idx); + + const ggml_type type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32; + + // pool the groups this ubatch completed; the last entry is a dummy when it completed none + ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_idxs); + rows = ggml_reshape_3d(ctx0, rows, 2*n_idx, kpool, n_new); + + ggml_tensor * pk = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2], 0); + ggml_tensor * pg = ggml_view_3d(ctx0, rows, n_idx, kpool, n_new, rows->nb[1], rows->nb[2], + ggml_row_size(rows->type, n_idx)); + + ggml_tensor * logits = ggml_add(ctx0, pg, layer.indexer_kpool_ape); + logits = ggml_cont(ctx0, ggml_permute(ctx0, logits, 1, 0, 2, 3)); // [kpool, n_idx, n_new] + // fold the pool axis into rows: soft_max maps ne[2] to a CUDA gridDim.y that caps at 65535 + ggml_tensor * probs = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, logits, kpool, n_idx*n_new)); + probs = ggml_reshape_3d(ctx0, probs, kpool, n_idx, n_new); + + pk = ggml_cont(ctx0, ggml_permute(ctx0, pk, 1, 0, 2, 3)); + ggml_tensor * pooled_new = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, pk)); // [1, n_idx, n_new] + pooled_new = ggml_reshape_2d(ctx0, pooled_new, n_idx, n_new); + + // the pooled key is normed and then rotated at its group's first position, so the score keeps + // the same relative geometry as the main attention + pooled_new = build_norm(pooled_new, layer.indexer_kpool_norm, nullptr, LLM_NORM_RMS, il); + { + ggml_tensor * pos = ggml_view_1d(ctx0, inp_kpool->new_pool_pos, n_new, 0); + ggml_tensor * p3 = ggml_reshape_3d(ctx0, pooled_new, n_idx, 1, n_new); + pooled_new = ggml_reshape_2d(ctx0, build_indexer_rope(p3, pos, n_idx, freq_base, il), n_idx, n_new); + } + cb(pooled_new, "indexer_pool_k_new", il); + + // write the fresh pooled keys into their rep rows, then read every pool back by cell: the + // older ones come from the rows earlier ubatches wrote + ggml_build_forward_expand(gf, kpool_cache.scatter_pooled(pooled_new, inp_kpool->new_pool_rep)); + + ggml_tensor * weights = ggml_mul_mat(ctx0, layer.indexer_proj, cur); + weights = ggml_scale(ctx0, weights, 1.0f / sqrtf(float(n_idx * n_idx_head))); + cb(weights, "indexer_weights", il); + + ggml_tensor * score = nullptr; + if (cparams.fused_lid) { + // the fused indexer reads every pool's key straight from its cache cell, without gathering them first + // the list goes through a GPU copy: read straight from the host input buffer through src[4], the scores + // come out right but the decode diverges (root cause not found, bit-identical with the copy) + score = ggml_lightning_indexer_rows(ctx0, iq, kpool_cache.pooled_rows(), weights, inp_kpool->pool_mask, + ggml_cont(ctx0, inp_kpool->pool_cells)); + res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); + } else { + ggml_tensor * pooled = kpool_cache.gather_pooled(inp_kpool->pool_cells); + pooled = ggml_reshape_3d(ctx0, pooled, n_idx, 1, n_pool); + cb(pooled, "indexer_pool_k", il); + + ggml_tensor * q_p = ggml_permute(ctx0, iq, 0, 2, 1, 3); // [n_idx, n_tokens, n_head] + ggml_tensor * k_p = ggml_permute(ctx0, pooled, 0, 2, 1, 3); // [n_idx, n_pool, 1] + + ggml_tensor * kq = ggml_mul_mat(ctx0, k_p, q_p); // [n_pool, n_tokens, n_head] + kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3)); // [n_head, n_tokens, n_pool] + score = ggml_relu(ctx0, kq); + score = ggml_mul(ctx0, score, weights); + score = ggml_sum_rows(ctx0, score); // [1, n_tokens, n_pool] + score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); // [n_pool, n_tokens, 1] + score = ggml_add(ctx0, score, inp_kpool->pool_mask); + } + cb(score, "indexer_score", il); + + const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool - hparams.indexer_kpool_select_tail); + ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered + + // keep the cells of the picked pools and of the current group, -inf elsewhere; every + // retained cell is in the token's own sequence, so the kq_mask is not needed on top + ggml_tensor * tail = hparams.indexer_kpool_select_tail ? inp_kpool->tail_idxs : nullptr; + GGML_ASSERT(kpool*n_top_pool + (tail ? tail->ne[0] : 0) == (int64_t) inp_kpool->n_sel); + + if (cparams.fused_top_k_mask) { + ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, type_mask, n_kv), 0.0f); + ggml_tensor * sel = ggml_top_k_mask(ctx0, keep, top_k, score, inp_kpool->pool_idxs, tail); + res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, sel, il}); + cb(sel, "indexer_sel", il); + + return sel; + } + + // Backends without a fused top-k mask scatter the same selection by hand. Everything this arch + // changes about the indexer is upstream of score, so from here on it is glm5-next's path. + ggml_tensor * sel_idx = nullptr; + { + // sel_mask marks the first min(nv, n_top_pool) slots as the live pools, so put the picked + // set in descending score order to match it (ggml_top_k leaves it unordered) + ggml_tensor * sel_score = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, score, 1, n_pool, n_tokens), top_k); // [1, n_top_pool, n_tokens] + ggml_tensor * sel_order = ggml_argsort(ctx0, + ggml_reshape_2d(ctx0, sel_score, n_top_pool, n_tokens), GGML_SORT_ORDER_DESC); + + ggml_tensor * ordered = ggml_get_rows(ctx0, + ggml_reshape_3d(ctx0, ggml_cast(ctx0, top_k, GGML_TYPE_F32), 1, n_top_pool, n_tokens), sel_order); + ordered = ggml_cast(ctx0, + ggml_cont(ctx0, ggml_reshape_2d(ctx0, ordered, n_top_pool, n_tokens)), GGML_TYPE_I32); + cb(ordered, "indexer_top_k", il); + + sel_idx = ggml_get_rows(ctx0, inp_kpool->pool_idxs, + ggml_reshape_1d(ctx0, ordered, n_top_pool*n_tokens)); // [kpool, n_top_pool*n_tokens] + sel_idx = ggml_reshape_2d(ctx0, sel_idx, kpool*n_top_pool, n_tokens); + + if (tail) { + // the tail, with n_kv standing in for the cells it does not have yet + sel_idx = ggml_concat(ctx0, sel_idx, tail, 0); + } + } + + const int64_t n_sel = sel_idx->ne[0]; + GGML_ASSERT(n_sel == (int64_t) inp_kpool->n_sel); + GGML_ASSERT(inp_kpool->sel_mask && inp_kpool->sel_mask->ne[0] == n_sel && + inp_kpool->sel_mask->ne[3] == n_tokens); + + ggml_build_forward_expand(gf, sel_idx); + + ggml_tensor * mask_all = ggml_new_tensor_4d(ctx0, type_mask, n_kv + n_sel, 1, 1, 1); + mask_all = ggml_fill(ctx0, mask_all, -INFINITY); + mask_all = ggml_repeat_4d(ctx0, mask_all, n_kv + n_sel, n_tokens, 1, 1); + mask_all = ggml_reshape_3d(ctx0, mask_all, 1, n_kv + n_sel, n_tokens); + + ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, type_mask, n_sel, 1, 1, 1); + zeros = ggml_fill(ctx0, zeros, 0.0f); + zeros = ggml_repeat_4d(ctx0, zeros, n_sel, n_tokens, 1, 1); + zeros = ggml_reshape_3d(ctx0, zeros, 1, n_sel, n_tokens); + + // Live slots (visible pools, real tail cells) address disjoint cells, and each dead slot writes + // its own dump row n_kv + slot, so a token's scatter indices stay unique: + // idx = dump + live*(idx - dump), live = exp(mask). + ggml_tensor * live = ggml_reshape_2d(ctx0, ggml_exp(ctx0, inp_kpool->sel_mask), n_sel, n_tokens); + ggml_tensor * dump = ggml_arange(ctx0, (float) n_kv, (float) (n_kv + n_sel), 1.0f); + ggml_tensor * idx_f = ggml_cast(ctx0, sel_idx, GGML_TYPE_F32); + idx_f = ggml_add(ctx0, ggml_mul(ctx0, ggml_sub(ctx0, idx_f, dump), live), dump); + sel_idx = ggml_cast(ctx0, idx_f, GGML_TYPE_I32); + + ggml_tensor * sel = ggml_set_rows(ctx0, mask_all, zeros, + ggml_reshape_3d(ctx0, sel_idx, n_sel, n_tokens, 1)); + + sel = ggml_view_2d(ctx0, sel, n_kv, n_tokens, sel->nb[2], 0); + sel = ggml_cont(ctx0, sel); + cb(sel, "indexer_sel", il); + + return sel; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_attn, + ggml_tensor * cur, + ggml_tensor * inp_pos, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il) { + const auto & layer = model.layers[il]; + + const int64_t n_head_l = hparams.n_head(il); + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k)); + const float freq_base_l = model.get_rope_freq_base(cparams, il); + + ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur); + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "q_resid", il); + + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr); + + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head_l, n_tokens, + ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l, 0); + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head_l, n_tokens, + ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head_l, + ggml_row_size(q->type, n_embd_head_qk_nope)); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + + // rope goes last so build_rope_shift() can shift the cache in place + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + ggml_tensor * Vcur = kv_cmpr; + + ggml_tensor * sel = nullptr; + // the draft block is a sparse layer and always owns an indexer, and is_indexer_full() only + // covers the trunk + if (inp_kpool != nullptr && + (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il))) { + sel = build_kpool_select(cur, qr, inp_pos, freq_base_l, layer, mctx_idx, inp_kpool, il); + } + + // wo is applied below so the gate can sit in front of it + ggml_tensor * attn; + + // with a rotated K cache wv_b has to come after the de-rotation, so it is applied here and not in build_attn_mha + auto apply_wv_b = [&](ggml_tensor * a) { + a = ggml_reshape_3d(ctx0, a, kv_lora_rank, n_head_l, n_tokens); + a = ggml_permute(ctx0, a, 0, 2, 1, 3); + a = ggml_mul_mat(ctx0, layer.wv_b, a); + a = ggml_permute(ctx0, a, 0, 2, 1, 3); + a = ggml_cont(ctx0, a); + return ggml_reshape_2d(ctx0, a, a->ne[0]*a->ne[1], n_tokens); + }; + + if (sel != nullptr) { + const auto * mctx_base = mctx_idx->get_base(); + const int64_t n_sel = inp_kpool->n_sel; + + // this layer writes the cache itself, so it rotates q and k like build_attn does + ggml_tensor * k_rot = inp_attn->self_k_rot[il]; + if (k_rot) { + Qcur = llama_mul_mat_hadamard(ctx0, Qcur, k_rot); + Kcur = llama_mul_mat_hadamard(ctx0, Kcur, k_rot); + } + + // The indexer's selection replaces the kq_mask, so this layer cannot go through build_attn: + // write the latent row ourselves, then read the whole cache back and let the mask decide. + ggml_tensor * k = mctx_base->get_k(ctx0, il); + ggml_tensor * v = ggml_view_4d(ctx0, k, kv_lora_rank, k->ne[1], k->ne[2], k->ne[3], + k->nb[1], k->nb[2], k->nb[3], 0); + + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Kcur); + ggml_build_forward_expand(gf, mctx_base->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il)); + + // split the selection into the streams the K view carries + const int64_t n_stream = k->ne[3]; + GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0); + sel = ggml_reshape_4d(ctx0, sel, sel->ne[0], sel->ne[1]/n_stream, 1, n_stream); + + // The sparse hint makes Metal attend the selection with its vec kernels, one query row at a time. A + // prefill ubatch over a short cache view (n_kv) is faster through the dense matrix kernels with the same + // mask: pp2048 on M5 Max with dense everywhere, at depth 0/4k/8k/16k/32k: +5/+29/+12/0/-16%. Small batches + // (decode, MTP/draft verification) would get the dense vec kernel over all n_kv, so they keep the hint. + // Tuned for Metal only; CUDA/Vulkan/SYCL also read the hint. + const int64_t n_kv_hint = n_tokens >= 32 && sel->ne[0] <= HY_V4_FLASH_DENSE_FA_MAX_KV ? 0 : n_sel; + attn = build_attn_mha(Qcur, k, v, nullptr, sel, nullptr, layer.attn_sinks, k_rot ? nullptr : layer.wv_b, + n_kv_hint, kq_scale, il); + + // A single-token selection is read by Metal straight from the indexer picks (the flash attention builds + // the cell list, see ggml_metal_op_hy4_mode4_ok): keep them alive until the attention runs, and make the + // read visible to the concurrency tracking. Other backends ignore these extra sources. Not for a batch: + // the [n_pool, n_tokens] scores would stay allocated through the attention for nothing. + if (n_tokens == 1) { + const ggml_tensor * tkm = sel; + while (tkm->view_src) { + tkm = tkm->view_src; + } + ggml_tensor * fa = res->get_fused_nodes().empty() ? nullptr : res->get_fused_nodes().back().tensor; + if (fa && fa->op == GGML_OP_FLASH_ATTN_EXT && tkm->op == GGML_OP_TOP_K_MASK && fa->src[6] == nullptr && + fa->src[3] && fa->src[3]->view_src == tkm) { + for (int j = 1; j <= 4; ++j) { + fa->src[5 + j] = tkm->src[j]; + } + } + } + if (k_rot) { + attn = llama_mul_mat_hadamard(ctx0, attn, k_rot); + attn = apply_wv_b(attn); + } + } else { + // null unless the K cache is quantized + ggml_tensor * k_rot = hparams.is_swa(il) ? inp_attn->self_k_rot_swa[il] : inp_attn->self_k_rot[il]; + + attn = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, k_rot ? nullptr : layer.wv_b, kq_scale, il); + if (k_rot) { + attn = apply_wv_b(attn); + } + } + cb(attn, "attn_kqv", il); + + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur); + gate = ggml_sigmoid(ctx0, gate); + attn = ggml_mul(ctx0, attn, gate); + cb(attn, "attn_gated", il); + + ggml_tensor * out = build_lora_mm(layer.wo, attn); + out = build_norm(out, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(out, "attn_out", il); + + return out; +} + +ggml_tensor * llama_model_hy_v4_flash::graph::build_latent_moe( + const llama_model & model, + ggml_tensor * cur, + int il) const { + const auto & layer = model.layers[il]; + + // the router scores the full-width input, the experts take the latent one + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, cur); + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * routed_in = layer.ffn_routed_down + ? ggml_mul_mat(ctx0, layer.ffn_routed_down, cur) + : cur; + + ggml_tensor * moe_out = build_moe_ffn(routed_in, + nullptr, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + logits); + cb(moe_out, "ffn_moe_out", il); + + if (layer.ffn_routed_up) { + moe_out = ggml_mul_mat(ctx0, layer.ffn_routed_up, moe_out); + cb(moe_out, "ffn_moe_up", il); + } + + ggml_tensor * shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(shexp, "ffn_shexp", il); + + return ggml_add(ctx0, moe_out, shexp); +} + +llama_model_hy_v4_flash::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t hc = hparams.dsv4_hc_mult; + + ggml_tensor * cur; + + ggml_tensor * inp = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + // With an indexer the memory is a composition, so the attention context has to be fetched out + // of it and the input told how to find it again on reuse. + const auto * mctx_idx = hparams.indexer_top_k > 0 + ? static_cast<const llama_memory_idx_iswa_context *>(mctx) : nullptr; + + llm_graph_input_attn_k_iswa * inp_attn = mctx_idx + ? build_attn_inp_k_iswa( + std::make_unique<llm_graph_input_attn_idx>(hparams, cparams, mctx_idx->get_attn()), + mctx_idx->get_attn()) + : build_attn_inp_k_iswa(); + + llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr; + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // Engram gathers go right after the token embeddings so they share the first graph split and + // the lazily read table stays on the host. An embedding-only ubatch has no token ids to hash, + // which matches the reference skipping Engram at its masked (image) positions. + auto inp_eng = std::make_unique<llm_graph_input_hy_v4_flash>( + static_cast<const llama_model_hy_v4_flash &>(model), inp_attn->mctx->get_base()); + std::vector<ggml_tensor *> engram_rows; + std::vector<ggml_tensor *> engram_valid; + for (int il = 0; ubatch.token && hparams.ple_n_heads > 0 && il < n_layer; ++il) { + if (!hparams.is_ple(il)) { + continue; + } + const int64_t n_cols = hparams.ple_n_heads; + + ggml_tensor * ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cols*n_tokens); + ggml_set_input(ids); + ggml_set_name(ids, ("engram_ids-" + std::to_string(il)).c_str()); + inp_eng->engram_ids.push_back(ids); + + ggml_tensor * valid = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cols*n_tokens); + ggml_set_input(valid); + ggml_set_name(valid, ("engram_valid-" + std::to_string(il)).c_str()); + inp_eng->engram_valid.push_back(valid); + engram_valid.push_back(valid); + + ggml_tensor * rows = ggml_get_rows(ctx0, model.per_layer_tok_embd, ids); + rows = ggml_reshape_3d(ctx0, rows, hparams.ple_head_dim, n_cols, n_tokens); + cb(rows, "engram_rows", il); + ggml_build_forward_expand(gf, rows); + engram_rows.push_back(rows); + } + res->add_input(std::move(inp_eng)); + + // one embedding feeds hc parallel residual streams + ggml_tensor * inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + // Engram writes into the streams before the attention pre-mix reads them + const int e = static_cast<const llama_model_hy_v4_flash &>(model).engram_index(il); + if (e >= 0 && !engram_rows.empty()) { + inpL = build_engram(model, inpL, engram_rows[e], engram_valid[e], il); + cb(inpL, "engram_out", il); + } + + ggml_tensor * residual = inpL; + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); + cur = build_norm(attn_hc.cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il); + + inpL = build_hc_post(cur, residual, attn_hc.post, il); + cb(inpL, "hc_attn_out", il); + + residual = inpL; + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + cur = build_norm(ffn_hc.cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + layer.ffn_up, NULL, NULL, + layer.ffn_gate, NULL, NULL, + layer.ffn_down, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + } else { + cur = build_latent_moe(model, cur, il); + } + + cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, ffn_hc.post, il); + cb(inpL, "l_out", il); + } + + // prune to the requested rows once, after every stream is done. Unmasked nextn embeddings want + // one row per token, so in that mode the narrowing waits until the state has been captured. + if (crop_before_nextn(inp_out_ids)) { + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd * hc, n_tokens); + flat = ggml_get_rows(ctx0, flat, inp_out_ids); + inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs); + } + + cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); + cb(cur, "hc_head", -1); + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + + // the draft block consumes this exact state through its hnorm, so expose it for the caller to + // hand back via llama_batch_ext_set_embd_state() + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (crop_after_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; +} + +llama_model_hy_v4_flash::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + graph(params, no_build{}) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "hy_v4_flash MTP needs the draft block, convert without --no-mtp"); + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && + "hy_v4_flash MTP block is missing"); + + const auto * mctx_idx = hparams.indexer_top_k > 0 + ? static_cast<const llama_memory_idx_iswa_context *>(mctx) : nullptr; + + llm_graph_input_attn_k_iswa * inp_attn = mctx_idx + ? build_attn_inp_k_iswa( + std::make_unique<llm_graph_input_attn_idx>(hparams, cparams, mctx_idx->get_attn()), + mctx_idx->get_attn()) + : build_attn_inp_k_iswa(); + + llm_graph_input_kpool * inp_kpool = mctx_idx ? build_inp_kpool(mctx_idx) : nullptr; + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // the draft takes the next token's embedding alongside the trunk's final hidden state, which + // is the same tensor the trunk's own output head reads: already collapsed over the hc streams + // and already passed through output_norm + auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * tok_embd = ubatch.token + ? ggml_get_rows(ctx0, model.tok_embd, inp->tokens) + : inp->embd; + ggml_tensor * prev_h = inp->h; + res->add_input(std::move(inp)); + + ggml_tensor * cur = ggml_concat(ctx0, + build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il), + build_norm(prev_h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il), 0); + cur = ggml_mul_mat(ctx0, layer.nextn.eh_proj, cur); + cb(cur, "mtp_eh_proj", il); + + // from here it is an ordinary sparse layer, except that the residual is the plain hidden state + // rather than one of the trunk's hc streams + ggml_tensor * residual = cur; + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cur = build_attention(model, inp_attn, cur, inp_pos, mctx_idx, inp_kpool, il); + cur = ggml_add(ctx0, cur, residual); + + if (crop_before_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + residual = cur; + cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cur = build_latent_moe(model, cur, il); + cur = build_norm(cur, layer.ffn_post_norm, nullptr, LLM_NORM_RMS, il); + cur = ggml_add(ctx0, cur, residual); + + cur = build_norm(cur, layer.nextn.shared_head_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_result_norm", -1); + + // the draft's own post-norm state, so a second draft step can be fed from this one + res->t_h_nextn = cur; + + if (crop_after_nextn(inp_out_ids)) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_lora_mm(model.output, cur); + cb(cur, "mtp_result_output", -1); + + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 1fc7cae1d..f8cdc66fc 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -9,6 +9,7 @@ #include <map> class llama_memory_hybrid_idx_context; +class llama_memory_idx_iswa_context; // ref: https://github.com/ggml-org/llama.cpp/pull/28068 static inline ggml_tensor * build_gdn_l2_norm(ggml_context * ctx, ggml_tensor * x, float eps) { @@ -2370,6 +2371,122 @@ struct llama_model_hy_v4 : public llama_model_base { }; +// HY_V4_FLASH: hy_v4's iHC + gated MLA + sinks, with a hybrid attention stack (sparse full +// layers alternating with sliding-window layers, each with its own head count and rope base), +// sandwich norms and routed experts that run in a latent space. +struct llama_model_hy_v4_flash : public llama_model_base { + llama_model_hy_v4_flash(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + // -1 when the layer has no Engram module, else its index among the Engram layers + int engram_index(int il) const; + + // n-gram hash of module e. Writes one row id per (token, column) into ids, and alongside it the + // per-column validity: a short history is padded, as in deepseek41, so only a position with no + // n-gram at all (an embedding position) is marked invalid. + void engram_hash(int e, const llama_ubatch & ubatch, const std::vector<llama_token> & prev, + std::vector<int32_t> & ids, std::vector<float> & valid) const; + + class llm_graph_input_kpool; + + // the memory only composes an iSWA pair, so the standard input cannot find its context in + // params.mctx by itself + class llm_graph_input_attn_idx; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + // the MTP graph reuses the helpers without running the trunk + struct no_build {}; + graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {} + + // iHC, same layout as hy_v4: pre reduces the hc streams to one and returns the per-stream + // post gates, post writes the sublayer output back, head collapses before the final norm. + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + }; + + // reduce the hc streams to one, weighted by the pre gates + ggml_tensor * build_hc_reduce( + ggml_tensor * x, + ggml_tensor * weights, + int il) const; + + hc_pre build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + int il) const; + + ggml_tensor * build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + int il) const; + + ggml_tensor * build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const; + + // mctx_idx and inp_kpool are null when the file carries no indexer, and the sparse layers + // then attend their whole context + ggml_tensor * build_attention( + const llama_model & model, + llm_graph_input_attn_k_iswa * inp_attn, + ggml_tensor * cur, + ggml_tensor * inp_pos, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il); + + llm_graph_input_kpool * build_inp_kpool(const llama_memory_idx_iswa_context * mctx_idx); + + // rotate only the trailing n_rot of each indexer head + ggml_tensor * build_indexer_rope(ggml_tensor * x, ggml_tensor * pos, int64_t n_idx, + float freq_base, int il) const; + + // score the pooled keys against the query and keep the best groups, plus the group the + // current token sits in + ggml_tensor * build_kpool_select( + ggml_tensor * cur, + ggml_tensor * qr, + ggml_tensor * inp_pos, + float freq_base, + const llama_layer & layer, + const llama_memory_idx_iswa_context * mctx_idx, + llm_graph_input_kpool * inp_kpool, + int il); + + // routed experts run at n_expert_latent, the router and the shared expert at n_embd + ggml_tensor * build_latent_moe( + const llama_model & model, + ggml_tensor * cur, + int il) const; + + // Engram writes its delta into the hc residual streams before the layer's attention pre-mix + ggml_tensor * build_engram( + const llama_model & model, + ggml_tensor * x, + ggml_tensor * rows, + ggml_tensor * valid, + int il) const; + }; + + // the draft block: one sparse layer fed the next token's embedding and the trunk's final + // hidden state, with no hyper-connections of its own + struct graph_mtp : public graph { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_hunyuan_vl : public llama_model_base { llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 3ed418140..7f7b0904a 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -2711,6 +2711,16 @@ extern "C" {
// broadcast:
// ne3 % ne33 == 0
//
+ // same, over the K rows listed in rows (I32 [n_out]): k holds every row ([n_embd, 1, n_rows_k, n_stream]),
+ // the result and the mask have one entry per list element (the gather is fused into the scoring)
+ GGML_API struct ggml_tensor * ggml_lightning_indexer_rows(
+ struct ggml_context * ctx,
+ struct ggml_tensor * q,
+ struct ggml_tensor * k,
+ struct ggml_tensor * weights,
+ struct ggml_tensor * mask,
+ struct ggml_tensor * rows);
+
GGML_API struct ggml_tensor * ggml_lightning_indexer(
struct ggml_context * ctx,
struct ggml_tensor * q,
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index dbdde6997..463f6801a 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -12457,6 +12457,7 @@ void ggml_compute_forward_lightning_indexer(
const ggml_tensor * k = dst->src[1];
const ggml_tensor * w = dst->src[2]; // weights
const ggml_tensor * m = dst->src[3]; // mask
+ const ggml_tensor * r = dst->src[4]; // optional list of the K rows to score
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT( q->type == GGML_TYPE_F32);
@@ -12484,7 +12485,8 @@ void ggml_compute_forward_lightning_indexer(
const int n_head = q->ne[1];
const int n_tokens = q->ne[2];
const int n_stream = q->ne[3];
- const int n_kv = k->ne[2];
+ const int n_kv = r ? (int) r->ne[0] : (int) k->ne[2];
+ const int32_t * rows = r ? (const int32_t *) r->data : nullptr;
ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float;
GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type");
@@ -12509,7 +12511,9 @@ void ggml_compute_forward_lightning_indexer(
const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3);
float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 );
for (int ik = ir0; ik < ir1; ++ik) {
- char * k_row = (char *) k->data + ik*nbk2 + s*nbk3;
+ const int64_t ir = rows ? rows[ik] : ik;
+ GGML_ASSERT(ir >= 0 && ir < k->ne[2]);
+ char * k_row = (char *) k->data + ir*nbk2 + s*nbk3;
if (k_to_float) {
k_to_float(k_row, k_row_f32, n_embd);
} else {
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 06956ce7a..bd2f5590b 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -6209,7 +6209,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_SOLVE_TRI:
return true;
case GGML_OP_LIGHTNING_INDEXER:
- return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
+ // the row-list form (src[4]) is Metal/CPU only
+ return op->src[4] == nullptr && ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
case GGML_OP_TOP_K_MASK:
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index 597c126ca..073872b90 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
@@ -12,6 +12,57 @@
#import <Metal/Metal.h>
+#include <mach/mach_time.h>
+#include <math.h>
+
+// hy4 profiling knobs, read once:
+// GGML_METAL_N_MAIN nodes encoded+committed by the calling thread first: <1 = fraction of the graph, >=1 = node count
+// GGML_METAL_CB_TIMING=1 log host encode and GPU start/end times of every graph command buffer
+static float hy4_metal_n_main(void) {
+ static float v = -1.0f;
+ if (v < 0.0f) {
+ v = 0.0f;
+ const char * env = getenv("GGML_METAL_N_MAIN");
+ if (env != NULL && *env != '\0') {
+ char * end = NULL;
+ const float f = strtof(env, &end);
+ if (*end == '\0' && isfinite(f) && f >= 0.0f && f <= 1e6f) {
+ v = f;
+ } else {
+ GGML_LOG_WARN("%s: ignoring invalid GGML_METAL_N_MAIN='%s'\n", __func__, env);
+ }
+ }
+ }
+ return v;
+}
+