enable subgroups; {{ env.wgsl.resourceDeclarations }} // Subgroup-per-row online normalization over a contiguous last axis of // moderate width. One SUBGROUP owns each row and a workgroup carries as many // rows as it has subgroups, so the (max, denom) merge is one pair of subgroup // collectives: no workgroup memory, no barriers. Each lane keeps its vec4 words // in registers between the statistics sweep and the normalizing store, so the // row is read from memory once. Each lane forms an online (maximum, // denominator) pair, and subgroup collectives merge those pairs. const WG: u32 = {{ workgroupSize }}u; // FLT_MAX, not -inf, as the online (m, d) accumulator init: merges must keep // `m - m` finite so an empty lane / all--inf row contributes the exact // accumulator identity (m, d) = (-FLT_MAX, 0). Operator epilogues interpret // a zero final denominator according to their public semantics. Using -inf // here changes +inf-row behavior. const FLT_MAX: f32 = 3.4028234663852886e38; fn is_finite_f32(value: f32) -> bool { return select(false, value <= FLT_MAX, value >= -FLT_MAX); } // x - m that is exactly 0 when x equals a finite m, so exp(shifted) == 1 // exactly at the row max. `x - x` on an infinite max is a legal fast-math // fold to 0, which would silently turn +inf rows finite — the explicit // equality test keeps the NaN propagation of the serial kernels. fn shifted_value(value: f32, maxValue: f32) -> f32 { let equalFiniteMax = select(false, value == maxValue, is_finite_f32(maxValue)); return select(value - maxValue, 0.0, equalFiniteMax); } fn exp_shift(value: f32, maxValue: f32) -> f32 { return exp(shifted_value(value, maxValue)); } var sgLaneClaims: atomic; struct SubgroupLogicalLanes { ltid: u32, ord: u32, rank: u32, count: u32, } fn subgroup_logical_lanes(rank: u32, count: u32) -> SubgroupLogicalLanes { // Every lane performs the atomic so no collective follows a lane guard: a // subgroup op after a closed `if (rank == 0u)` block is a reconvergence // hazard, since every subgroup operation must be reached in uniform control // flow. Only the lane-0 lane adds its subgroup's claim, every other lane // adds 0 and discards its snapshot. Lane 0 is the lowest lane, which is the // lane `subgroupBroadcastFirst` reads. let ticket = atomicAdd(&sgLaneClaims, select(0u, count | (1u << 16u), rank == 0u)); let claim = subgroupBroadcastFirst(ticket); return SubgroupLogicalLanes((claim & 0xffffu) + rank, claim >> 16u, rank, count); } @compute @workgroup_size({{ workgroupSize }}) fn main(@builtin(workgroup_id) wid: vec3, @builtin(subgroup_invocation_id) sgLaneId: u32, @builtin(subgroup_size) sgSize: u32) { // Rows tile the folded workgroup grid; the lanes of one subgroup share a row. // Which subgroup owns which row, and which lane holds which columns, come from // the logical lane coordinates, so no assumption is made about which // invocations share a subgroup. let L = subgroup_logical_lanes(sgLaneId, sgSize); let sgLane = L.rank; let rowsPerWorkgroup = WG / sgSize; let row = (wid.x + wid.y * {{ DISPATCH_FOLD_WIDTH }}u) * rowsPerWorkgroup + L.ord; // A subgroup past the last row runs the collectives on an empty row and // stores nothing, so every collective stays in uniform control flow. let rowValid = row < params.rows; let vecLimit = select(0u, params.vecCols, rowValid); let rowBase = row * params.vecCols; {% if isInnerBroadcast %} let biasBase = (row / {{ innerRepeat }}u) * params.vecCols; {% else %} let biasBase = (row % {{ biasBlockCountSpec }}u) * params.vecCols; {% endif %} // Lane-local online (m, d) over the lane's strided words; f32 accumulation. var m = -FLT_MAX; var d = 0.0; {% for i in range(vecsPerLane) %} var v{{ i }} = vec4(0.0); let c{{ i }} = sgLane + {{ i }}u * sgSize; if (c{{ i }} < vecLimit) { v{{ i }} = vec4(x[rowBase + c{{ i }}]) + vec4(bias[biasBase + c{{ i }}]); let mNew{{ i }} = max(m, max(max(v{{ i }}.x, v{{ i }}.y), max(v{{ i }}.z, v{{ i }}.w))); d = d * exp_shift(m, mNew{{ i }}) + exp_shift(v{{ i }}.x, mNew{{ i }}) + exp_shift(v{{ i }}.y, mNew{{ i }}) + exp_shift(v{{ i }}.z, mNew{{ i }}) + exp_shift(v{{ i }}.w, mNew{{ i }}); m = mNew{{ i }}; } {% endfor %} // Subgroup merge: a lane with no words contributes d == 0 (exact identity). let finalM = subgroupMax(m); let finalD = subgroupAdd(d * exp_shift(m, finalM)); // BiasSoftmax treats an entirely additive-masked row as zero. let inv = select(0.0, 1.0 / finalD, finalD > 0.0); {% for i in range(vecsPerLane) %} if (c{{ i }} < vecLimit) { let o{{ i }} = vec4( exp_shift(v{{ i }}.x, finalM), exp_shift(v{{ i }}.y, finalM), exp_shift(v{{ i }}.z, finalM), exp_shift(v{{ i }}.w, finalM)) * inv; y[rowBase + c{{ i }}] = vec4<{{ scalar }}>(o{{ i }}); } {% endfor %} }