Download build/webgpu/softmax-subgroup-rows.wgsl.jinja from webgpu-kernels/com.microsoft.BiasSoftmax: direct link, hf CLI and curl.
- Browser
- Download file 4.92 kB
-
https://huggingface.co/kernels/webgpu-kernels/com.microsoft.BiasSoftmax/resolve/v1/build/webgpu/softmax-subgroup-rows.wgsl.jinja
- Command line
-
hf download hf://webgpu-kernels/com.microsoft.BiasSoftmax@v1/build/webgpu/softmax-subgroup-rows.wgsl.jinja
-
curl -L -o softmax-subgroup-rows.wgsl.jinja https://huggingface.co/kernels/webgpu-kernels/com.microsoft.BiasSoftmax/resolve/v1/build/webgpu/softmax-subgroup-rows.wgsl.jinja
4.92 kB
| enable subgroups; | |
| {{ env.wgsl.resourceDeclarations }} | |
| // Subgroup-per-row online normalization over a contiguous last axis of | |
| // moderate width. One SUBGROUP owns each row and a workgroup carries as many | |
| // rows as it has subgroups, so the (max, denom) merge is one pair of subgroup | |
| // collectives: no workgroup memory, no barriers. Each lane keeps its vec4 words | |
| // in registers between the statistics sweep and the normalizing store, so the | |
| // row is read from memory once. Each lane forms an online (maximum, | |
| // denominator) pair, and subgroup collectives merge those pairs. | |
| const WG: u32 = {{ workgroupSize }}u; | |
| // FLT_MAX, not -inf, as the online (m, d) accumulator init: merges must keep | |
| // `m - m` finite so an empty lane / all--inf row contributes the exact | |
| // accumulator identity (m, d) = (-FLT_MAX, 0). Operator epilogues interpret | |
| // a zero final denominator according to their public semantics. Using -inf | |
| // here changes +inf-row behavior. | |
| const FLT_MAX: f32 = 3.4028234663852886e38; | |
| fn is_finite_f32(value: f32) -> bool { | |
| return select(false, value <= FLT_MAX, value >= -FLT_MAX); | |
| } | |
| // x - m that is exactly 0 when x equals a finite m, so exp(shifted) == 1 | |
| // exactly at the row max. `x - x` on an infinite max is a legal fast-math | |
| // fold to 0, which would silently turn +inf rows finite — the explicit | |
| // equality test keeps the NaN propagation of the serial kernels. | |
| fn shifted_value(value: f32, maxValue: f32) -> f32 { | |
| let equalFiniteMax = select(false, value == maxValue, is_finite_f32(maxValue)); | |
| return select(value - maxValue, 0.0, equalFiniteMax); | |
| } | |
| fn exp_shift(value: f32, maxValue: f32) -> f32 { | |
| return exp(shifted_value(value, maxValue)); | |
| } | |
| var<workgroup> sgLaneClaims: atomic<u32>; | |
| struct SubgroupLogicalLanes { | |
| ltid: u32, | |
| ord: u32, | |
| rank: u32, | |
| count: u32, | |
| } | |
| fn subgroup_logical_lanes(rank: u32, count: u32) -> SubgroupLogicalLanes { | |
| // Every lane performs the atomic so no collective follows a lane guard: a | |
| // subgroup op after a closed `if (rank == 0u)` block is a reconvergence | |
| // hazard, since every subgroup operation must be reached in uniform control | |
| // flow. Only the lane-0 lane adds its subgroup's claim, every other lane | |
| // adds 0 and discards its snapshot. Lane 0 is the lowest lane, which is the | |
| // lane `subgroupBroadcastFirst` reads. | |
| let ticket = atomicAdd(&sgLaneClaims, select(0u, count | (1u << 16u), rank == 0u)); | |
| let claim = subgroupBroadcastFirst(ticket); | |
| return SubgroupLogicalLanes((claim & 0xffffu) + rank, claim >> 16u, rank, count); | |
| } | |
| @compute @workgroup_size({{ workgroupSize }}) | |
| fn main(@builtin(workgroup_id) wid: vec3<u32>, | |
| @builtin(subgroup_invocation_id) sgLaneId: u32, | |
| @builtin(subgroup_size) sgSize: u32) { | |
| // Rows tile the folded workgroup grid; the lanes of one subgroup share a row. | |
| // Which subgroup owns which row, and which lane holds which columns, come from | |
| // the logical lane coordinates, so no assumption is made about which | |
| // invocations share a subgroup. | |
| let L = subgroup_logical_lanes(sgLaneId, sgSize); | |
| let sgLane = L.rank; | |
| let rowsPerWorkgroup = WG / sgSize; | |
| let row = (wid.x + wid.y * {{ DISPATCH_FOLD_WIDTH }}u) * rowsPerWorkgroup + L.ord; | |
| // A subgroup past the last row runs the collectives on an empty row and | |
| // stores nothing, so every collective stays in uniform control flow. | |
| let rowValid = row < params.rows; | |
| let vecLimit = select(0u, params.vecCols, rowValid); | |
| let rowBase = row * params.vecCols; | |
| {% if isInnerBroadcast %} | |
| let biasBase = (row / {{ innerRepeat }}u) * params.vecCols; | |
| {% else %} | |
| let biasBase = (row % {{ biasBlockCountSpec }}u) * params.vecCols; | |
| {% endif %} | |
| // Lane-local online (m, d) over the lane's strided words; f32 accumulation. | |
| var m = -FLT_MAX; | |
| var d = 0.0; | |
| {% for i in range(vecsPerLane) %} | |
| var v{{ i }} = vec4<f32>(0.0); | |
| let c{{ i }} = sgLane + {{ i }}u * sgSize; | |
| if (c{{ i }} < vecLimit) { | |
| v{{ i }} = vec4<f32>(x[rowBase + c{{ i }}]) + vec4<f32>(bias[biasBase + c{{ i }}]); | |
| let mNew{{ i }} = max(m, max(max(v{{ i }}.x, v{{ i }}.y), max(v{{ i }}.z, v{{ i }}.w))); | |
| d = d * exp_shift(m, mNew{{ i }}) | |
| + exp_shift(v{{ i }}.x, mNew{{ i }}) + exp_shift(v{{ i }}.y, mNew{{ i }}) | |
| + exp_shift(v{{ i }}.z, mNew{{ i }}) + exp_shift(v{{ i }}.w, mNew{{ i }}); | |
| m = mNew{{ i }}; | |
| } | |
| {% endfor %} | |
| // Subgroup merge: a lane with no words contributes d == 0 (exact identity). | |
| let finalM = subgroupMax(m); | |
| let finalD = subgroupAdd(d * exp_shift(m, finalM)); | |
| // BiasSoftmax treats an entirely additive-masked row as zero. | |
| let inv = select(0.0, 1.0 / finalD, finalD > 0.0); | |
| {% for i in range(vecsPerLane) %} | |
| if (c{{ i }} < vecLimit) { | |
| let o{{ i }} = vec4<f32>( | |
| exp_shift(v{{ i }}.x, finalM), exp_shift(v{{ i }}.y, finalM), | |
| exp_shift(v{{ i }}.z, finalM), exp_shift(v{{ i }}.w, finalM)) * inv; | |
| y[rowBase + c{{ i }}] = vec4<{{ scalar }}>(o{{ i }}); | |
| } | |
| {% endfor %} | |
| } | |