com.microsoft.BiasSoftmax / build /webgpu /softmax-subgroup-rows.wgsl.jinja
Xenova's picture
Xenova HF Staff
sync 6fdf6301e2bb
66f0d7a verified
Raw History Blame
4.92 kB
enable subgroups;
{{ env.wgsl.resourceDeclarations }}
// Subgroup-per-row online normalization over a contiguous last axis of
// moderate width. One SUBGROUP owns each row and a workgroup carries as many
// rows as it has subgroups, so the (max, denom) merge is one pair of subgroup
// collectives: no workgroup memory, no barriers. Each lane keeps its vec4 words
// in registers between the statistics sweep and the normalizing store, so the
// row is read from memory once. Each lane forms an online (maximum,
// denominator) pair, and subgroup collectives merge those pairs.
const WG: u32 = {{ workgroupSize }}u;
// FLT_MAX, not -inf, as the online (m, d) accumulator init: merges must keep
// `m - m` finite so an empty lane / all--inf row contributes the exact
// accumulator identity (m, d) = (-FLT_MAX, 0). Operator epilogues interpret
// a zero final denominator according to their public semantics. Using -inf
// here changes +inf-row behavior.
const FLT_MAX: f32 = 3.4028234663852886e38;
fn is_finite_f32(value: f32) -> bool {
return select(false, value <= FLT_MAX, value >= -FLT_MAX);
}
// x - m that is exactly 0 when x equals a finite m, so exp(shifted) == 1
// exactly at the row max. `x - x` on an infinite max is a legal fast-math
// fold to 0, which would silently turn +inf rows finite — the explicit
// equality test keeps the NaN propagation of the serial kernels.
fn shifted_value(value: f32, maxValue: f32) -> f32 {
let equalFiniteMax = select(false, value == maxValue, is_finite_f32(maxValue));
return select(value - maxValue, 0.0, equalFiniteMax);
}
fn exp_shift(value: f32, maxValue: f32) -> f32 {
return exp(shifted_value(value, maxValue));
}
var<workgroup> sgLaneClaims: atomic<u32>;
struct SubgroupLogicalLanes {
ltid: u32,
ord: u32,
rank: u32,
count: u32,
}
fn subgroup_logical_lanes(rank: u32, count: u32) -> SubgroupLogicalLanes {
// Every lane performs the atomic so no collective follows a lane guard: a
// subgroup op after a closed `if (rank == 0u)` block is a reconvergence
// hazard, since every subgroup operation must be reached in uniform control
// flow. Only the lane-0 lane adds its subgroup's claim, every other lane
// adds 0 and discards its snapshot. Lane 0 is the lowest lane, which is the
// lane `subgroupBroadcastFirst` reads.
let ticket = atomicAdd(&sgLaneClaims, select(0u, count | (1u << 16u), rank == 0u));
let claim = subgroupBroadcastFirst(ticket);
return SubgroupLogicalLanes((claim & 0xffffu) + rank, claim >> 16u, rank, count);
}
@compute @workgroup_size({{ workgroupSize }})
fn main(@builtin(workgroup_id) wid: vec3<u32>,
@builtin(subgroup_invocation_id) sgLaneId: u32,
@builtin(subgroup_size) sgSize: u32) {
// Rows tile the folded workgroup grid; the lanes of one subgroup share a row.
// Which subgroup owns which row, and which lane holds which columns, come from
// the logical lane coordinates, so no assumption is made about which
// invocations share a subgroup.
let L = subgroup_logical_lanes(sgLaneId, sgSize);
let sgLane = L.rank;
let rowsPerWorkgroup = WG / sgSize;
let row = (wid.x + wid.y * {{ DISPATCH_FOLD_WIDTH }}u) * rowsPerWorkgroup + L.ord;
// A subgroup past the last row runs the collectives on an empty row and
// stores nothing, so every collective stays in uniform control flow.
let rowValid = row < params.rows;
let vecLimit = select(0u, params.vecCols, rowValid);
let rowBase = row * params.vecCols;
{% if isInnerBroadcast %}
let biasBase = (row / {{ innerRepeat }}u) * params.vecCols;
{% else %}
let biasBase = (row % {{ biasBlockCountSpec }}u) * params.vecCols;
{% endif %}
// Lane-local online (m, d) over the lane's strided words; f32 accumulation.
var m = -FLT_MAX;
var d = 0.0;
{% for i in range(vecsPerLane) %}
var v{{ i }} = vec4<f32>(0.0);
let c{{ i }} = sgLane + {{ i }}u * sgSize;
if (c{{ i }} < vecLimit) {
v{{ i }} = vec4<f32>(x[rowBase + c{{ i }}]) + vec4<f32>(bias[biasBase + c{{ i }}]);
let mNew{{ i }} = max(m, max(max(v{{ i }}.x, v{{ i }}.y), max(v{{ i }}.z, v{{ i }}.w)));
d = d * exp_shift(m, mNew{{ i }})
+ exp_shift(v{{ i }}.x, mNew{{ i }}) + exp_shift(v{{ i }}.y, mNew{{ i }})
+ exp_shift(v{{ i }}.z, mNew{{ i }}) + exp_shift(v{{ i }}.w, mNew{{ i }});
m = mNew{{ i }};
}
{% endfor %}
// Subgroup merge: a lane with no words contributes d == 0 (exact identity).
let finalM = subgroupMax(m);
let finalD = subgroupAdd(d * exp_shift(m, finalM));
// BiasSoftmax treats an entirely additive-masked row as zero.
let inv = select(0.0, 1.0 / finalD, finalD > 0.0);
{% for i in range(vecsPerLane) %}
if (c{{ i }} < vecLimit) {
let o{{ i }} = vec4<f32>(
exp_shift(v{{ i }}.x, finalM), exp_shift(v{{ i }}.y, finalM),
exp_shift(v{{ i }}.z, finalM), exp_shift(v{{ i }}.w, finalM)) * inv;
y[rowBase + c{{ i }}] = vec4<{{ scalar }}>(o{{ i }});
}
{% endfor %}
}