File size: 6,260 Bytes
77f8a08 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | {{ env.wgsl.resourceDeclarations }}
// Rotary positional embedding. The dispatch pins one slice on z and folds that slice's own
// rotation groups across x and y, so every row of a slice shares one base offset and all the
// head geometry below is a compile-time constant: every division in the index math is by a
// literal rather than by a uniform.
//
// A slice is one batch at rank 3, where a token's heads are adjacent and a workgroup covers a
// whole token, so it reads each cos/sin row once. At rank 4 a token's heads are a sequence
// apart, so a slice is instead one batch and ITEMS consecutive heads: the invocation gathers
// its cos/sin row once and rotates the same rotation group in each of those heads.
const WG: u32 = {{ workgroupSize }}u;
const ITEMS: u32 = {{ itemsPerLane }}u;
// Rotation groups per head row, the row's stride, and the number of groups that rotate, all
// counted in the storage element the bindings use: a vec4 on the four-pair route and a scalar
// on the single-pair route.
const HEAD_UNITS: u32 = {{ headUnits }}u;
const HEAD_STRIDE: u32 = {{ headStride }}u;
const ROT_UNITS: u32 = {{ rotUnits }}u;
{% if hasTail %}
// First element past the rotary window.
const ROT_END: u32 = {{ rotStart }}u;
{% endif %}
const NUM_HEADS: u32 = {{ numHeads }}u;
{% if rank == 4 %}
const HEAD_BLOCKS: u32 = {{ headBlocks }}u;
{% endif %}
{% if hasTail %}
{% macro copy_tail(base) %}
let tail = ROT_END + (group - ROT_UNITS) * 2u;
y[{{ base }} + tail] = x[{{ base }} + tail];
{% if tailGuard %}
// An odd head size leaves the last group with a single element to copy.
if (tail + 1u < HEAD_STRIDE) {
y[{{ base }} + tail + 1u] = x[{{ base }} + tail + 1u];
}
{% else %}
y[{{ base }} + tail + 1u] = x[{{ base }} + tail + 1u];
{% endif %}
{% endmacro %}
{% endif %}
{% macro rotate(base) %}
{% if interleaved and useVec4 %}
// Four interleaved pairs span two vec4s: (w0.x, w0.y), (w0.z, w0.w), (w1.x, w1.y),
// (w1.z, w1.w). Deinterleave, rotate, and reinterleave on the way back out.
let pairBase = {{ base }} + group * 2u;
let w0 = vec4<f32>(x[pairBase]);
let w1 = vec4<f32>(x[pairBase + 1u]);
let a = vec4<f32>(w0.x, w0.z, w1.x, w1.z);
let b = vec4<f32>(w0.y, w0.w, w1.y, w1.w);
let ra = a * cf - b * sf;
let rb = a * sf + b * cf;
y[pairBase] = {{ castOut }}(vec4<f32>(ra.x, rb.x, ra.y, rb.y));
y[pairBase + 1u] = {{ castOut }}(vec4<f32>(ra.z, rb.z, ra.w, rb.w));
{% elif interleaved %}
// Adjacent elements form the rotated pair.
let pairBase = {{ base }} + group * 2u;
let a = {{ castIn }}(x[pairBase]);
let b = {{ castIn }}(x[pairBase + 1u]);
y[pairBase] = {{ castOut }}(a * cf - b * sf);
y[pairBase + 1u] = {{ castOut }}(a * sf + b * cf);
{% else %}
// Split halves: ROT_UNITS is both the rotating group count and the offset from an
// element of the first half to its partner in the second.
let a = {{ castIn }}(x[{{ base }} + group]);
let b = {{ castIn }}(x[{{ base }} + ROT_UNITS + group]);
y[{{ base }} + group] = {{ castOut }}(a * cf - b * sf);
y[{{ base }} + ROT_UNITS + group] = {{ castOut }}(a * sf + b * cf);
{% endif %}
{% endmacro %}
@compute @workgroup_size(WG)
fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) {
{% if rank == 3 %}
// wg.y carries the high bits of the group index past the per-axis dispatch fold width.
let slice = wg.z;
let sliceUnits = params.sliceRows * HEAD_UNITS;
let sliceBase = slice * params.sliceRows * HEAD_STRIDE;
let laneUnit = (wg.x + wg.y * {{ DISPATCH_FOLD_WIDTH }}u) * (WG * ITEMS) + lid.x;
{% for item in range(itemsPerLane) %}
{
let unit = laneUnit + {{ item }}u * WG;
if (unit < sliceUnits) {
let group = unit % HEAD_UNITS;
let row = unit / HEAD_UNITS;
let base = sliceBase + row * HEAD_STRIDE;
let token = row / NUM_HEADS;
{% if hasTail %}
if (group >= ROT_UNITS) {
{{ copy_tail("base") }}
} else {
{% endif %}
{% if posOffset %}
// One base offset for the whole request: token s reads cache row p[0] + s.
let position = position_ids[0] + token;
{% else %}
// The (batch_size, sequence_length) table. params.sliceRows / NUM_HEADS is the sequence
// length, so this slice's table row is slice * sequence_length + token.
let position = position_ids[slice * (params.sliceRows / NUM_HEADS) + token];
{% endif %}
let cache = position * ROT_UNITS + group;
let cf = {{ castIn }}(cos_cache[cache]);
let sf = {{ castIn }}(sin_cache[cache]);
{{ rotate("base") }}
{% if hasTail %}
}
{% endif %}
}
}
{% endfor %}
{% else %}
// A rank-4 slice is one batch and ITEMS consecutive heads; wg.y carries the high bits of
// the group index past the per-axis dispatch fold width.
let slice = wg.z;
let headBase = slice % HEAD_BLOCKS * ITEMS;
let batch = slice / HEAD_BLOCKS;
let sliceUnits = params.sliceRows * HEAD_UNITS;
let unit = (wg.x + wg.y * {{ DISPATCH_FOLD_WIDTH }}u) * WG + lid.x;
if (unit >= sliceUnits) {
return;
}
let group = unit % HEAD_UNITS;
let token = unit / HEAD_UNITS;
let rowBase = (batch * NUM_HEADS + headBase) * params.sliceRows + token;
{% if hasTail %}
if (group >= ROT_UNITS) {
{% for item in range(itemsPerLane) %}
{
let base = (rowBase + {{ item }}u * params.sliceRows) * HEAD_STRIDE;
{{ copy_tail("base") }}
}
{% endfor %}
return;
}
{% endif %}
{% if posOffset %}
// One base offset for the whole request: token s reads cache row p[0] + s.
let position = position_ids[0] + token;
{% else %}
// The (batch_size, sequence_length) table; params.sliceRows is the sequence length.
let position = position_ids[batch * params.sliceRows + token];
{% endif %}
// Gathered once for the whole head block: cos and sin depend on the token and the rotation
// group, never on the head.
let cache = position * ROT_UNITS + group;
let cf = {{ castIn }}(cos_cache[cache]);
let sf = {{ castIn }}(sin_cache[cache]);
{% for item in range(itemsPerLane) %}
{
let base = (rowBase + {{ item }}u * params.sliceRows) * HEAD_STRIDE;
{{ rotate("base") }}
}
{% endfor %}
{% endif %}
}
|