File size: 20,216 Bytes
3eb68da | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 | {% set precastOperands = precastOperands | default(false) %}
{% set precastScalar = precastScalar | default("f32") %}
enable subgroups;
{% if pinSubgroupSize32 %}
enable subgroup_size_control;
{% endif %}
enable chromium_experimental_subgroup_matrix;
diagnostic(off, chromium.subgroup_matrix_uniformity);
{{ env.wgsl.resourceDeclarations }}
// Rare-boundary fallback: reproduce the reference's binary64 scale division
// and accumulator product using integer significands; no f64 feature required.
fn quant_u96_add(a: vec3<u32>, b: vec3<u32>) -> vec3<u32> {
let low = a.x + b.x;
let middle = a.y + b.y;
let with_carry = middle + u32(low < a.x);
return vec3<u32>(low, with_carry, a.z + b.z + u32(middle < a.y) + u32(with_carry < middle));
}
fn quant_u96_shr(a: vec3<u32>) -> vec3<u32> {
return vec3<u32>((a.x >> 1u) | (a.y << 31u), (a.y >> 1u) | (a.z << 31u), a.z >> 1u);
}
fn quant_u96_round_shift(value: vec3<u32>, shift: u32) -> vec3<u32> {
var q = value;
var sticky = false;
var guard = false;
for (var bit = 0u; bit < shift; bit++) {
sticky = sticky || guard;
guard = (q.x & 1u) != 0u;
q = quant_u96_shr(q);
}
if (guard && (sticky || (q.x & 1u) != 0u)) {
q = quant_u96_add(q, vec3<u32>(1u, 0u, 0u));
}
return q;
}
fn quant_float_significand(bits: u32) -> u32 {
return (bits & 0x7fffffu) | select(0x800000u, 0u, (bits & 0x7f800000u) == 0u);
}
fn quant_float_exponent(bits: u32) -> i32 {
return max(1, i32((bits >> 23u) & 255u)) - 150;
}
fn quant_round_binary64(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
if (acc == 0) { return 0; }
let a_bits = bitcast<u32>(a_s);
let b_bits = bitcast<u32>(b_s);
let y_bits = bitcast<u32>(y_s);
let a = quant_float_significand(a_bits);
let b = quant_float_significand(b_bits);
let denominator = quant_float_significand(y_bits);
if (a == 0u || b == 0u) { return 0; }
// The exact product of two 24-bit significands occupies at most 48 bits.
let p0 = (a & 65535u) * (b & 65535u);
let cross = (a >> 16u) * (b & 65535u) + (b >> 16u) * (a & 65535u);
let product_low = p0 + (cross << 16u);
let product_high = (a >> 16u) * (b >> 16u) + (cross >> 16u) + u32(product_low < p0);
var input_bit = select(31 - i32(countLeadingZeros(product_low)), 63 - i32(countLeadingZeros(product_high)), product_high != 0u);
var remainder = 0u;
var significant_bits = 0u;
var leading_exponent = 0;
var significand = vec2<u32>(0u);
// Long division generates the 53 bits of the binary64 multiplier. The
// remainder fits in 25 bits, including its left shift. At most 24 leading
// zero bits precede the first significant bit.
loop {
var incoming = 0u;
if (input_bit >= 32) { incoming = (product_high >> u32(input_bit - 32)) & 1u; }
else if (input_bit >= 0) { incoming = (product_low >> u32(input_bit)) & 1u; }
remainder = (remainder << 1u) | incoming;
let digit = u32(remainder >= denominator);
if (digit != 0u) { remainder -= denominator; }
if (significant_bits != 0u || digit != 0u) {
if (significant_bits == 0u) { leading_exponent = input_bit; }
significand = vec2<u32>((significand.x << 1u) | digit, (significand.y << 1u) | (significand.x >> 31u));
significant_bits++;
}
input_bit--;
if (significant_bits == 53u) { break; }
}
let twice_remainder = remainder << 1u;
if (twice_remainder > denominator || (twice_remainder == denominator && (significand.x & 1u) != 0u)) {
let low = significand.x + 1u;
significand = vec2<u32>(low, significand.y + u32(low == 0u));
}
let scale_exponent = quant_float_exponent(a_bits) + quant_float_exponent(b_bits) - quant_float_exponent(y_bits) + leading_exponent - 52;
// abs(INT_MIN) is representable as u32. Multiply the rounded multiplier's
// significand by the exact accumulator into three little-endian u32 limbs.
var magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
var operand = vec3<u32>(significand, 0u);
var product = vec3<u32>(0u);
loop {
if ((magnitude & 1u) != 0u) { product = quant_u96_add(product, operand); }
magnitude >>= 1u;
if (magnitude == 0u) { break; }
operand = vec3<u32>(operand.x << 1u, (operand.y << 1u) | (operand.x >> 31u), (operand.z << 1u) | (operand.y >> 31u));
}
var product_top = 31 - i32(countLeadingZeros(product.x));
if (product.y != 0u) { product_top = 63 - i32(countLeadingZeros(product.y)); }
if (product.z != 0u) { product_top = 95 - i32(countLeadingZeros(product.z)); }
if (product_top + scale_exponent >= 10) { return select(1024, -1024, acc < 0); }
if (product_top + scale_exponent < -2) { return 0; }
let product_shift = u32(max(0, product_top - 52));
let rounded_product = quant_u96_round_shift(product, product_shift);
let result_exponent = scale_exponent + i32(product_shift);
var rounded = rounded_product;
if (result_exponent < 0) { rounded = quant_u96_round_shift(rounded_product, u32(-result_exponent)); }
else { rounded.x <<= u32(result_exponent); }
return select(i32(rounded.x), -i32(rounded.x), acc < 0);
}
// A power-of-two division and an exactly represented <=48-bit product need
// no binary64 boundary resolver. This includes ordinary dyadic quantization.
fn quant_exact_dyadic_product(acc: i32, scale: f32) -> bool {
let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
let significand = quant_float_significand(bitcast<u32>(scale));
let integer_bits = 32u - countLeadingZeros(magnitude) - countTrailingZeros(magnitude);
let scale_bits = 32u - countLeadingZeros(significand) - countTrailingZeros(significand);
return integer_bits + scale_bits <= 48u;
}
fn quant_normal_expansion_range(v: f32) -> bool {
let exponent = (bitcast<u32>(v) >> 23u) & 255u;
// Reserve two binary32 significands plus four carry/error bits above the
// minimum normal exponent, so residual arithmetic avoids FTZ underflow.
return exponent >= (1u + 2u * 24u + 4u) && exponent < 255u;
}
fn round_expansion_ratio(numerator_hi: f32, numerator_lo: f32, denominator: f32, dyadic_scale: f32, dyadic_candidate: bool, acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
let estimate = numerator_hi / denominator;
// Anything outside this range saturates after adding an 8-bit zero point;
// returning early also keeps floor/i32 conversion away from Inf and overflow.
if (estimate <= -1024.0) { return -1024; }
if (estimate >= 1024.0) { return 1024; }
let lo = floor(estimate);
let hi = lo + 1.0;
let midpoint = lo + 0.5;
let midpoint_hi = fma(midpoint, denominator, 0.0);
if ((bitcast<u32>(midpoint_hi) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
let midpoint_lo = fma(midpoint, denominator, -midpoint_hi);
// The high-word subtraction is exact for these similarly sized products
// (Sterbenz); adding the residual difference therefore preserves its sign
// even when the rounded high words straddle adjacent f32 values.
let midpoint_delta = fma(1.0, numerator_hi - midpoint_hi, numerator_lo - midpoint_lo);
// Escalate ambiguous expansion signs instead of changing the tie tolerance.
// Six FMA/product tails and their additions fit within this conservative
// 16*u^2 error envelope (u=2^-24), including binary64 reference rounding.
if ((bitcast<u32>(midpoint_delta) & 0x7f800000u) == 0x7f800000u) { return quant_round_binary64(acc, a_s, b_s, y_s); }
if (abs(midpoint_delta) <= max(abs(numerator_hi), abs(midpoint_hi)) * 0x1p-44) {
if (!dyadic_candidate || !quant_exact_dyadic_product(acc, dyadic_scale)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
}
if (midpoint_delta < 0.0) {
return i32(lo);
}
if (midpoint_delta > 0.0) {
return i32(hi);
}
let half_lo = floor(lo * 0.5);
let lo_is_even = lo - half_lo * 2.0 == 0.0;
return i32(select(hi, lo, lo_is_even));
}
// Multiplication by a power of two keeps every i32 bit in binary64. Round the
// exact dyadic value directly, including INT_MIN, half-even ties and saturation.
fn quant_round_power_two(acc: i32, exponent: i32) -> i32 {
let magnitude = select(u32(acc), 0u - u32(acc), acc < 0);
var rounded = 0u;
if (exponent >= 0) {
if (exponent >= 10) { return select(1024, -1024, acc < 0); }
rounded = min(magnitude, 1024u >> u32(exponent)) << u32(exponent);
} else {
let shift = u32(-exponent);
if (shift >= 32u) { return 0; }
let whole = magnitude >> shift;
let remainder = magnitude & ((1u << shift) - 1u);
let half = 1u << (shift - 1u);
rounded = min(1024u, whole + u32(remainder > half || (remainder == half && (whole & 1u) != 0u)));
}
return select(i32(rounded), -i32(rounded), acc < 0);
}
fn round_requantized_precise(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
if (acc == 0) { return 0; }
let a_bits = bitcast<u32>(a_s);
let b_bits = bitcast<u32>(b_s);
let y_bits = bitcast<u32>(y_s);
if (((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u && min(a_bits, min(b_bits, y_bits)) != 0u && max(a_bits, max(b_bits, y_bits)) < 0x7f800000u) {
let exponent = i32(a_bits >> 23u) + i32(b_bits >> 23u) - i32(y_bits >> 23u) - 127;
return quant_round_power_two(acc, exponent);
}
let acc_f = f32(acc);
// Every i32 high part aligned to 256 is exactly representable in f32.
// Recover the conversion residual without an out-of-range i32(acc_f)
// conversion when INT_MAX rounds up to 2^31.
let acc_tail = fma(-1.0, acc_f - f32(acc & -256), f32(acc & 255));
// Equal b/y scales cancel exactly. Bypass their product and division so a
// rounded intermediate cannot move a near-half value onto the tie.
if (bitcast<u32>(b_s) == bitcast<u32>(y_s)) {
let direct_hi = fma(acc_f, a_s, 0.0);
let direct_lo = fma(acc_tail, a_s, fma(acc_f, a_s, -direct_hi));
if (!quant_normal_expansion_range(direct_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
return round_expansion_ratio(direct_hi, direct_lo, 1.0, a_s, true, acc, a_s, b_s, y_s);
}
// Keep the scale product as a two-float expansion. Its FMA residual preserves
// which side of a rounding midpoint a near-half value occupies.
let ab_hi = fma(a_s, b_s, 0.0);
let ab_lo = fma(a_s, b_s, -ab_hi);
if (!quant_normal_expansion_range(ab_hi) || !quant_normal_expansion_range(y_s) || y_s > 0x1p126) { return quant_round_binary64(acc, a_s, b_s, y_s); }
let product_hi = fma(acc_f, ab_hi, 0.0);
if (!quant_normal_expansion_range(product_hi)) { return quant_round_binary64(acc, a_s, b_s, y_s); }
let rounded_acc_tail = fma(acc_f, ab_hi, -product_hi) + fma(acc_f, ab_lo, 0.0);
let product_tail = fma(acc_tail, ab_hi, rounded_acc_tail) + fma(acc_tail, ab_lo, 0.0);
// Compare the expansion directly. Renormalizing it adds a cancelling
// subtraction/addition chain that can erase the residual during lowering.
let denominator_bits = bitcast<u32>(y_s);
let denominator_power_two = (denominator_bits & 0x7fffffu) == 0u;
let dyadic_candidate = denominator_power_two && ab_lo == 0.0;
return round_expansion_ratio(product_hi, product_tail, y_s, ab_hi, dyadic_candidate, acc, a_s, b_s, y_s);
}
// A conservative binary32 error filter accepts only a result safely inside one
// integer rounding bin. Ambiguous boundaries keep the full expansion/binary64
// path; this is an exact-result shortcut, not an approximate quantization mode.
fn round_requantized(acc: i32, a_s: f32, b_s: f32, y_s: f32) -> i32 {
if (acc == 0) { return 0; }
let a_bits = bitcast<u32>(a_s);
let b_bits = bitcast<u32>(b_s);
let y_bits = bitcast<u32>(y_s);
let ea = (a_bits >> 23u) & 255u;
let eb = (b_bits >> 23u) & 255u;
let ey = (y_bits >> 23u) & 255u;
// Certify the range before arithmetic: overflow need not produce infinity.
// Any nonzero i32 converts to a float with biased exponent <=158. These
// scale-only bounds cover every accumulator and can be shared across lanes.
// Reserve two significands plus carry bits above FTZ, and a factor of two
// below overflow for products, division and the expansion midpoint product.
if (((a_bits | b_bits | y_bits) & 0x80000000u) != 0u ||
min(ea, eb) == 0u || max(ea, eb) >= 255u ||
ea + eb < 180u || ea + eb > 347u ||
ey < 53u || ey > 242u || ea + eb > ey + 220u) {
return quant_round_binary64(acc, a_s, b_s, y_s);
}
let scale = fma(a_s, b_s, 0.0);
let product = fma(f32(acc), scale, 0.0);
let estimate = product / y_s;
if (estimate <= -1024.0) { return -1024; }
if (estimate >= 1024.0) { return 1024; }
let nearest = round(estimate);
// 32 binary32 unit roundoffs reserve room for the i32 conversion, two
// products, quotient, and distance arithmetic, plus binary64 rounding.
let margin = max(1.0, abs(estimate)) * 0x1p-19;
if (abs(estimate - nearest) < 0.5 - margin) { return i32(nearest); }
{% if device.adapterInfo.vendor != "apple" %}
// A rounded half-integer estimate can hide which side the exact f32-scale
// ratio lies on. Resolve that one-ulp neighborhood conservatively outside
// the compact fast path; exact power-of-two ratios already have a cheap
// and correct path in round_requantized_precise.
if (abs(abs(estimate - nearest) - 0.5) <= max(1.0, abs(estimate)) * 0x1p-24) {
let dyadic_scales = ((a_bits | b_bits | y_bits) & 0x807fffffu) == 0u &&
min(a_bits, min(b_bits, y_bits)) != 0u &&
max(a_bits, max(b_bits, y_bits)) < 0x7f800000u;
if (!dyadic_scales) { return quant_round_binary64(acc, a_s, b_s, y_s); }
}
{% endif %}
return round_requantized_precise(acc, a_s, b_s, y_s);
}
// Adjusted byte integers fit exactly in f32. Each bounded f32 dot product
// has absolute magnitude at most 2^24; int32 adds then preserve wraparound.
const TILE_M: u32 = {{ sgmatTileM }}u;
const TILE_N: u32 = {{ sgmatTileN }}u;
const TILE_K: u32 = {{ sgmatTileK }}u;
{% if precastOperands %}
const PADDED_K: u32 = {{ preparedK }}u;
const PADDED_N: u32 = {{ preparedN }}u;
var<workgroup> matrix_out: array<f32, {{ sgmatSubgroups * 8 * 64 }}>;
{% else %}
var<workgroup> tile_A: array<f32, {{ sgmatTileM * sgmatTileK }}>;
var<workgroup> tile_B: array<f32, {{ sgmatTileK * sgmatTileN }}>;
{% if not sgmatReuseStaging %}
var<workgroup> scratch: array<array<array<f32, 64>, 4>, {{ sgmatSubgroups }}>;
{% endif %}
{% endif %}
@compute @workgroup_size({{ sgmatWorkgroup }}){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }}
fn main(@builtin(workgroup_id) wid: vec3<u32>,
{% if not precastOperands or not matrixSubgroupId %}
@builtin(local_invocation_index) tid: u32,
{% endif %}
@builtin(subgroup_invocation_id) lane: u32,
@builtin(subgroup_id) subgroup_index: u32) {
let subgroup = subgroup_index;
let base_A = (subgroup % {{ sgmatRowGroups }}u) * 16u;
let base_B = (subgroup / {{ sgmatRowGroups }}u) * 32u;
let row = lane / 4u;
let col = (lane % 4u) * 2u;
// Bound adjusted byte magnitudes from scalar zero points. A column-wise
// B zero point keeps the full byte-difference bound for the whole tile.
let max_a = {% if matrixZeroPerRow %}255u{% else %}u32(max(abs({{ 0 if aUnsigned else (0 - 128) }} - i32(a_zero_point[0])), abs({{ 255 if aUnsigned else 127 }} - i32(a_zero_point[0])))){% endif %};
let max_b = {% if hasBZero and not bZeroPerColumn %}u32(max(abs({{ 0 if bUnsigned else (0 - 128) }} - i32(b_zero_point[0])), abs({{ 255 if bUnsigned else 127 }} - i32(b_zero_point[0])))){% else %}{{ 255 if hasBZero or bUnsigned else 128 }}u{% endif %};
let partial_k = (16777216u / (max_a * max_b) / TILE_K) * TILE_K;
var totals: array<i32, 16>;
for (var partial = 0u; partial < params.K; partial += partial_k) {
{% for r in range(2) %}{% for c in range(4) %}
var matC{{ r }}{{ c }}: subgroup_matrix_result<f32, 8, 8>;
{% endfor %}{% endfor %}
let partial_end = min(params.K, partial + partial_k);
for (var kb = partial; kb < partial_end; kb += TILE_K) {
{% if not precastOperands %}
for (var index = tid; index < TILE_M * TILE_K; index += {{ sgmatWorkgroup }}u) {
let m = wid.y * TILE_M + index / TILE_K;
let k = kb + index % TILE_K;
var value = f32(0.0);
if (m < params.M && k < params.K) {
value = f32(i32(a[m * params.K + k]) - i32(a_zero_point[{{ "m" if matrixZeroPerRow else "0" }}]));
}
tile_A[index] = value;
}
for (var index = tid; index < TILE_K * TILE_N; index += {{ sgmatWorkgroup }}u) {
let k = kb + index / TILE_N;
let n = wid.x * TILE_N + index % TILE_N;
var value = f32(0.0);
if (k < params.K && n < params.N) {
value = f32(i32(b[{% if matrixBatchB %}wid.z * params.bBatchStride + {% endif %}k * params.N + n]) - i32(b_zero_point[{{ "n" if bZeroPerColumn else "0" }}]));
}
tile_B[index] = value;
}
workgroupBarrier();
{% endif %}
for (var step = 0u; step < TILE_K; step += 8u) {
{% for r in range(2) %}
let matA{{ r }} = subgroupMatrixLoad<subgroup_matrix_left<{{ precastScalar if precastOperands else "f32" }}, 8, 8>, row_major>(&{{ "a" if precastOperands else "tile_A" }}, {% if precastOperands %}(wid.y * TILE_M + base_A + {{ r * 8 }}u) * PADDED_K + kb + step{% else %}(base_A + {{ r * 8 }}u) * TILE_K + step{% endif %}, {{ "PADDED_K" if precastOperands else "TILE_K" }});
{% endfor %}
{% for c in range(4) %}
let matB{{ c }} = subgroupMatrixLoad<subgroup_matrix_right<{{ precastScalar if precastOperands else "f32" }}, 8, 8>, row_major>(&{{ "b" if precastOperands else "tile_B" }}, {% if precastOperands %}{% if matrixBatchB %}wid.z * PADDED_K * PADDED_N + {% endif %}(kb + step) * PADDED_N + wid.x * TILE_N + base_B + {{ c * 8 }}u{% else %}step * TILE_N + base_B + {{ c * 8 }}u{% endif %}, {{ "PADDED_N" if precastOperands else "TILE_N" }});
{% endfor %}
{% for r in range(2) %}{% for c in range(4) %}
matC{{ r }}{{ c }} = subgroupMatrixMultiplyAccumulate(matA{{ r }}, matB{{ c }}, matC{{ r }}{{ c }});
{% endfor %}{% endfor %}
}
{% if not precastOperands %}
workgroupBarrier();
{% endif %}
}
{% if precastOperands %}
// The prepared operands need no staging barriers. Publish all eight
// matrices together before converting the bounded partial to int32.
{% for r in range(2) %}{% for c in range(4) %}
subgroupMatrixStore<row_major>(&matrix_out, subgroup * 512u + {{ r * 256 + c * 64 }}u, matC{{ r }}{{ c }}, 8u);
{% endfor %}{% endfor %}
workgroupBarrier();
{% for r in range(2) %}{% for c in range(4) %}{% for h in range(2) %}
totals[{{ r * 8 + c * 2 + h }}] += i32(matrix_out[subgroup * 512u + {{ r * 256 + c * 64 }}u + row * 8u + col + {{ h }}u]);
{% endfor %}{% endfor %}{% endfor %}
workgroupBarrier();
{% else %}
{% if sgmatReuseStaging %}
// All staged A reads have completed. Reuse its storage for one result band
// at a time: each subgroup needs 4 * 64 floats, exactly its A-tile share.
{% endif %}
{% for r in range(2) %}
{% for c in range(4) %}
subgroupMatrixStore<row_major>({% if sgmatReuseStaging %}&tile_A{% else %}&scratch[subgroup][{{ c }}]{% endif %}, {% if sgmatReuseStaging %}subgroup * 256u + {{ c * 64 }}u{% else %}0u{% endif %}, matC{{ r }}{{ c }}, 8u);
{% endfor %}
workgroupBarrier();
{% for c in range(4) %}{% for h in range(2) %}
totals[{{ r * 8 + c * 2 + h }}] += i32({% if sgmatReuseStaging %}tile_A[subgroup * 256u + {{ c * 64 }}u + row * 8u + col + {{ h }}u]{% else %}scratch[subgroup][{{ c }}][row * 8u + col + {{ h }}u]{% endif %});
{% endfor %}{% endfor %}
workgroupBarrier();
{% endfor %}
{% endif %}
}
{% for r in range(2) %}{% for c in range(4) %}{% for h in range(2) %}
{
let m = wid.y * TILE_M + base_A + {{ r * 8 }}u + row;
let n = wid.x * TILE_N + base_B + {{ c * 8 + h }}u + col;
if (m < params.M && n < params.N) {
let acc = totals[{{ r * 8 + c * 2 + h }}]{% if matrixHasBias %} + bias[{{ "m" if matrixBiasPerRow else "n" }}]{% endif %};
let rounded = round_requantized(acc, f32(a_scale[0]), f32(b_scale[{{ "m" if matrixScalePerRow else ("n" if bScalePerColumn else "0") }}]), f32(y_scale[0]));
let q = clamp(rounded + i32(y_zero_point[0]), {{ qMin }}, {{ qMax }});
y[{% if matrixBatchB %}wid.z * params.yBatchStride + {% endif %}m * params.N + n] = {{ "u32(q)" if yUnsigned else "q" }};
}
}
{% endfor %}{% endfor %}{% endfor %}
}
|