{{ env.wgsl.resourceDeclarations }} {% if prepareSpatialA or prepareSpatialB %} {% macro spatial_constants() %} const KERNEL_H: u32 = {{ kernelHSpec }}u; const KERNEL_W: u32 = {{ kernelWSpec }}u; const KERNEL_HW: u32 = KERNEL_H * KERNEL_W; const KSIZE: u32 = KERNEL_HW; const STRIDE_H: u32 = {{ strideHSpec }}u; const STRIDE_W: u32 = {{ strideWSpec }}u; const DILATION_H: u32 = {{ dilationHSpec }}u; const DILATION_W: u32 = {{ dilationWSpec }}u; const PAD_TOP: i32 = {{ padTopSpec }}; const PAD_LEFT: i32 = {{ padLeftSpec }}; {% endmacro %} {% macro spatial_sample(value="value", input="x", zero="x_zero_point", sampleK="k", sampleScalar="auto") %} let ic = {{ sampleK }} / KSIZE; let kq = {{ sampleK }} - ic * KSIZE; let kh = kq / KERNEL_W; let kw = kq - kh * KERNEL_W; let oh = pos / params.outW; let ow = pos - oh * params.outW; let ih = i32(oh * STRIDE_H + kh * DILATION_H) - PAD_TOP; let iw = i32(ow * STRIDE_W + kw * DILATION_W) - PAD_LEFT; var {{ value }}: {{ bScalar if sampleScalar == "auto" else sampleScalar }} = {{ zero }}[0]; if (ih >= 0 && ih < i32(params.inH) && iw >= 0 && iw < i32(params.inW)) { {{ value }} = {{ input }}[{% if sampleChannelsLast is defined and sampleChannelsLast %}((batch * params.inH + u32(ih)) * params.inW + u32(iw)) * params.inChannels + ic{% else %}((batch * params.inChannels + ic) * params.inH + u32(ih)) * params.inW + u32(iw){% endif %}]; } {% endmacro %} {{ spatial_constants() }} {% endif %} @compute @workgroup_size({{ prepareWorkgroup }}) fn main(@builtin(workgroup_id) wid: vec3, @builtin(local_invocation_index) tid: u32) { let offset = ((wid.y * params.groupsX + wid.x) * {{ prepareWorkgroup }}u + tid) * 4u; if (offset >= params.count) { return; } if (offset < params.aCount) { let m = offset / params.paddedK; let k = offset % params.paddedK; {% for h in range(4) %} { var value = 0.0; if (m < params.M && k + {{ h }}u < params.K) { {% if prepareSpatialA %} let batch = m / params.outCount; let pos = m % params.outCount; let sourceK = k + {{ h }}u; {{ spatial_sample("sample", "a", "a_zero_point", "sourceK", aScalar) }} value = f32(i32(sample) - i32(a_zero_point[0])); {% else %} value = f32(i32(a[m * params.K + k + {{ h }}u]) - i32(a_zero_point[{{ "m" if prepareZeroPerRow else "0" }}])); {% endif %} } aFloat[offset + {{ h }}u] = {% if precastScalar == "f16" %}f16(value){% else %}value{% endif %}; } {% endfor %} } else { let bOffset = offset - params.aCount; {% if prepareBatchB %} let batch = bOffset / (params.paddedK * params.paddedN); let localOffset = bOffset % (params.paddedK * params.paddedN); {% endif %} let k = {{ "localOffset" if prepareBatchB else "bOffset" }} / params.paddedN; let n = {{ "localOffset" if prepareBatchB else "bOffset" }} % params.paddedN; {% for h in range(4) %} { var value = 0.0; if (k < params.K && n + {{ h }}u < params.N) { {% if prepareSpatialB %} let pos = n + {{ h }}u; {{ spatial_sample("sample", "b", "b_zero_point") }} value = f32(i32(sample) - i32(b_zero_point[0])); {% else %} value = f32(i32(b[{% if prepareTransposeB %}(n + {{ h }}u) * params.K + k{% else %}{% if prepareBatchB %}batch * params.K * params.N + {% endif %}k * params.N + n + {{ h }}u{% endif %}]) - i32(b_zero_point[{% if bZeroPerColumn %}n + {{ h }}u{% else %}0{% endif %}])); {% endif %} } bFloat[bOffset + {{ h }}u] = {% if precastScalar == "f16" %}f16(value){% else %}value{% endif %}; } {% endfor %} } }