com.microsoft.QLinearConv / build /webgpu /quant-exact-prepare.wgsl.jinja
Xenova's picture
Xenova HF Staff
sync 6fdf6301e2bb
3eb68da verified
Raw History Blame
3.58 kB
{{ env.wgsl.resourceDeclarations }}
{% if prepareSpatialA or prepareSpatialB %}
{% macro spatial_constants() %}
const KERNEL_H: u32 = {{ kernelHSpec }}u;
const KERNEL_W: u32 = {{ kernelWSpec }}u;
const KERNEL_HW: u32 = KERNEL_H * KERNEL_W;
const KSIZE: u32 = KERNEL_HW;
const STRIDE_H: u32 = {{ strideHSpec }}u;
const STRIDE_W: u32 = {{ strideWSpec }}u;
const DILATION_H: u32 = {{ dilationHSpec }}u;
const DILATION_W: u32 = {{ dilationWSpec }}u;
const PAD_TOP: i32 = {{ padTopSpec }};
const PAD_LEFT: i32 = {{ padLeftSpec }};
{% endmacro %}
{% macro spatial_sample(value="value", input="x", zero="x_zero_point", sampleK="k", sampleScalar="auto") %}
let ic = {{ sampleK }} / KSIZE;
let kq = {{ sampleK }} - ic * KSIZE;
let kh = kq / KERNEL_W;
let kw = kq - kh * KERNEL_W;
let oh = pos / params.outW;
let ow = pos - oh * params.outW;
let ih = i32(oh * STRIDE_H + kh * DILATION_H) - PAD_TOP;
let iw = i32(ow * STRIDE_W + kw * DILATION_W) - PAD_LEFT;
var {{ value }}: {{ bScalar if sampleScalar == "auto" else sampleScalar }} = {{ zero }}[0];
if (ih >= 0 && ih < i32(params.inH) && iw >= 0 && iw < i32(params.inW)) {
{{ value }} = {{ input }}[{% if sampleChannelsLast is defined and sampleChannelsLast %}((batch * params.inH + u32(ih)) * params.inW + u32(iw)) * params.inChannels + ic{% else %}((batch * params.inChannels + ic) * params.inH + u32(ih)) * params.inW + u32(iw){% endif %}];
}
{% endmacro %}
{{ spatial_constants() }}
{% endif %}
@compute @workgroup_size({{ prepareWorkgroup }})
fn main(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
let offset = ((wid.y * params.groupsX + wid.x) * {{ prepareWorkgroup }}u + tid) * 4u;
if (offset >= params.count) { return; }
if (offset < params.aCount) {
let m = offset / params.paddedK;
let k = offset % params.paddedK;
{% for h in range(4) %}
{
var value = 0.0;
if (m < params.M && k + {{ h }}u < params.K) {
{% if prepareSpatialA %}
let batch = m / params.outCount;
let pos = m % params.outCount;
let sourceK = k + {{ h }}u;
{{ spatial_sample("sample", "a", "a_zero_point", "sourceK", aScalar) }}
value = f32(i32(sample) - i32(a_zero_point[0]));
{% else %}
value = f32(i32(a[m * params.K + k + {{ h }}u]) - i32(a_zero_point[{{ "m" if prepareZeroPerRow else "0" }}]));
{% endif %}
}
aFloat[offset + {{ h }}u] = {% if precastScalar == "f16" %}f16(value){% else %}value{% endif %};
}
{% endfor %}
} else {
let bOffset = offset - params.aCount;
{% if prepareBatchB %}
let batch = bOffset / (params.paddedK * params.paddedN);
let localOffset = bOffset % (params.paddedK * params.paddedN);
{% endif %}
let k = {{ "localOffset" if prepareBatchB else "bOffset" }} / params.paddedN;
let n = {{ "localOffset" if prepareBatchB else "bOffset" }} % params.paddedN;
{% for h in range(4) %}
{
var value = 0.0;
if (k < params.K && n + {{ h }}u < params.N) {
{% if prepareSpatialB %}
let pos = n + {{ h }}u;
{{ spatial_sample("sample", "b", "b_zero_point") }}
value = f32(i32(sample) - i32(b_zero_point[0]));
{% else %}
value = f32(i32(b[{% if prepareTransposeB %}(n + {{ h }}u) * params.K + k{% else %}{% if prepareBatchB %}batch * params.K * params.N + {% endif %}k * params.N + n + {{ h }}u{% endif %}]) - i32(b_zero_point[{% if bZeroPerColumn %}n + {{ h }}u{% else %}0{% endif %}]));
{% endif %}
}
bFloat[bOffset + {{ h }}u] = {% if precastScalar == "f16" %}f16(value){% else %}value{% endif %};
}
{% endfor %}
}
}