// Lfm3OmniForConditionalGeneration WebGPU library (model hints: lfm3omni). Import { default } or the named class from this file.
function Yd(e,t){let r=Math.min(e.length,t.length),n=0;for(;n<r&&e[n]===t[n];)n+=1;return n}var $3=["a","b"];function wu(e){let t=0;for(;t<e.length&&e[t].role==="system";)t+=1;return e.slice(0,t)}function Jd(e,t){return e.rewindableLengthAtMost?.(t)??(t>0&&(e.canTruncateTo?.(t)??!0)?t:0)}function ep(e,t,r){if(r.length===0)return[];try{let[n,a]=$3.map(s=>e.encode(t([...r,{role:"user",content:s}]),{add_special_tokens:!1}).ids);return n.slice(0,Yd(n,a))}catch{return[]}}function Fr(e,t,r){return t.length===0?"":e.decode(t,r)}function Xd(e,t,r,n=[]){for(let s=n.length;s>=0;--s){let o=[...r,...n.slice(0,s)];if(o.length===0)continue;let i=Fr(e,o,{skip_special_tokens:!1});if(!t.startsWith(i))continue;let u=e.encode(t.slice(i.length),{add_special_tokens:!1}).ids,l=[...o,...u];if(Fr(e,l,{skip_special_tokens:!1})===t)return{fullyEncodedIds:l,inputIds:l,prefixLength:r.length}}let a=e.encode(t,{add_special_tokens:!1}).ids;return{fullyEncodedIds:a,inputIds:a,prefixLength:Yd(r,a)}}var Xn=class{#e;#t=[];#n=[];#r=[];constructor(t){this.#e=t}get cacheLength(){return this.#t.length}prepare(t,r){if(r===void 0&&this.#r.length===0)return{...Xd(this.#e,t,this.#t,this.#n),imageSpans:[]};let n=r?.images??[],{ids:a,placeholders:s}=H3(this.#t,this.#r),o=Xd(this.#e,t,a,this.#n),i=o.prefixLength;for(let d=0;d<this.#r.length;++d){let p=this.#r[d],f=n[d];if(f===void 0||f.digest!==p.digest||f.tokens!==p.tokens||tp(f,r)!==p.tokenId){i=Math.min(i,s[d]);break}}let u=d=>V3(d,r),l=u(o.inputIds);return{fullyEncodedIds:(o.inputIds===o.fullyEncodedIds?l:u(o.fullyEncodedIds)).ids,inputIds:l.ids,prefixLength:Z3(a,i,this.#r,s),imageSpans:l.spans}}commit(t,r,n,a=[]){let s=[...t,...r];if(!Number.isInteger(n)||n<t.length||n>s.length)throw new Error(`chat token ledger cannot commit cache length ${n} for ${t.length} input + ${r.length} generated tokens`);this.#t=s.slice(0,n),this.#n=s.slice(n),this.#r=a.filter(o=>o.start+o.tokens<=n)}rewind(t){if(!Number.isInteger(t)||t<0||t>this.#t.length)throw new Error(`chat token ledger cannot rewind to ${t} of ${this.#t.length} cached tokens`);let r=this.#r.find(n=>n.start<t&&t<n.start+n.tokens);if(r)throw new Error(`chat token ledger cannot rewind to ${t}: it splits the image at rows ${r.start}..${r.start+r.tokens-1}`);this.#t=this.#t.slice(0,t),this.#n=[],this.#r=this.#r.filter(n=>n.start<t)}clear(){this.#t=[],this.#n=[],this.#r=[]}};function H3(e,t){let r=[],n=[],a=0;for(let s of t){for(;a<s.start;++a)r.push(e[a]);n.push(r.length),r.push(e[s.start]),a=s.start+s.tokens}for(;a<e.length;++a)r.push(e[a]);return{ids:r,placeholders:n}}function tp(e,t){return e.tokenId??t.imageTokenId}function V3(e,t){let r=t?.images??[],n=new Set(t?t.placeholderTokenIds??[t.imageTokenId]:[]),a=[],s=[];for(let o of e){if(!n.has(o)){a.push(o);continue}let i=r[s.length];if(i===void 0)throw new Error(`prompt holds more media placeholders than its ${r.length} item(s)`);let u=tp(i,t);if(u!==o)throw new Error(`media item ${s.length} expands token ${u}, but its placeholder is ${o}`);s.push({start:a.length,tokens:i.tokens,digest:i.digest,image:s.length,tokenId:u});for(let l=0;l<i.tokens;++l)a.push(o)}if(s.length!==r.length)throw new Error(`prompt holds ${s.length} media placeholder(s) but ${r.length} item(s) were given`);return{ids:a,spans:s}}function Z3(e,t,r,n){let a=Math.min(t,e.length);for(let s=0;s<r.length&&n[s]<t;++s)a+=r[s].tokens-1;return a}var rp=Object.freeze({reasoningOpen:"<think>",reasoningClose:"</think>"});function X3(e,t){return e?.assistantContent?.(t)??t.visibleText}function np(e,t){return{...e?.assistantMessageFields?.(t)??{},role:"assistant",content:X3(e,t)}}function ap(e){return e.replaceAll("\\x00","\0")}function mn(e){return typeof e=="number"&&Number.isFinite(e)}function Sr(e){return e instanceof Error?e.message:String(e)}function Ws(e){return e.byteOffset===0&&e.byteLength===e.buffer.byteLength&&e.buffer instanceof ArrayBuffer?e.buffer:e.slice().buffer}function Ma(e,t){if(e.length===0)return new Uint8Array(0);if(e.length===1&&e[0].byteLength===t)return e[0];let r=new Uint8Array(t),n=0;for(let a of e)r.set(a,n),n+=a.byteLength;return r}function Wa(e,t){let r=new t(e.reduce((a,s)=>a+s.length,0)),n=0;for(let a of e)r.set(a,n),n+=a.length;return r}function sp(e){let t=e.length;for(;t>0&&e[t-1]==="\uFFFD";)t--;return t}function op(e){let t=[],r="",n=0,a=0,s="",o=0,i=()=>e(t.slice(a)),u=c=>{if(!c.startsWith(s))return"";let d=c.slice(s.length);return s=c,r+=d,d},l=()=>{if(t.length-a<=6)return;let c=t.length-2,d=e(t.slice(c)),p=()=>d===""||d.startsWith("\uFFFD");for(;c>a+1&&p();)c-=1,d=e(t.slice(c));p()||(a=c,s=d,o=sp(d))};return{push(c){t.push(c);let d=i(),p=sp(d),f=p!==d.length;n=!f||p>o?0:n+1,o=p;let m=!f||n>3?d:d.slice(0,p),h=u(m);return s===d&&l(),h},flush(){if(t.length===0)return"";let c=e(t);if(n=0,a=0,c.startsWith(r)){let f=c.slice(r.length);return r=s=c,f}let d=0;for(;d<r.length&&d<c.length&&r[d]===c[d];)d+=1;let p=c.slice(d);return r=s=c,p},get text(){return r}}}var ip=/^<function=([^>\n]+)>([\s\S]*)<\/function>$/,bu=/<parameter=([^>\n]+)>\n?/y,up="</parameter>",Y3=/^<function\s+name="([^"]*)"\s*>([\s\S]*?)(?:<\/function>)?$/,vu=/<param\s+name="([^"]*)"\s*>/y,Fa="</param>",Qa="<![CDATA[";var J3=[{open:"<tool_call>",close:"</tool_call>",inMarkup:!1},{open:"<function",close:"</function>",inMarkup:!0}],cp=Object.freeze([{type:"function",function:{name:"webgpu_delimiter_probe",description:"probe",parameters:{type:"object",properties:{}}}}]);function dp(e,t){let r=J3.flatMap(({open:a,close:s,inMarkup:o})=>{let i=t(a),u=t(s);return typeof i=="number"&&typeof u=="number"?[{text:a,open:i,close:u,inMarkup:o}]:[]}),n=(e===null?void 0:r.find(a=>e.includes(a.text)))??r.find(a=>!a.inMarkup);return n?{open:n.open,close:n.close,inMarkup:n.inMarkup}:null}function ew(e){let t=e.trim();if(t.startsWith("{"))return nw(t);if(/^<function\s/.test(t))return tw(t);let r=ip.exec(t)??(t.startsWith("<function=")?ip.exec(`${t}
</function>`):null);if(!r)return{error:'the call is neither <function=...> nor <function name="..."> markup, nor a JSON object'};let n=r[2],a=[],s=0;for(;;){for(;s<n.length&&/\s/.test(n[s]);)s+=1;if(s===n.length)break;bu.lastIndex=s;let o=bu.exec(n);if(!o)return{error:`unexpected text in the call: ${JSON.stringify(n.slice(s,s+40))}`};let i=bu.lastIndex,u=n.indexOf(up,i),l=n.indexOf("<parameter=",i);if(u>=0&&(l<0||u<l)){a.push([o[1].trim(),{text:n.slice(i,u).replace(/\n$/,"")}]),s=u+up.length;continue}let c=l>=0?l:n.length;a.push([o[1].trim(),{text:n.slice(i,c).replace(/\s+$/,"")}]),s=c}return{name:r[1].trim(),arguments:a}}function tw(e){let t=Y3.exec(e);if(!t)return{error:`the call's <function> tag does not parse: ${JSON.stringify(e.slice(0,60))}`};let r=t[2],n=[],a=0;for(;;){for(;a<r.length&&/\s/.test(r[a]);)a+=1;if(a===r.length)break;vu.lastIndex=a;let s=vu.exec(r);if(!s)return{error:`unexpected text in the call: ${JSON.stringify(r.slice(a,a+40))}`};let o=vu.lastIndex;if(r.startsWith(Qa,o)){let c=r.indexOf("]]>",o+Qa.length);if(c>=0){for(n.push([s[1],{text:r.slice(o+Qa.length,c)}]),a=c+3;a<r.length&&/\s/.test(r[a]);)a+=1;r.startsWith(Fa,a)&&(a+=Fa.length);continue}n.push([s[1],{text:r.slice(o+Qa.length).replace(/(?:\]\]?)?\s*$/,"")}]);break}let i=r.indexOf(Fa,o),u=r.indexOf("<param",o);if(i>=0&&(u<0||i<u)){n.push([s[1],{text:r.slice(o,i)}]),a=i+Fa.length;continue}let l=u>=0?u:r.length;n.push([s[1],{text:r.slice(o,l).replace(/\s+$/,"")}]),a=l}return{name:t[1].trim(),arguments:n}}function rw(e){let t=Object.entries(e.function.arguments??{}).map(([r,n])=>{let a=typeof n!="string"?JSON.stringify(n):/[<&\n]/.test(n)?`${Qa}${n}]]>`:n;return`<param name="${r}">${a}${Fa}`});return`<function name="${e.function.name}">${t.join("")}</function>`}function pp(e){let t=e.tool_calls;if(e.role!=="assistant"||typeof e.content!="string"||!Array.isArray(t)||t.length===0)return e;let{tool_calls:r,...n}=e,a=e.content,s=t.map((o,i)=>(i>0||a?`
`:"")+rw(o)).join("");return{...n,content:a+s}}function nw(e){let t;try{t=JSON.parse(e)}catch{return{error:"the call's JSON does not parse"}}let r=t;if(typeof r!="object"||r===null||typeof r.name!="string")return{error:`the call's JSON has no string "name"`};let n=r.arguments??{};return typeof n!="object"||n===null||Array.isArray(n)?{error:`the call's "arguments" is not an object`}:{name:r.name,arguments:Object.entries(n).map(([a,s])=>[a,{value:s}])}}var aw=/^-?(?:0|[1-9]\d*)$/,sw=/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/;function lp(e,t){switch(e){case"string":return{value:t};case"integer":return aw.test(t)&&Number.isSafeInteger(Number(t))?{value:Number(t)}:void 0;case"number":return sw.test(t)?{value:Number(t)}:void 0;case"boolean":return t==="true"?{value:!0}:t==="false"?{value:!1}:void 0;case"null":return t==="null"?{value:null}:void 0;case"object":case"array":try{let r=JSON.parse(t);return fp(e,r)?{value:r}:void 0}catch{return}default:return}}function fp(e,t){switch(e){case"string":return typeof t=="string";case"integer":return typeof t=="number"&&Number.isSafeInteger(t);case"number":return typeof t=="number"&&Number.isFinite(t);case"boolean":return typeof t=="boolean";case"null":return t===null;case"object":return typeof t=="object"&&t!==null&&!Array.isArray(t);case"array":return Array.isArray(t);default:return!1}}function ow(e){return e.type===void 0?null:typeof e.type=="string"?[e.type]:e.type}function iw(e,t,r){let n=ow(r),a;if("text"in t)if(n===null)a=t.text;else{let s=n.map(o=>lp(o,t.text)).filter(o=>o!==void 0);if(s.length===0)return{error:`argument "${e}": ${JSON.stringify(t.text)} is not ${n.join(" or ")}`};if(s.length>1&&n.includes("string"))return{error:`argument "${e}": ${JSON.stringify(t.text)} could be ${n.filter(o=>lp(o,t.text)).join(" or ")}`};a=s[0].value}else if(a=t.value,n!==null&&!n.some(s=>fp(s,a)))return{error:`argument "${e}": ${JSON.stringify(a)} is not ${n.join(" or ")}`};return r.enum&&!r.enum.some(s=>uw(s,a))?{error:`argument "${e}": ${JSON.stringify(a)} is not one of ${r.enum.map(s=>JSON.stringify(s)).join(", ")}`}:{value:a}}function uw(e,t){return JSON.stringify(e)===JSON.stringify(t)}var _u=0;function lw(){return _u=(_u+1)%16777216,`call_${_u.toString(36)}${Math.floor(Math.random()*4294967296).toString(36)}`}function mp(e,t){let r=lw(),n=ew(e);if("error"in n)return{id:r,type:"function",function:{name:"",arguments:{}},errors:[n.error]};let a=[],s=t.find(l=>l.function.name===n.name);s||a.push(`no tool named "${n.name}" was offered`);let o=s?.function.parameters??{},i=o.properties??{},u={};for(let[l,c]of n.arguments){if(Object.hasOwn(u,l)){a.push(`argument "${l}" is given twice`);continue}let d=Object.hasOwn(i,l)?i[l]:void 0,p=o.additionalProperties,f=d??(typeof p=="object"?p:p===!0?{}:void 0);if(s&&f===void 0){a.push(`argument "${l}" is not a parameter of "${n.name}"`),u[l]="text"in c?c.text:c.value;continue}let m=iw(l,c,f??{});"error"in m?(a.push(m.error),u[l]="text"in c?c.text:c.value):u[l]=m.value}for(let l of o.required??[])Object.hasOwn(u,l)||a.push(`required argument "${l}" is missing`);return{id:r,type:"function",function:{name:n.name,arguments:u},...a.length>0?{errors:a}:{}}}function yu(e){if(!Array.isArray(e))throw new TypeError('tools must be an array of { type: "function", function: { name, parameters } }');let t=new Set;for(let r of e){let n=r?.function;if(r?.type!=="function"||typeof n?.name!="string"||n.name==="")throw new TypeError('each tool must be { type: "function", function: { name, description?, parameters? } }');if(t.has(n.name))throw new TypeError(`tool "${n.name}" is offered twice`);t.add(n.name)}}function gp(e,t){let r=ku(e.minimumCapacity??0,"minimumCapacity"),n=e.maxLength??e.max_length;if(n!==void 0){if(n<r)throw new Error(`maxLength ${n} is smaller than minimumCapacity ${r}`);return n}return Math.max(t,r)}function ku(e,t){if(typeof e!="number"||!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer, got ${String(e)}`);return e}function hp(e){return ku(e??0,"max_new_tokens")}function ja(e,t){let r=ku(e,"input length")+hp(t);if(!Number.isSafeInteger(r))throw new Error(`generation capacity exceeds the safe integer range: ${r}`);return r}var Au=64;function Qr(e){if(!e.do_sample)return null;let t=e.temperature??1,r=e.top_k??50,n=e.top_p??1,a=e.min_p??0;if(typeof t!="number"||!Number.isFinite(t)||t<0)throw new RangeError(`temperature must be a finite number >= 0, got ${String(t)}`);if(!Number.isInteger(r)||r<1||r>Au)throw new RangeError(`top_k must be an integer from 1 to ${Au} (sampling draws among at most ${Au} candidates), got ${String(r)}`);if(typeof n!="number"||!(n>0&&n<=1))throw new RangeError(`top_p must be in (0, 1], got ${String(n)}`);if(typeof a!="number"||!(a>=0&&a<=1))throw new RangeError(`min_p must be in [0, 1], got ${String(a)}`);if(e.seed!==void 0&&!Number.isSafeInteger(e.seed))throw new RangeError(`seed must be an integer, got ${String(e.seed)}`);if(t===0||r===1)return null;let s=e.seed===void 0?Math.floor(Math.random()*4294967296):e.seed;return{top_k:r,temperature:t,top_p:n,min_p:a,seed:Number(BigInt.asUintN(32,BigInt(s)))}}function Yn(e,t={}){if(e.images!==void 0&&e.images.length>0)throw new Error("this model does not accept images; load its vision-language wrapper to generate with them");return{maxNewTokens:hp(e.max_new_tokens),eosTokenId:e.eos_token_id??t.eosTokenId??null,stopOnEos:e.stop_on_eos!==!1,onPrefillDone:e.on_prefill_done??null,onPrefillProgress:e.on_prefill_progress??null,signal:e.signal??null}}function Jn(e,t){return t==null?!1:Array.isArray(t)?t.includes(e):e===t}function gn(e){if(e instanceof Uint32Array)return e;if(ArrayBuffer.isView(e)&&Object.prototype.toString.call(e)==="[object Uint32Array]")return Uint32Array.from(e);if(!Array.isArray(e))throw new Error("input_ids must be a token row or a batch containing one row");let t=e.length>0&&Array.isArray(e[0]);if(t&&e.length!==1)throw new Error(`input_ids supports batch size 1, got ${e.length} rows`);let r=t?e[0]:e;for(let n=0;n<r.length;++n){let a=r[n];if(typeof a!="number"||!Number.isSafeInteger(a)||a<0||a>4294967295)throw new Error(`input_ids[${n}] must be a uint32 integer, got ${String(a)}`)}return Uint32Array.from(r)}var cw=3,dw=256,ea=class extends Error{required;capacity;constructor(t,r){super(`the conversation needs ${t} tokens (prompt plus one reply token) but the context holds ${r}`),this.name="ContextOverflowError",this.required=t,this.capacity=r}},hn=class extends Error{constructor(t){super(t),this.name="BusyError"}},ta=class extends Error{reason;constructor(t,r,n){super(`the GPU device was lost (${t})${r?`: ${r}`:""}`,n),this.name="DeviceLostError",this.reason=t}};function Su(e,t){if(typeof e!="number"||!Number.isSafeInteger(e)||e<1)throw new RangeError(`${t} must be a positive integer, got ${String(e)}`);return e}var Fs=class{#e=!1;#t="";text="";push(t){let r=t;if(!this.#e){if(r=r.replace(/^\s+/,""),r==="")return"";this.#e=!0}let n=this.#t+r,a=/\s+$/.exec(n),s=a?n.slice(0,a.index):n;return this.#t=a?a[0]:"",this.text+=s,s}};function bp(e,t){let r=null;for(let n of t){let a=e.indexOf(n);a!==-1&&(r===null||a<r.at||a===r.at&&n.length>r.needle.length)&&(r={at:a,needle:n})}return r}function vp(e,t){let r=0;for(let n of t)for(let a=Math.min(n.length-1,e.length);a>r;--a)if(e.endsWith(n.slice(0,a))){r=a;break}return r}var Qs=class{#e;#t="";matched=!1;constructor(t){this.#e=t}push(t){if(this.matched)return"";let r=this.#t+t,n=bp(r,this.#e);if(n!==null)return this.matched=!0,this.#t="",r.slice(0,n.at);let a=vp(r,this.#e);return this.#t=r.slice(r.length-a),r.slice(0,r.length-a)}flush(){let t=this.#t;return this.#t="",t}},wp=new WeakMap;function pw(e,t){let r=wp.get(e);r||wp.set(e,r=new Map);let n=r.get(t);if(n===void 0){let a=Fr(e,[t],{skip_special_tokens:!1});n=a!==""&&Fr(e,[t],{skip_special_tokens:!0})===""?a:null,r.set(t,n)}return n}var Tu=class{#e;#t;#n;#r;#a;#s;#l;#u;#o;#i=null;calls=[];#c=this.#m();#d=[];reasoning=new Fs;answer=new Fs;#p;#f;reasoningTokens=0;thought;constructor(t){this.#l=t.special,this.#u=t.decodeRaw;let r=t.markup;this.#e=r?.reasoningOpen??null,this.#t=r?[r.reasoningOpen,...r.answerHeaders??[]]:[],this.#n=r?[r.reasoningClose]:[],this.#r=t.calls?.open??null,this.#a=t.calls?.close??null,this.#s=t.calls?.inMarkup??!1,this.#o=t.startInReasoning?"reasoning":"text",this.thought=t.startInReasoning,this.#p=new Qs(t.stopStrings),this.#f=new Qs(t.stopStrings)}closeOpenCall(){this.#i!==null&&(this.calls.push(this.#i.length>0?this.#u(this.#i):""),this.#i=null)}get stopped(){return this.#p.matched||this.#f.matched}#m(){return op(t=>this.#u(t))}push(t){if(this.#i!==null)return t===this.#a?(this.#s&&this.#i.push(t),this.calls.push(this.#i.length>0?this.#u(this.#i):""),this.#i=null):this.#i.push(t),[];if(t===this.#r&&this.#o==="text"){let s=this.#h(!0);return this.#i=this.#s?[t]:[],s}let r=this.#o,n=this.#l(t);n!==null?(this.#g(this.#c.flush(),!1),this.#c=this.#m(),this.#g(n,!0)):this.#g(this.#c.push(t),!1);let a=this.#h(!1);return(r==="reasoning"||this.#o==="reasoning")&&(this.reasoningTokens+=1),a}finish(){return this.#g(this.#c.flush(),!1),[...this.#h(!0),...this.#b("reasoning"),...this.#b("text")]}#g(t,r){t!==""&&this.#d.push({text:t,special:r})}#h(t){let r=[];for(;;){let n=this.#d.map(o=>o.text).join("");if(n==="")return r;let a=this.#o==="reasoning"?this.#n:this.#t,s=bp(n,a);if(s===null){let o=t?0:vp(n,a);return r.push(...this.#y(this.#o,this.#w(n.length-o))),r}r.push(...this.#y(this.#o,this.#w(s.at))),this.#w(s.needle.length),this.#o==="reasoning"?(r.push(...this.#b("reasoning")),this.#o="text"):s.needle===this.#e&&(r.push(...this.#b("text")),this.#o="reasoning",this.thought=!0)}}#w(t){let r="",n=t;for(;n>0;){let a=this.#d[0],s=a.text.slice(0,n);a.special||(r+=s),n-=s.length,s.length===a.text.length?this.#d.shift():a.text=a.text.slice(s.length)}return r}#b(t){let r=t==="reasoning"?this.#p:this.#f;return r.matched?[]:this.#v(t,r.flush())}#y(t,r){return r===""?[]:this.#v(t,(t==="reasoning"?this.#p:this.#f).push(r))}#v(t,r){if(r==="")return[];let n=(t==="reasoning"?this.reasoning:this.answer).push(r);return n===""?[]:[{type:t,text:n}]}},js=class{result;#e=new AbortController;#t=[];#n;#r=null;#a=!1;#s=null;#l=!1;#u=performance.now();#o=0;#i=0;#c=0;constructor(t,r){let n=()=>{};if(r)if(r.aborted)this.#e.abort(r.reason);else{let i=()=>this.#e.abort(r.reason);r.addEventListener("abort",i,{once:!0}),n=()=>r.removeEventListener("abort",i)}let a,s=new Promise((i,u)=>{a=u});this.#n=a;let o=t(this);o.catch(()=>{}),this.result=Promise.race([o,s]).then(i=>(this.#d(null),i),i=>{throw this.#d({error:i}),i}).finally(n),this.result.catch(()=>{})}cancel(){this.#e.abort()}get completion_tokens(){return this.#c}get timings(){let t=this.#i||performance.now(),r=this.#o?t-this.#o:0;return{ttft_ms:(this.#o||t)-this.#u,decode_ms:r,tokens_per_second:this.#c>1&&r>0?(this.#c-1)/(r/1e3):0}}generated(){this.#c++===0&&(this.#o=performance.now())}ended(){return this.#i||=performance.now(),this.timings}get signal(){return this.#e.signal}emit(t){this.#a||(this.#t.push(t),this.#r?.())}abandon(t){this.#n(t),this.#e.abort(t)}#d(t){this.#i||=performance.now(),this.#a=!0,this.#s=t,this.#r?.()}[Symbol.asyncIterator](){if(this.#l)throw new TypeError("a ChatTurn's events can be iterated once");return this.#l=!0,{next:async()=>{for(;;){let t=this.#t.shift();if(t)return{value:t,done:!1};if(this.#a){if(this.#s)throw this.#s.error;return{value:void 0,done:!0}}await new Promise(r=>{this.#r=r}),this.#r=null}},return:async()=>(this.#a||this.cancel(),{value:void 0,done:!0})}}},Ns=class{#e;#t;#n;#r;#a=new Map;#s=null;#l=null;constructor(t){this.#e=t.host,this.#t=t.state,this.#n=t.ledger,this.#r=t.release;let r=Su(t.options?.max_checkpoints??cw,"max_checkpoints");t.state.cache.setRewindPointCapacity?.(r)}get_seq_length(){return this.#t.cache.get_seq_length()}get max_length(){return this.#t.cache.maxLength}generate(t,r={}){return new js(n=>this.#m(t,r,n,!1),r.signal)}prime(t,r={}){let n=[...wu(t),{role:"user",content:""}];return new js(a=>this.#m(n,r,a,!0),r.signal)}count_tokens(t,r={}){this.#u(),r.tools!==void 0&&yu(r.tools);let{inputIds:n,imageTokens:a}=this.#f(t,this.#e.templateArgs(this.#c(r)),r.tools??null);return{prompt_tokens:n.length,image_tokens:a}}reset(){if(this.#u(),this.#s)throw new hn("a turn is running on this session; cancel it before resetting");this.#i()}#u(){if(this.#l)throw new Error("this ChatSession has been disposed")}dispose(){return this.#l??=this.#o()}async#o(){let t=this.#s;t&&(t.cancel(),await t.result.catch(()=>{})),this.#r()}deviceLost(t){this.#s?.abandon(t)}#i(){this.#t.cache.truncate(0),this.#n.clear()}#c(t){return{...t.chat_template_kwargs,...t.enable_thinking!==void 0?{enable_thinking:t.enable_thinking}:{}}}#d(t){let r=new Map;if(this.#a.size===0)return{messages:t,replayed:r};let n=t.findLastIndex(s=>s.role==="user");return{messages:t.map((s,o)=>{if(s.role!=="assistant")return s;let i=this.#a.get(xu(s));if(!i)return s;let u=o>n?"after_last_user":"before_last_user";if(i.media.length===0)return this.#e.historyMessage({...i.context,position:u});let l=this.#e.historyMessage({...i.context,visibleText:i.context.rawText,position:u});return r.set(l,i.media),l}),replayed:r}}#p(t,r,n,a){let s=new Set(a.filter(o=>o.role==="assistant").map(o=>xu(o)));for(let o of this.#a.keys())s.has(o)||this.#a.delete(o);this.#a.set(xu(t),{context:r,media:n})}#f(t,r,n){let a=this.#e,s=this.#d(t),o=a.renderPrompt(s.messages,!0,r,n),i=a.planImages(s.messages,s.replayed),u=this.#n.prepare(o,i),l=u.inputIds,c=u.imageSpans.reduce((d,p)=>d+(p.tokenId===i?.imageTokenId?p.tokens:0),0);return{prompt:o,images:i,reconciled:u,inputIds:l,imageTokens:c}}async#m(t,r,n,a){this.#u();let s=this.#e.acquireLease();this.#s=n;try{return await this.#g(t,r,n,a)}finally{this.#s=null,s()}}async#g(t,r,n,a){let s=this.#e,o=this.#t,i=o.cache,u=this.#n,l=n.signal,c=Su(r.max_new_tokens??s.defaultMaxNewTokens,"max_new_tokens"),d=r.eos_token_id??s.eosTokenIds,p=this.#w(r),f=mw(r.stop_strings),m=r.tools??null;m!==null&&yu(m);let h=s.speech,g=r.generation_mode;if(g!==void 0&&!["text","audio","interleaved"].includes(g))throw new TypeError(`generation_mode must be "text", "audio" or "interleaved", got ${String(g)}`);if(!h&&(g??"text")!=="text")throw new TypeError(`this model does not speak (generation_mode "${g}")`);let w=r.audio_chunk_frames===void 0?void 0:Su(r.audio_chunk_frames,"audio_chunk_frames");i.get_seq_length()!==u.cacheLength&&this.#i();let b=s.templateArgs(this.#c(r)),{prompt:v,images:y,reconciled:S,inputIds:C,imageTokens:k}=this.#f(t,b,m),B=i.maxLength;if(C.length+1>B)throw new ea(C.length+1,B);let P=Math.min(c,B-C.length),_=S.imageSpans,x=(re,Z)=>y?_.filter(U=>U.start>=re&&U.start<Z).map(U=>y.images[U.image]):[],R=[],O=[],N=[],Q=[],V=h?{generationMode:g,audioChunkFrames:w,toolCallToken:m?.length?s.toolCallDelimiters?.open??null:null,sink:{frame:re=>{O.push(re),N.push(h.planSpeech(re))},audio:re=>{Q.push(re),n.emit({type:"audio",audio:re})}}}:null,$=!1,M=0,j=0,A=0,T=!1,E=({done:re})=>{T=!0,n.emit({type:"prefill",done:j+re,total:A})},X=r.decode_pipeline_depth,z=(re,Z,U,te,ae,ne)=>{let xe=x(te,te+re.length);return s.streamTokens(o,{suffixIds:re,maxNewTokens:Z,eosTokenId:d,stopOnEos:U,signal:l,onPrefillProgress:E,...xe.length>0?{images:xe}:{},...X!==void 0?{decodePipelineDepth:X}:{},...ae?{sampling:ae}:{},...ne?{speech:ne}:{}})},G=async re=>{let Z=i.get_seq_length();for await(let U of z(C.slice(Z,re),1,!1,Z,null,null));if(j+=re-Z,i.get_seq_length()!==re)throw new Error(`prefill left the cache at ${i.get_seq_length()}, not ${re}`)},L=s.replyMarkup,K=new Tu({special:re=>pw(s.tokenizer,re),decodeRaw:re=>Fr(s.tokenizer,re,{skip_special_tokens:!1}),markup:L,calls:s.toolCallDelimiters,startInReasoning:L!==null&&v.lastIndexOf(L.reasoningOpen)>v.lastIndexOf(L.reasoningClose),stopStrings:f});try{try{let U=Math.min(S.prefixLength,C.length-1),te=Jd(i,U);if(te===0?this.#i():te!==i.get_seq_length()&&(i.truncate(te),u.rewind(te)),M=i.get_seq_length(),A=C.length-M,M===0){let De=await this.#b(t,b,C,G,m);De>0&&(M=De,A=C.length-De)}if(a)return this.#h(n,M);i.get_seq_length()>0&&await i.captureRewindPoint?.();let ae=i.get_seq_length(),ne=C.length-ae;i.captureRewindPoint&&ne>=2&&(ne>=dw||x(ae,C.length).length>0)&&(await G(C.length-1),await i.captureRewindPoint());let xe=i.get_seq_length();for await(let De of z(C.slice(xe),P,!0,xe,p,V)){R.length===0&&T&&n.emit({type:"prefill",done:A,total:A}),n.generated(),R.push(De);for(let Me of K.push(De))n.emit(Me);if(l.aborted||K.stopped)break}l.aborted&&($=!0)}catch(U){if(!(l.aborted&&U===l.reason))throw U;$=!0}for(let U of K.finish())n.emit(U);let re=i.get_seq_length(),Z=[..._,...gw(R,N,C.length,h?.audioOutTokenId)];re>=C.length?u.commit(C,R,re,Z):u.commit(C.slice(0,re),[],re,Z)}catch(re){throw this.#i(),re}let D=n.ended(),F=!$&&!K.stopped&&R.length<P;F&&K.closeOpenCall();let H=$?"cancelled":!K.stopped&&R.length>=P?"length":F&&K.calls.length>0?"tool_calls":"stop",Y=H==="tool_calls"?K.calls.map(re=>mp(re,m??[])):null,ce={role:"assistant",content:O.length>0?[{type:"text",text:K.answer.text},{type:"audio_out",codes:Wa(O,Int32Array),audio:Wa(Q,Float32Array)}]:K.answer.text,...K.thought?{reasoning_content:K.reasoning.text}:{},...Y?{tool_calls:Y}:{}};for(let re of Y??[])n.emit({type:"tool_call",tool_call:re});return a||this.#p(ce,{visibleText:Fr(s.tokenizer,R,{skip_special_tokens:!0}),rawText:Fr(s.tokenizer,R,{skip_special_tokens:!1}),templateArgs:b},N,t),{finish_reason:H,...H==="length"?{limit:P<c?"context":"max_new_tokens"}:{},...p?{seed:p.seed}:{},message:ce,usage:{prompt_tokens:C.length,cached_tokens:M,image_tokens:k,completion_tokens:R.length,reasoning_tokens:K.reasoningTokens},timings:D}}#h(t,r){let n=this.#t.cache.get_seq_length();return{finish_reason:t.signal.aborted?"cancelled":"stop",message:{role:"assistant",content:""},usage:{prompt_tokens:n,cached_tokens:r,image_tokens:0,completion_tokens:0,reasoning_tokens:0},timings:t.ended()}}#w(t){let r=this.#e.generationConfig;return Qr({do_sample:t.do_sample??r.do_sample,...fw({temperature:t.temperature??r.temperature,top_k:t.top_k??r.top_k,top_p:t.top_p??r.top_p,min_p:t.min_p??r.min_p,seed:t.seed})})}async#b(t,r,n,a,s){let o=this.#e,i=this.#t.cache;if(!i.captureRewindPoint)return 0;let u=wu(t);if(u.length===0)return 0;let l=ep(o.tokenizer,c=>o.renderPrompt(c,!1,r,s),u);if(l.length===0||l.length>=n.length)return 0;for(let c=0;c<l.length;++c)if(l[c]!==n[c])return 0;return await o.restoreSystemPrefix?.(this.#t,l)?(this.#n.commit(l,[],l.length),l.length):(await a(l.length),this.#n.commit(l,[],l.length),await i.captureRewindPoint(),await o.persistSystemPrefix?.(this.#t,l),0)}};function fw(e){return Object.fromEntries(Object.entries(e).filter(([,t])=>t!==void 0))}function mw(e){if(e===void 0)return[];if(!Array.isArray(e)||e.some(t=>typeof t!="string"||t===""))throw new TypeError("stop_strings must be an array of non-empty strings");return e}function xu(e){let t=e.tool_calls?.map(n=>[n.function?.name,n.function?.arguments])??null,r=typeof e.content=="string"?e.content:e.content.map(n=>n.type==="text"?n.text:n.type==="audio_out"?`[${n.codes.join(",")}]`:`[${n.type}]`);return JSON.stringify([r,e.reasoning_content??null,t])}function gw(e,t,r,n){if(t.length===0)return[];let a=[];for(let s=0;s<e.length;++s){if(e[s]!==n)continue;let o=t[a.length];if(o===void 0)throw new Error(`the reply holds more audio_out tokens than the ${t.length} frames it reported`);a.push({start:r+s,tokens:1,digest:o.digest,image:-1,tokenId:n})}return a}var hw=Object.freeze({vendor:"",architecture:"",device:"",description:""}),Na=Object.freeze(["maxTextureDimension1D","maxTextureDimension2D","maxTextureDimension3D","maxTextureArrayLayers","maxBindGroups","maxBindGroupsPlusVertexBuffers","maxBindingsPerBindGroup","maxDynamicUniformBuffersPerPipelineLayout","maxDynamicStorageBuffersPerPipelineLayout","maxSampledTexturesPerShaderStage","maxSamplersPerShaderStage","maxStorageBuffersPerShaderStage","maxStorageTexturesPerShaderStage","maxUniformBuffersPerShaderStage","maxUniformBufferBindingSize","maxStorageBufferBindingSize","minUniformBufferOffsetAlignment","minStorageBufferOffsetAlignment","maxVertexBuffers","maxBufferSize","maxVertexAttributes","maxVertexBufferArrayStride","maxInterStageShaderVariables","maxColorAttachments","maxColorAttachmentBytesPerSample","maxComputeWorkgroupStorageSize","maxComputeInvocationsPerWorkgroup","maxComputeWorkgroupSizeX","maxComputeWorkgroupSizeY","maxComputeWorkgroupSizeZ","maxComputeWorkgroupsPerDimension","maxImmediateSize","maxStorageBuffersInVertexStage","maxStorageBuffersInFragmentStage","maxStorageTexturesInVertexStage","maxStorageTexturesInFragmentStage"]),jr=Object.freeze(["shader-f16","subgroups","subgroup-size-control","chromium-experimental-subgroup-matrix","atomic-vec2u-min-max","timestamp-query","texture-formats-tier1","texture-formats-tier2"]),xr=65535,ww=Object.freeze({maxTextureDimension1D:8192,maxTextureDimension2D:8192,maxTextureDimension3D:2048,maxTextureArrayLayers:256,maxBindGroups:4,maxBindGroupsPlusVertexBuffers:24,maxBindingsPerBindGroup:1e3,maxDynamicUniformBuffersPerPipelineLayout:8,maxDynamicStorageBuffersPerPipelineLayout:4,maxSampledTexturesPerShaderStage:16,maxSamplersPerShaderStage:16,maxStorageBuffersPerShaderStage:10,maxStorageTexturesPerShaderStage:4,maxUniformBuffersPerShaderStage:12,maxUniformBufferBindingSize:64*1024,maxStorageBufferBindingSize:128*1024*1024,minUniformBufferOffsetAlignment:256,minStorageBufferOffsetAlignment:256,maxVertexBuffers:8,maxBufferSize:256*1024*1024,maxVertexAttributes:16,maxVertexBufferArrayStride:2048,maxInterStageShaderVariables:16,maxColorAttachments:8,maxColorAttachmentBytesPerSample:32,maxComputeInvocationsPerWorkgroup:256,maxComputeWorkgroupSizeX:256,maxComputeWorkgroupSizeY:256,maxComputeWorkgroupSizeZ:64,maxComputeWorkgroupStorageSize:16*1024,maxComputeWorkgroupsPerDimension:xr,maxImmediateSize:0}),_p=new WeakSet,yp=new WeakMap;function Ue(e=null){if(e!==null&&typeof e=="object"&&_p.has(e))return e;let t=e!==null&&typeof e=="object"?e:void 0;if(t!==void 0){let s=yp.get(t);if(s!==void 0&&s.adapterInfo===t.adapterInfo&&s.features===t.features&&s.wgslLanguageFeatures===t.wgslLanguageFeatures&&s.wgslCompileFailures===t.wgslCompileFailures&&s.limits===t.limits)return s.normalized}let r=Us(e)?e:{},n=Us(r.adapterInfo)?r.adapterInfo:{},a={adapterInfo:_w(n),features:ra(r.features),wgslLanguageFeatures:za(r.wgslLanguageFeatures),wgslCompileFailures:Bu(r.wgslCompileFailures),limits:$s(r.limits)};return _p.add(a),t!==void 0&&yp.set(t,{normalized:a,adapterInfo:t.adapterInfo,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,wgslCompileFailures:t.wgslCompileFailures,limits:t.limits}),a}function Iu(e){let t=new Set(e),r={size:t.size,has(n){return t.has(n)},entries(){return t.entries()},forEach(n,a){t.forEach(s=>n.call(a,s,s,r))},keys(){return t.keys()},values(){return t.values()},[Symbol.iterator](){return t[Symbol.iterator]()}};return r}function ra(e=[]){return Iu(Gu(e))}function za(e=[]){return Iu(Gu(e))}function Bu(e=[]){return Iu(Gu(e))}function Nt(e=null){let t=Ue(e);return{adapterInfo:{...t.adapterInfo},features:Array.from(t.features.values()).sort(),wgslLanguageFeatures:Array.from(t.wgslLanguageFeatures.values()).sort(),wgslCompileFailures:Array.from(t.wgslCompileFailures.values()).sort(),limits:{...t.limits}}}function Eu(e,t){if(!e)return null;for(let n of e.features??[])if(!t.features.has(n))return`requires device.features.has("${n}")`;for(let[n,a]of Object.entries(e.limits??{})){if(a===void 0)continue;let s=t.limits[n];if(typeof s!="number"||s<a)return`requires device.limits.${n} >= ${a}`}if(e.subgroupMinSize!==void 0){let n=t.adapterInfo.subgroupMinSize;if(typeof n!="number")return`requires adapterInfo.subgroupMinSize >= ${e.subgroupMinSize} (adapter does not report subgroup sizes)`;if(n<e.subgroupMinSize)return`requires adapterInfo.subgroupMinSize >= ${e.subgroupMinSize}`}let r=e.subgroupMatrixConfigs;if(r!==void 0&&r.length>0){let n=t.adapterInfo.subgroupMatrixConfigs;if(Array.isArray(n)&&!r.some(a=>n.some(s=>bw(a,s))))return`requires adapterInfo.subgroupMatrixConfigs to match one of [${r.map(zs).join(", ")}] (device reports [${n.map(zs).join(", ")}])`}return null}function bw(e,t){return(e.componentType===void 0||e.componentType===t.componentType)&&(e.resultComponentType===void 0||e.resultComponentType===t.resultComponentType)&&(e.M===void 0||e.M===t.M)&&(e.N===void 0||e.N===t.N)&&(e.K===void 0||e.K===t.K)}function zs(e){return`${e.componentType??"*"}-${e.resultComponentType??"*"}-${e.M??"*"}x${e.N??"*"}x${e.K??"*"}`}var Ks=Object.freeze(["f32","f16","u32","i32","u8","i8"]);function vw(e){if(!Us(e))return null;let t=e.componentType,r=e.resultComponentType,{M:n,N:a,K:s}=e;return!Ks.includes(t)||!Ks.includes(r)||!Cu(n)||!Cu(a)||!Cu(s)?null:Object.freeze({componentType:t,resultComponentType:r,M:n,N:a,K:s})}function na(e){if(e==null||typeof e=="string"||typeof e[Symbol.iterator]!="function")return null;let t=[];for(let r of e){let n=vw(r);n!==null&&t.push(n)}return t.sort((r,n)=>zs(r).localeCompare(zs(n))),Object.freeze(t)}function Cu(e){return typeof e=="number"&&Number.isInteger(e)&&e>0}function Ru(e){let t=e.adapterInfo;return{vendor:typeof t.vendor=="string"?t.vendor.toLowerCase():"",architecture:typeof t.architecture=="string"?t.architecture.toLowerCase():"",...typeof t.subgroupMinSize=="number"?{subgroupMinSize:t.subgroupMinSize}:{},...typeof t.subgroupMaxSize=="number"?{subgroupMaxSize:t.subgroupMaxSize}:{},isFallbackAdapter:t.isFallbackAdapter===!0,...Array.isArray(t.subgroupMatrixConfigs)?{subgroupMatrixConfigs:t.subgroupMatrixConfigs}:{}}}function _w(e){let{subgroupMatrixConfigs:t,...r}=e,n=na(t);return{...hw,...r,...n!==null?{subgroupMatrixConfigs:n}:{}}}function $s(e){let t=Us(e)?e:{},r={...ww};for(let a of Na){let s=yw(t,a);s!==void 0&&(r[a]=s)}for(let[a,s]of Object.entries(t))typeof s=="number"&&Number.isFinite(s)&&(r[a]=s);let n=r.maxComputeWorkgroupsPerDimension;return typeof n=="number"&&(r.maxComputeWorkgroupsPerDimension=Math.min(n,xr)),r}function Gu(e){return e==null?[]:Array.isArray(e)?e.filter(Pu):typeof e[Symbol.iterator]=="function"?Array.from(e).filter(Pu):typeof e.values=="function"?Array.from(e.values()).filter(Pu):[]}function yw(e,t){let r=e[t];return typeof r=="number"&&Number.isFinite(r)?r:void 0}function Pu(e){return typeof e=="string"}function Us(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}var Aw=Object.freeze({float16:"float16",float32:"float32",int8:"int8",int16:"int16",int32:"int32",int64:"int64",uint8:"uint8",uint32:"uint32",bool:"bool"}),wn=Object.freeze({...Aw,int64:"int64"});function bn(e,t){return e===t||e==="int64"&&(t==="int32"||t==="uint32")}var Qe=Object.freeze({float16:{storageByteSize:2,onDiskByteSize:2,wgslScalar:"f16",arrayCtor:Uint16Array},float32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"f32",arrayCtor:Float32Array},int8:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"i32",arrayCtor:Int32Array},int16:{storageByteSize:4,onDiskByteSize:2,wgslScalar:"i32",arrayCtor:Int32Array},int32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"i32",arrayCtor:Int32Array},int64:{storageByteSize:8,onDiskByteSize:8,wgslScalar:"vec2<u32>",arrayCtor:BigInt64Array},uint8:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"u32",arrayCtor:Uint32Array},uint32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"u32",arrayCtor:Uint32Array},bool:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"u32",arrayCtor:Uint32Array}});function Le(e){if(Object.hasOwn(Qe,e))return e;throw new Error(`Unsupported dtype: ${e}`)}function Ae(e){return Qe[e].storageByteSize}function Lu(e){return e==="uint8"}function Hs(e,t){if(!Lu(e))throw new Error(`${t}: the packed storage layout is defined for uint8 tensors, not ${e}`)}function Ye(e,t,r){if(!Number.isSafeInteger(t)||t<0)throw new Error(`tensorStorageByteLength: element count must be a nonnegative safe integer, got ${t}`);let n;if(r==="packed"?(Hs(e,"tensorStorageByteLength"),n=at(t,4)):n=t*Ae(e),!Number.isSafeInteger(n))throw new Error("tensorStorageByteLength: byte length exceeds safe integer storage");return n}function Ap(e){let t=new Uint32Array(Math.ceil(e.length/4));for(let r=0;r<e.length;r+=1){let n=e[r];if(!Number.isInteger(n)||n<0||n>255)throw new Error(`packUint8Elements: element ${r} (${n}) is not a byte`);t[r>>>2]|=n<<(r&3)*8}return t}function kp(e,t){if(e.length*4<t)throw new Error(`unpackUint8Elements: ${e.length} words cannot hold ${t} elements`);let r=new Uint32Array(t);for(let n=0;n<t;n+=1)r[n]=e[n>>>2]>>>(n&3)*8&255;return r}function ke(e){if(!Array.isArray(e))throw new Error("shape must be an array");let t=1;for(let r of e){if(!Number.isInteger(r)||r<0)throw new Error(`invalid shape dimension: ${r}`);t*=r}return t}function at(e,t){return Math.ceil(e/t)*t}function ct(e){return Math.max(4,at(e,4))}function Vs(e,t){return e.length===t.length&&e.every((r,n)=>r===t[n])}function wt(e){return e!==null&&typeof e=="object"&&typeof e.dtype=="string"&&Array.isArray(e.shape)&&typeof e.size=="number"&&typeof e.byteLength=="number"&&"buffer"in e&&"runtime"in e}function vn(){}function he(e,t){let r=ke([...t]);return{dtype:e,shape:[...t],size:r,byteLength:r*Ae(e),byteOffset:0,buffer:{destroy(){}},runtime:null,destroy(){}}}function _n(e){if(e instanceof BigInt64Array)throw new Error("Opaque int64 data cannot be used for numeric tensor arithmetic");return e}function kw(e){return Object.fromEntries(Object.entries(e).filter(([,t])=>t!==void 0))}var Sw=/^vec([234])<(u32|i32|f32)>$/;function Sp(e){if(e==="u32"||e==="i32"||e==="f32")return{align:4,size:4,scalar:e,components:1};let t=Sw.exec(e);if(!t)throw new Error(`Unsupported uniform field type: ${e}`);let r=Number(t[1]);return{align:r===2?8:16,size:r===3?12:r*4,scalar:t[2],components:r}}function xw(e){return e==="u32"||e==="i32"||e==="f32"}function qu(e,t={}){return Ka(e,"u32",t)}function Ou(e,t={}){return Ka(e,"i32",t)}function Du(e,t={}){return Ka(e,"f32",t)}function Ka(e,t,r={}){return Ua(e,"uniform field"),Sp(t),Object.freeze({kind:"uniform-field",name:e,type:t,semantic:r.semantic,required:r.required??!r.internal,internal:!!r.internal,default:r.default,description:r.description})}function Zs(e,t,r={}){if(Ua(e,"uniform struct"),!Array.isArray(t)||t.length===0)throw new Error(`uniform struct ${e} requires at least one field`);let n=t.map(a=>Cw(a,e));return Object.freeze({kind:"uniform-struct",name:e,fields:Object.freeze(Pw(n,r))})}function Mu(e){Wu(e);let t=e.fields.map(r=>`${r.name}: ${r.type}`).join(", ");return`struct ${e.name} { ${t} };`}function Xs(e,t={}){if(Wu(e),!e.fields.every(i=>xw(i.type)))return Tw(e,t);let r=Math.max(16,Math.ceil(e.fields.length*4/16)*16),n=new ArrayBuffer(r),a=new Uint32Array(n),s=new Int32Array(n),o=new Float32Array(n);return e.fields.forEach((i,u)=>{let l=t[i.name];if(l===void 0&&i.required)throw new Error(`Missing uniform field ${e.name}.${i.name}`);if(l===void 0&&(l=i.default??0),typeof l!="number")throw new Error(`Uniform ${e.name}.${i.name} expects a scalar number`);xp(i.type,u,l,`${e.name}.${i.name}`,a,s,o)}),new Uint32Array(n)}function Tw(e,t){let r=[],n=0;for(let u of e.fields){let l=Sp(u.type);n=at(n,l.align),r.push({field:u,layout:l,offset:n}),n+=l.size}let a=new ArrayBuffer(Math.max(16,at(n,16))),s=new Uint32Array(a),o=new Int32Array(a),i=new Float32Array(a);for(let{field:u,layout:l,offset:c}of r){let d=t[u.name];if(d===void 0&&u.required)throw new Error(`Missing uniform field ${e.name}.${u.name}`);d===void 0&&(d=u.default??(l.components===1?0:new Array(l.components).fill(0)));let p=typeof d=="number"?[d]:d;if(p.length!==l.components)throw new Error(`Uniform ${e.name}.${u.name} expects ${l.components} component(s), got ${p.length}`);for(let f=0;f<l.components;++f)xp(l.scalar,(c+f*4)/4,p[f],`${e.name}.${u.name}[${f}]`,s,o,i)}return new Uint32Array(a)}function xp(e,t,r,n,a,s,o){if(e==="u32"){if(!Number.isInteger(r)||r<0||r>4294967295)throw new Error(`Uniform ${n} must be an integer u32 in [0, 4294967295]`);a[t]=r;return}if(e==="i32"){if(!Number.isInteger(r)||r<-2147483648||r>2147483647)throw new Error(`Uniform ${n} must be an integer i32 in [-2147483648, 2147483647]`);s[t]=r;return}if(Number.isNaN(r))throw new Error(`Uniform ${n} must not be NaN (f32)`);o[t]=r}function Tp(e,t,r,n){return e.createUniformU32(Xs(t,r),n)}function Cp(e){return Wu(e),{kind:e.kind,name:e.name,fields:e.fields.map(t=>kw({name:t.name,type:t.type,required:t.required,internal:t.internal,semantic:t.semantic,default:t.default,description:t.description}))}}function Cw(e,t){if(!e||e.kind!=="uniform-field")throw new Error(`uniform struct ${t} fields must be created with u32(), i32(), or f32()`);return e}function Pw(e,t){let r=t.alignFieldsTo??4;if(!Number.isInteger(r)||r<=0)throw new Error(`alignFieldsTo must be a positive integer, got ${r}`);let n=[...e],a=n.length%r;if(a===0)return Object.freeze(n);let s=r-a,o=0,i=new Set(n.map(u=>u.name));for(let u=0;u<s;++u){for(;i.has(`_pad${o}`);)o++;let l=`_pad${o++}`;i.add(l),n.push(Ka(l,"u32",{internal:!0,required:!1,default:0}))}return Object.freeze(n)}function Wu(e){if(!e||e.kind!=="uniform-struct")throw new Error("Expected a uniform struct schema")}function Ua(e,t){if(typeof e!="string"||!/^[A-Za-z_][A-Za-z0-9_]*$/.test(e))throw new Error(`${t} name must be a WGSL-compatible identifier, got ${e}`)}var Ip=Object.freeze({read:"read-only-storage",read_write:"storage"});function Fu(e,t={}){Ua(e,"storage binding");let r=t.access??"read";if(!(r in Ip))throw new Error(`storage binding ${e} has unsupported access ${r}`);return Object.freeze({kind:"storage",name:e,arg:t.arg,access:r,elementType:t.elementType??"f32",...t.length!==void 0?{length:t.length}:{},semantic:t.semantic})}function Qu(e,t,r={}){Ua(e,"uniform binding");let n;if(Gw(t)?n=Zs(r.structName??Rw(e),t):n=t,!n||n.kind!=="uniform-struct")throw new Error(`uniform binding ${e} requires a uniformStruct schema or field array`);return Object.freeze({kind:"uniform",name:e,struct:n,semantic:r.semantic})}function aa(e){if(!Array.isArray(e)||e.length===0)throw new Error("bindGroup requires at least one binding");let t=new Set;return Object.freeze(e.map((r,n)=>{if(!r||r.kind!=="storage"&&r.kind!=="uniform")throw new Error("bindGroup entries must be storage() or uniform() bindings");if(t.has(r.name))throw new Error(`duplicate bindGroup binding name: ${r.name}`);return t.add(r.name),Object.freeze({...r,binding:n})}))}function ju(e,t={}){return aa(e).map(r=>Iw(r,t)).join(`
`)}function Bp(e){let t=[],r=new Map;for(let n of aa(e)){if(n.kind!=="uniform")continue;let a=JSON.stringify(Cp(n.struct)),s=r.get(n.struct.name);if(s!==void 0){if(s!==a)throw new Error(`uniform struct ${n.struct.name} is declared with conflicting schemas`);continue}r.set(n.struct.name,a),t.push(Mu(n.struct))}return t.join(`
`)}function Ep(e,t,r,n={}){let a=aa(t),s=n.labelPrefix??"kernel",o=[],i=()=>{let l=[];for(let c of o.splice(0))try{c()}catch(d){l.push(d)}if(l.length===1)throw l[0];if(l.length>1)throw new AggregateError(l,"WebGPU binding cleanup failed")},u=[];try{for(let l of a){let c=r?.[l.name];if(c==null)throw new Error(`Missing resource for binding ${l.name}`);if(l.kind==="uniform"){let m;Ew(c)?m=c:(m=Tp(e,l.struct,c,`${s}-${l.name}`),typeof m.destroy=="function"&&o.push(()=>m.destroy?.())),u.push({buffer:m,type:"uniform",binding:l.binding});continue}let d=c,p=typeof d.byteOffset=="number"?d.byteOffset:0,f=typeof d.byteLength=="number"?ct(d.byteLength):void 0;if(l.minByteSize!==void 0&&f!==void 0&&f<l.minByteSize)throw new Error(`binding ${l.name}: bound tensor provides ${f} bytes but the declared fixed length requires ${l.minByteSize}`);u.push({tensor:c,type:Ip[l.access],binding:l.binding,...p?{offset:p}:{},...f!==void 0?{size:f}:{}})}return{bindings:u,cleanup:i}}catch(l){try{i()}catch(c){throw new AggregateError([l,c],"WebGPU binding materialization and cleanup both failed")}throw l}}function Iw(e,t){if(e.kind==="storage"){let r=e.access==="read_write"?"read_write":"read",n=Nu(e.elementType,t,`binding ${e.name} elementType`);return e.length!==void 0?`@group(0) @binding(${e.binding}) var<storage, ${r}> ${e.name}: array<${n}, ${Gp(e.length,t,e.name)}u>;`:`@group(0) @binding(${e.binding}) var<storage, ${r}> ${e.name}: array<${n}>;`}return`@group(0) @binding(${e.binding}) var<uniform> ${e.name}: ${e.struct.name};`}var Pp={f32:4,u32:4,i32:4,f16:2};function Bw(e){if(/^atomic<\s*[ui]32\s*>$/.test(e))return 4;if(/^atomic<\s*(?:vec2u|vec2<\s*u32\s*>)\s*>$/.test(e))return 8;let t=/^vec([234])<\s*([a-z0-9]+)\s*>$/.exec(e);if(t){let n=Pp[t[2]];if(n!==void 0){let a=Number(t[1]);return(a===3?4:a)*n}}let r=Pp[e];if(r!==void 0)return r;throw new Error(`unsupported storage element type for a sized binding: ${e}`)}function Rp(e,t){return e.map(r=>{if(r.kind!=="storage"||r.length===void 0)return r;let n=Nu(r.elementType,t,`binding ${r.name} elementType`),a=Gp(r.length,t,r.name);return{...r,minByteSize:a*Bw(String(n))}})}function Gp(e,t,r){let n=typeof e=="number"?e:Nu(e,t,`binding ${r} length`),a=typeof n=="number"?n:Number(n);if(!Number.isSafeInteger(a)||a<=0)throw new Error(`binding ${r} length must resolve to a positive integer, got ${String(n)}`);return a}function Nu(e,t,r){if(typeof e!="string"||!e.startsWith("$"))return e;let n=e.slice(1),a=t[n];if(a==null)throw new Error(`Missing template value ${n} for ${r}`);return a}function Ew(e){return e!==null&&typeof e=="object"&&typeof e.destroy=="function"&&!("shape"in e)}function Rw(e){return e.length===0?e:e[0].toUpperCase()+e.slice(1)}function Gw(e){return Array.isArray(e)}var Lw=Object.defineProperty,qw=(e,t,r)=>t in e?Lw(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,Ow=(e,t,r)=>(qw(e,typeof t!="symbol"?t+"":t,r),r),q=Object.freeze({Text:"Text",NumericLiteral:"NumericLiteral",StringLiteral:"StringLiteral",Identifier:"Identifier",Equals:"Equals",OpenParen:"OpenParen",CloseParen:"CloseParen",OpenStatement:"OpenStatement",CloseStatement:"CloseStatement",OpenExpression:"OpenExpression",CloseExpression:"CloseExpression",OpenSquareBracket:"OpenSquareBracket",CloseSquareBracket:"CloseSquareBracket",OpenCurlyBracket:"OpenCurlyBracket",CloseCurlyBracket:"CloseCurlyBracket",Comma:"Comma",Dot:"Dot",Colon:"Colon",Pipe:"Pipe",CallOperator:"CallOperator",AdditiveBinaryOperator:"AdditiveBinaryOperator",MultiplicativeBinaryOperator:"MultiplicativeBinaryOperator",ExponentiationBinaryOperator:"ExponentiationBinaryOperator",ComparisonBinaryOperator:"ComparisonBinaryOperator",UnaryOperator:"UnaryOperator",Comment:"Comment"}),It=class{constructor(e,t){this.value=e,this.type=t}};function Lp(e){return/\w/.test(e)}function yn(e){return/[0-9]/.test(e)}function qp(e){return/\s/.test(e)}var Dw=[["{%",q.OpenStatement],["%}",q.CloseStatement],["{{",q.OpenExpression],["}}",q.CloseExpression],["(",q.OpenParen],[")",q.CloseParen],["{",q.OpenCurlyBracket],["}",q.CloseCurlyBracket],["[",q.OpenSquareBracket],["]",q.CloseSquareBracket],[",",q.Comma],[".",q.Dot],[":",q.Colon],["|",q.Pipe],["<=",q.ComparisonBinaryOperator],[">=",q.ComparisonBinaryOperator],["==",q.ComparisonBinaryOperator],["!=",q.ComparisonBinaryOperator],["<",q.ComparisonBinaryOperator],[">",q.ComparisonBinaryOperator],["+",q.AdditiveBinaryOperator],["-",q.AdditiveBinaryOperator],["~",q.AdditiveBinaryOperator],["**",q.ExponentiationBinaryOperator],["*",q.MultiplicativeBinaryOperator],["//",q.MultiplicativeBinaryOperator],["/",q.MultiplicativeBinaryOperator],["%",q.MultiplicativeBinaryOperator],["=",q.Equals]],Mw=new Map([["n",`
`],["t","	"],["r","\r"],["b","\b"],["f","\f"],["v","\v"],["'","'"],['"','"'],["\\","\\"]]);function Ww(e,t={}){return e.endsWith(`
`)&&(e=e.slice(0,-1)),t.lstrip_blocks&&(e=e.replace(/^[ \t]*({[#%-])/gm,"$1")),t.trim_blocks&&(e=e.replace(/([#%-]})\n/g,"$1")),e.replace(/(\s*){%(-?)\s*(?:end)?generation\s*(-?)%}(\s*)/gs,(r,n,a,s,o)=>(a?"":n)+(s?"":o))}function no(e,t={}){let r=[],n=Ww(e,t),a=0,s=0,o=l=>{let c="";for(;l(n[a]);){if(n[a]==="\\"){if(++a,a>=n.length)throw new SyntaxError("Unexpected end of input");let d=n[a++],p=Mw.get(d);if(p===void 0)throw new SyntaxError(`Unexpected escaped character: ${d}`);c+=p;continue}if(c+=n[a++],a>=n.length)throw new SyntaxError("Unexpected end of input")}return c},i=()=>{let l=r.at(-1);l&&l.type===q.Text&&(l.value=l.value.trimEnd(),l.value===""&&r.pop())},u=()=>{for(;a<n.length&&qp(n[a]);)++a};e:for(;a<n.length;){let l=r.at(-1)?.type;if(l===void 0||l===q.CloseStatement||l===q.CloseExpression||l===q.Comment){let d="";for(;a<n.length&&!(n[a]==="{"&&(n[a+1]==="%"||n[a+1]==="{"||n[a+1]==="#"));)d+=n[a++];if(d.length>0){r.push(new It(d,q.Text));continue}}if(n[a]==="{"&&n[a+1]==="#"){a+=2;let d=n[a]==="-";d&&++a;let p="";for(;n[a]!=="#"||n[a+1]!=="}";){if(a+2>=n.length)throw new SyntaxError("Missing end of comment tag");p+=n[a++]}let f=p.endsWith("-");f&&(p=p.slice(0,-1)),d&&i(),r.push(new It(p,q.Comment)),a+=2,f&&u();continue}if(n.slice(a,a+3)==="{%-"){i(),r.push(new It("{%",q.OpenStatement)),a+=3;continue}if(n.slice(a,a+3)==="{{-"){i(),r.push(new It("{{",q.OpenExpression)),s=0,a+=3;continue}if(o(qp),n.slice(a,a+3)==="-%}"){r.push(new It("%}",q.CloseStatement)),a+=3,u();continue}if(n.slice(a,a+3)==="-}}"){r.push(new It("}}",q.CloseExpression)),a+=3,u();continue}let c=n[a];if(c==="-"||c==="+"){let d=r.at(-1)?.type;if(d===q.Text||d===void 0)throw new SyntaxError(`Unexpected character: ${c}`);switch(d){case q.Identifier:case q.NumericLiteral:case q.StringLiteral:case q.CloseParen:case q.CloseSquareBracket:break;default:{++a;let p=o(yn);if(p.length>0&&n[a]==="."&&yn(n[a+1])){++a;let f=o(yn);p=`${p}.${f}`}r.push(new It(`${c}${p}`,p.length>0?q.NumericLiteral:q.UnaryOperator));continue}}}for(let[d,p]of Dw){if(d==="}}"&&s>0)continue;if(n.slice(a,a+d.length)===d){r.push(new It(d,p)),p===q.OpenExpression?s=0:p===q.OpenCurlyBracket?++s:p===q.CloseCurlyBracket&&--s,a+=d.length;continue e}}if(c==="'"||c==='"'){++a;let d=o(p=>p!==c);r.push(new It(d,q.StringLiteral)),++a;continue}if(yn(c)){let d=o(yn);if(r.at(-1)?.type!==q.Dot&&n[a]==="."&&yn(n[a+1])){++a;let p=o(yn);d=`${d}.${p}`}r.push(new It(d,q.NumericLiteral));continue}if(Lp(c)){let d=o(Lp);r.push(new It(d,q.Identifier));continue}throw new SyntaxError(`Unexpected character: ${c}`)}return r}var Et=class{type="Statement"},Fw=class extends Et{constructor(e){super(),this.body=e}type="Program"},Qw=class extends Et{constructor(e,t,r){super(),this.test=e,this.body=t,this.alternate=r}type="If"},jw=class extends Et{constructor(e,t,r,n){super(),this.loopvar=e,this.iterable=t,this.body=r,this.defaultBlock=n}type="For"},Nw=class extends Et{type="Break"},zw=class extends Et{type="Continue"},Kw=class extends Et{constructor(e,t,r){super(),this.assignee=e,this.value=t,this.body=r}type="Set"},Uw=class extends Et{constructor(e,t,r){super(),this.name=e,this.args=t,this.body=r}type="Macro"},$w=class extends Et{constructor(e){super(),this.value=e}type="Comment"},bt=class extends Et{type="Expression"},Hw=class extends bt{constructor(e,t,r){super(),this.object=e,this.property=t,this.computed=r}type="MemberExpression"},Op=class extends bt{constructor(e,t){super(),this.callee=e,this.args=t}type="CallExpression"},Nr=class extends bt{constructor(e){super(),this.value=e}type="Identifier"},oa=class extends bt{constructor(e){super(),this.value=e}type="Literal"},Vw=class extends oa{type="IntegerLiteral"},Zw=class extends oa{type="FloatLiteral"},Dp=class extends oa{type="StringLiteral"},Xw=class extends oa{type="ArrayLiteral"},Mp=class extends oa{type="TupleLiteral"},Yw=class extends oa{type="ObjectLiteral"},sa=class extends bt{constructor(e,t,r){super(),this.operator=e,this.left=t,this.right=r}type="BinaryExpression"},Jw=class extends bt{constructor(e,t){super(),this.operand=e,this.filter=t}type="FilterExpression"},eb=class extends Et{constructor(e,t){super(),this.filter=e,this.body=t}type="FilterStatement"},tb=class extends bt{constructor(e,t){super(),this.lhs=e,this.test=t}type="SelectExpression"},rb=class extends bt{constructor(e,t,r){super(),this.operand=e,this.negate=t,this.test=r}type="TestExpression"},Wp=class extends bt{constructor(e,t){super(),this.operator=e,this.argument=t}type="UnaryExpression"},nb=class extends bt{constructor(e=void 0,t=void 0,r=void 0){super(),this.start=e,this.stop=t,this.step=r}type="SliceExpression"},zu=class extends bt{constructor(e,t){super(),this.key=e,this.value=t}type="KeywordArgumentExpression"},ab=class extends bt{constructor(e){super(),this.argument=e}type="SpreadExpression"},sb=class extends bt{constructor(e){super(),this.argument=e}type="KeywordSpreadExpression"},ob=class extends Et{constructor(e,t,r){super(),this.call=e,this.callerArgs=t,this.body=r}type="CallStatement"},ib=class extends bt{constructor(e,t,r){super(),this.condition=e,this.trueExpr=t,this.falseExpr=r}type="Ternary"};function ao(e){let t=new Fw([]),r=0;function n(A,T){let E=e[r++];if(!E||E.type!==A)throw new Error(`Parser Error: ${T}. ${E.type} !== ${A}.`);return E}function a(A){if(!u(A))throw new SyntaxError(`Expected ${A}`);++r}function s(){switch(e[r].type){case q.Comment:return new $w(e[r++].value);case q.Text:return l();case q.OpenStatement:return c();case q.OpenExpression:return d();default:throw new SyntaxError(`Unexpected token type: ${e[r].type}`)}}function o(...A){return r+A.length<=e.length&&A.every((T,E)=>T===e[r+E].type)}function i(...A){return e[r]?.type===q.OpenStatement&&e[r+1]?.type===q.Identifier&&A.includes(e[r+1]?.value)}function u(...A){return r+A.length<=e.length&&A.every((T,E)=>e[r+E].type==="Identifier"&&T===e[r+E].value)}function l(){return new Dp(n(q.Text,"Expected text token").value)}function c(){if(n(q.OpenStatement,"Expected opening statement token"),e[r].type!==q.Identifier)throw new SyntaxError(`Unknown statement, got ${e[r].type}`);let A=e[r].value,T;switch(A){case"set":++r,T=p();break;case"if":++r,T=f(),n(q.OpenStatement,"Expected {% token"),a("endif"),n(q.CloseStatement,"Expected %} token");break;case"macro":++r,T=m(),n(q.OpenStatement,"Expected {% token"),a("endmacro"),n(q.CloseStatement,"Expected %} token");break;case"for":++r,T=g(),n(q.OpenStatement,"Expected {% token"),a("endfor"),n(q.CloseStatement,"Expected %} token");break;case"call":{++r;let E=null;o(q.OpenParen)&&(E=_("parameters"));let X=j();if(X.type!=="Identifier")throw new SyntaxError("Expected identifier following call statement");let z=_("call");n(q.CloseStatement,"Expected closing statement token");let G=[];for(;!i("endcall");)G.push(s());n(q.OpenStatement,"Expected '{%'"),a("endcall"),n(q.CloseStatement,"Expected closing statement token");let L=new Op(X,z);T=new ob(L,E,G);break}case"break":++r,n(q.CloseStatement,"Expected closing statement token"),T=new Nw;break;case"continue":++r,n(q.CloseStatement,"Expected closing statement token"),T=new zw;break;case"filter":{++r;let E=j();E instanceof Nr&&o(q.OpenParen)&&(E=P(E)),n(q.CloseStatement,"Expected closing statement token");let X=[];for(;!i("endfilter");)X.push(s());n(q.OpenStatement,"Expected '{%'"),a("endfilter"),n(q.CloseStatement,"Expected '%}'"),T=new eb(E,X);break}default:throw new SyntaxError(`Unknown statement type: ${A}`)}return T}function d(){n(q.OpenExpression,"Expected opening expression token");let A=w();return n(q.CloseExpression,"Expected closing expression token"),A}function p(){let A=h(),T=null,E=[];if(o(q.Equals))++r,T=h();else{for(n(q.CloseStatement,"Expected %} token");!i("endset");)E.push(s());n(q.OpenStatement,"Expected {% token"),a("endset")}return n(q.CloseStatement,"Expected closing statement token"),new Kw(A,T,E)}function f(){let A=w();n(q.CloseStatement,"Expected closing statement token");let T=[],E=[];for(;!i("elif","else","endif");)T.push(s());if(i("elif")){++r,++r;let X=f();E.push(X)}else if(i("else"))for(++r,++r,n(q.CloseStatement,"Expected closing statement token");!i("endif");)E.push(s());return new Qw(A,T,E)}function m(){let A=j();if(A.type!=="Identifier")throw new SyntaxError("Expected identifier following macro statement");let T=_("parameters");n(q.CloseStatement,"Expected closing statement token");let E=[];for(;!i("endmacro");)E.push(s());return new Uw(A,T,E)}function h(A=!1){let T=A?j:w,E=[T()],X=o(q.Comma);for(;X&&(++r,E.push(T()),!!o(q.Comma)););return X?new Mp(E):E[0]}function g(){let A=h(!0);if(!(A instanceof Nr||A instanceof Mp))throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${A.type} instead`);if(!u("in"))throw new SyntaxError("Expected `in` keyword following loop variable");++r;let T=w();n(q.CloseStatement,"Expected closing statement token");let E=[];for(;!i("endfor","else");)E.push(s());let X=[];if(i("else"))for(++r,++r,n(q.CloseStatement,"Expected closing statement token");!i("endfor");)X.push(s());return new jw(A,T,E,X)}function w(){return b()}function b(){let A=v();if(u("if")){++r;let T=v();if(u("else")){++r;let E=b();return new ib(T,A,E)}else return new tb(A,T)}return A}function v(){let A=y();for(;u("or");){let T=e[r];++r;let E=y();A=new sa(T,A,E)}return A}function y(){let A=S();for(;u("and");){let T=e[r];++r;let E=S();A=new sa(T,A,E)}return A}function S(){let A;for(;u("not");){let T=e[r];++r;let E=S();A=new Wp(T,E)}return A??C()}function C(){let A=k();for(;;){let T;if(u("not","in"))T=new It("not in",q.Identifier),r+=2;else if(u("in"))T=e[r++];else if(o(q.ComparisonBinaryOperator))T=e[r++];else break;let E=k();A=new sa(T,A,E)}return A}function k(){let A=N();for(;o(q.AdditiveBinaryOperator);){let T=e[r];++r;let E=N();A=new sa(T,A,E)}return A}function B(){let A=O(j());return o(q.OpenParen)?P(A):A}function P(A){let T=new Op(A,_("call"));return T=O(T),o(q.OpenParen)&&(T=P(T)),T}function _(A){n(q.OpenParen,"Expected opening parenthesis for arguments list");let T=x(A);return n(q.CloseParen,"Expected closing parenthesis for arguments list"),T}function x(A){let T=[],E=new Set,X=!1,z=!1;for(;!o(q.CloseParen);){let G=o(q.ExponentiationBinaryOperator),L=o(q.MultiplicativeBinaryOperator)&&e[r].value==="*";if(A==="parameters"&&(G||L))throw new SyntaxError("Argument unpacking is not allowed in parameter declarations");if(G){if(++r,T.push(new sb(w())),o(q.Comma)&&++r,!o(q.CloseParen))throw new SyntaxError("Expected closing parenthesis: `**` must be applied to the final argument");break}let K;if(L){if(z)throw new SyntaxError("Only one `*` argument unpacking is allowed");z=!0,++r;let D=w();K=new ab(D)}else{if(K=w(),o(q.Equals)){if(++r,!(K instanceof Nr))throw new SyntaxError(A==="parameters"?"Expected identifier for parameter declaration":"Expected identifier for keyword argument");let D=w();K=new zu(K,D)}if(A==="parameters"){if(!(K instanceof Nr||K instanceof zu))throw new SyntaxError("Expected identifier for parameter declaration");let D=K instanceof Nr?K.value:K.key.value;if(E.has(D))throw new SyntaxError(`Duplicate parameter name: ${D}`);E.add(D)}if(K instanceof zu)X=!0;else{if(X)throw new SyntaxError(A==="call"?"Positional arguments must come before keyword arguments":"Non-default argument follows default argument");if(z)throw new SyntaxError("Positional arguments must not follow `*` argument unpacking")}}T.push(K),o(q.Comma)&&++r}return T}function R(){let A=[],T=!1;for(;!o(q.CloseSquareBracket);)o(q.Colon)?(A.push(void 0),++r,T=!0):(A.push(w()),o(q.Colon)&&(++r,T=!0));if(A.length===0)throw new SyntaxError("Expected at least one argument for member/slice expression");if(T){if(A.length>3)throw new SyntaxError("Expected 0-3 arguments for slice expression");return new nb(...A)}return A[0]}function O(A){for(;o(q.Dot)||o(q.OpenSquareBracket);){let T=e[r];++r;let E,X=T.type===q.OpenSquareBracket;if(X)E=R(),n(q.CloseSquareBracket,"Expected closing square bracket");else if(E=j(),E.type!=="Identifier"&&E.type!=="IntegerLiteral")throw new SyntaxError("Expected identifier or integer following dot operator");A=new Hw(A,E,X)}return A}function N(){let A=Q();for(;o(q.MultiplicativeBinaryOperator);){let T=e[r++],E=Q();A=new sa(T,A,E)}return A}function Q(){let A=V();for(;o(q.ExponentiationBinaryOperator);){let T=e[r++],E=V();A=new sa(T,A,E)}return A}function V(){let A=$();for(;u("is");){++r;let T=u("not");T&&++r;let E=j();if(!(E instanceof Nr))throw new SyntaxError("Expected identifier for the test");A=new rb(A,T,E)}return A}function $(){let A=M();for(;o(q.Pipe);){++r;let T=j();if(!(T instanceof Nr))throw new SyntaxError("Expected identifier for the filter");o(q.OpenParen)&&(T=P(T)),A=new Jw(A,T)}return A}function M(){let A=e[r];if(A&&(A.type===q.UnaryOperator||A.type===q.AdditiveBinaryOperator)&&(A.value==="-"||A.value==="+")){let T=e[r++],E=M();return new Wp(T,E)}return B()}function j(){let A=e[r++];switch(A.type){case q.NumericLiteral:{let T=A.value;return T.includes(".")?new Zw(Number(T)):new Vw(Number(T))}case q.StringLiteral:{let T=A.value;for(;o(q.StringLiteral);)T+=e[r++].value;return new Dp(T)}case q.Identifier:return new Nr(A.value);case q.OpenParen:{let T=h();return n(q.CloseParen,"Expected closing parenthesis, got ${tokens[current].type} instead."),T}case q.OpenSquareBracket:{let T=[];for(;!o(q.CloseSquareBracket);)T.push(w()),o(q.Comma)&&++r;return++r,new Xw(T)}case q.OpenCurlyBracket:{let T=new Map;for(;!o(q.CloseCurlyBracket);){let E=w();n(q.Colon,"Expected colon between key and value in object literal");let X=w();T.set(E,X),o(q.Comma)&&++r}return++r,new Yw(T)}default:throw new SyntaxError(`Unexpected token: ${A.type}`)}}for(;r<e.length;)t.body.push(s());return t}function ub(e,t,r=1){if(t===void 0&&(t=e,e=0),r===0)throw new Error("range() step must not be zero");let n=[];if(r>0)for(let a=e;a<t;a+=r)n.push(a);else for(let a=e;a>t;a+=r)n.push(a);return n}function Fp(e,t,r,n=1){let a=Math.sign(n);a>=0?(t=(t??=0)<0?Math.max(e.length+t,0):Math.min(t,e.length),r=(r??=e.length)<0?Math.max(e.length+r,0):Math.min(r,e.length)):(t=(t??=e.length-1)<0?Math.max(e.length+t,-1):Math.min(t,e.length-1),r=(r??=-1)<-1?Math.max(e.length+r,-1):Math.min(r,e.length-1));let s=[];for(let o=t;a*o<a*r;o+=n)s.push(e[o]);return s}function lb(e){return e.replace(/\b\w/g,t=>t.toUpperCase())}function cb(e){return db(new Date,e)}function db(e,t){let r=new Intl.DateTimeFormat(void 0,{month:"long"}),n=new Intl.DateTimeFormat(void 0,{month:"short"}),a=s=>s<10?"0"+s:s.toString();return t.replace(/%[YmdbBHM%]/g,s=>{switch(s){case"%Y":return e.getFullYear().toString();case"%m":return a(e.getMonth()+1);case"%d":return a(e.getDate());case"%b":return n.format(e);case"%B":return r.format(e);case"%H":return a(e.getHours());case"%M":return a(e.getMinutes());case"%%":return"%";default:return s}})}function pb(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function fb(e,t,r,n){if(n===0)return e;let a=n==null||n<0?1/0:n,s=t.length===0?new RegExp("(?=)","gu"):new RegExp(pb(t),"gu");return e.replaceAll(s,o=>a>0?(--a,r):o)}var Qp=class extends Error{},jp=class extends Error{},mb=new Map,Ut=class{type="RuntimeValue";value;get builtins(){return mb}constructor(e=void 0){this.value=e}__bool__(){return new le(!!this.value)}toString(){return String(this.value)}},de=class extends Ut{type="IntegerValue"},Ie=class extends Ut{type="FloatValue";toString(){return Object.is(this.value,-0)?"-0.0":this.value%1===0?this.value.toFixed(1):this.value.toString()}},ee=class extends Ut{type="StringValue";_builtins;get builtins(){return this._builtins??=new Map([["upper",new Oe(()=>new ee(this.value.toUpperCase()))],["lower",new Oe(()=>new ee(this.value.toLowerCase()))],["strip",new Oe(()=>new ee(this.value.trim()))],["title",new Oe(()=>new ee(lb(this.value)))],["capitalize",new Oe(()=>new ee(this.value.charAt(0).toUpperCase()+this.value.slice(1)))],["length",new de(this.value.length)],["rstrip",new Oe(()=>new ee(this.value.trimEnd()))],["lstrip",new Oe(()=>new ee(this.value.trimStart()))],["startswith",new Oe(e=>{if(e.length===0)throw new Error("startswith() requires at least one argument");let t=e[0];if(t instanceof ee)return new le(this.value.startsWith(t.value));if(t instanceof we){for(let r of t.value){if(!(r instanceof ee))throw new Error("startswith() tuple elements must be strings");if(this.value.startsWith(r.value))return new le(!0)}return new le(!1)}throw new Error("startswith() argument must be a string or tuple of strings")})],["endswith",new Oe(e=>{if(e.length===0)throw new Error("endswith() requires at least one argument");let t=e[0];if(t instanceof ee)return new le(this.value.endsWith(t.value));if(t instanceof we){for(let r of t.value){if(!(r instanceof ee))throw new Error("endswith() tuple elements must be strings");if(this.value.endsWith(r.value))return new le(!0)}return new le(!1)}throw new Error("endswith() argument must be a string or tuple of strings")})],["split",new Oe(e=>{let t=e[0]??new je;if(!(t instanceof ee||t instanceof je))throw new Error("sep argument must be a string or null");let r=e[1]??new de(-1);if(!(r instanceof de))throw new Error("maxsplit argument must be a number");let n=[];if(t instanceof je){let a=this.value.trimStart();for(let{0:s,index:o}of a.matchAll(/\S+/g)){if(r.value!==-1&&n.length>=r.value&&o!==void 0){n.push(s+a.slice(o+s.length));break}n.push(s)}}else{if(t.value==="")throw new Error("empty separator");n=this.value.split(t.value),r.value!==-1&&n.length>r.value&&n.push(n.splice(r.value).join(t.value))}return new we(n.map(a=>new ee(a)))})],["replace",new Oe(e=>{if(e.length<2)throw new Error("replace() requires at least two arguments");let t=e[0],r=e[1];if(!(t instanceof ee&&r instanceof ee))throw new Error("replace() arguments must be strings");let n;if(e.length>2?e[2].type==="KeywordArgumentsValue"?n=e[2].value.get("count")??new je:n=e[2]:n=new je,!(n instanceof de||n instanceof je))throw new Error("replace() count argument must be a number or null");return new ee(fb(this.value,t.value,r.value,n.value))})]])}},le=class extends Ut{type="BooleanValue"},gb=/[\x7f-\uffff]/g;function Np(e){return e.replace(gb,t=>"\\u"+t.charCodeAt(0).toString(16).padStart(4,"0"))}function zr(e,t={},r=0,n=!0){let{indent:a=null,ensureAscii:s=!1,separators:o=null,sortKeys:i=!1}=t,u,l;switch(o?[u,l]=o:a?(u=",",l=": "):(u=", ",l=": "),e.type){case"NullValue":return"null";case"UndefinedValue":return n?"null":"undefined";case"IntegerValue":case"FloatValue":case"BooleanValue":return JSON.stringify(e.value);case"StringValue":{let c=JSON.stringify(e.value);return s&&(c=Np(c)),c}case"ArrayValue":case"NamespaceValue":case"ObjectValue":{let c=a?" ".repeat(a):"",d=`
`+c.repeat(r),p=d+c;if(e.type==="ArrayValue"){let f=e.value.map(m=>zr(m,t,r+1,n));return a?`[${p}${f.join(`${u}${p}`)}${d}]`:`[${f.join(u)}]`}else{let f=Array.from(e.value.entries());i&&(f=f.sort(([h],[g])=>h.localeCompare(g)));let m=f.map(([h,g])=>{let w=JSON.stringify(h);s&&(w=Np(w));let b=`${w}${l}${zr(g,t,r+1,n)}`;return a?`${p}${b}`:b});return a?`{${m.join(u)}${d}}`:`{${m.join(u)}}`}}default:throw new Error(`Cannot convert to JSON: ${e.type}`)}}var st=class extends Ut{type="ObjectValue";_builtins;__bool__(){return new le(this.value.size>0)}get builtins(){return this._builtins??=new Map([["get",new Oe(([e,t])=>{if(!(e instanceof ee))throw new Error(`Object key must be a string: got ${e.type}`);return this.value.get(e.value)??t??new je})],["items",new Oe(()=>this.items())],["keys",new Oe(()=>this.keys())],["values",new Oe(()=>this.values())],["dictsort",new Oe(e=>{let t=new Map,r=e.filter(i=>i instanceof An?(t=i.value,!1):!0),n=r.at(0)??t.get("case_sensitive")??new le(!1);if(!(n instanceof le))throw new Error("case_sensitive must be a boolean");let a=r.at(1)??t.get("by")??new ee("key");if(!(a instanceof ee))throw new Error("by must be a string");if(!["key","value"].includes(a.value))throw new Error("by must be either 'key' or 'value'");let s=r.at(2)??t.get("reverse")??new le(!1);if(!(s instanceof le))throw new Error("reverse must be a boolean");let o=Array.from(this.value.entries()).map(([i,u])=>new we([new ee(i),u])).sort((i,u)=>{let l=a.value==="key"?0:1,c=i.value[l],d=u.value[l],p=Ku(c,d,n.value);return s.value?-p:p});return new we(o)})]])}items(){return new we(Array.from(this.value.entries()).map(([e,t])=>new we([new ee(e),t])))}keys(){return new we(Array.from(this.value.keys()).map(e=>new ee(e)))}values(){return new we(Array.from(this.value.values()))}toString(){return zr(this,{},0,!1)}},An=class extends st{type="KeywordArgumentsValue"},$u=class extends Ut{type="NamespaceValue";toString(){return zr(this,{},0,!1)}},we=class extends Ut{type="ArrayValue";_builtins;get builtins(){return this._builtins??=new Map([["length",new de(this.value.length)]])}__bool__(){return new le(this.value.length>0)}toString(){return zr(this,{},0,!1)}},zp=class extends we{type="TupleValue"},Oe=class extends Ut{type="FunctionValue"},je=class extends Ut{type="NullValue"},qe=class extends Ut{type="UndefinedValue"};function hb(e){let t;if(e instanceof we?t=e.value:e instanceof ee&&(t=Array.from(e.value,a=>new ee(a))),!t||t.length!==2)throw new Error("namespace expected an object or an iterable of [key, value] pairs");let[r,n]=t;if(!(r instanceof ee))throw new Error("namespace keys must be strings");return[r,n]}var Hp=class{constructor(e){this.parent=e}variables=new Map([["namespace",new Oe(e=>{let t=e.slice(),r;if(t.at(-1)instanceof An&&(r=t.pop()),t.length>1)throw new Error(`namespace expected at most 1 argument, got ${t.length}`);let n=new Map;if(t.length===1){let a=t[0];if(a instanceof st)for(let[s,o]of a.value)n.set(s,o);else if(a instanceof we)for(let s of a.value){let[o,i]=hb(s);n.set(o.value,i)}else throw new Error(`'${a.type}' object is not iterable`)}if(r)for(let[a,s]of r.value)n.set(a,s);return new $u(n)})]]);tests=Hp.TESTS;set(e,t){return this.declareVariable(e,Js(t))}declareVariable(e,t){if(this.variables.has(e))throw new SyntaxError(`Variable already declared: ${e}`);return this.variables.set(e,t),t}setVariable(e,t){return this.variables.set(e,t),t}resolve(e){if(this.variables.has(e))return this;if(this.parent)return this.parent.resolve(e);throw new Error(`Unknown variable: ${e}`)}lookupVariable(e){try{return this.resolve(e).variables.get(e)??new qe}catch{return new qe}}},Bt=Hp;Ow(Bt,"TESTS",new Map([["boolean",e=>e.type==="BooleanValue"],["callable",e=>e instanceof Oe],["odd",e=>{if(!(e instanceof de))throw new Error(`cannot odd on ${e.type}`);return e.value%2!==0}],["even",e=>{if(!(e instanceof de))throw new Error(`cannot even on ${e.type}`);return e.value%2===0}],["false",e=>e.type==="BooleanValue"&&!e.value],["true",e=>e.type==="BooleanValue"&&e.value],["none",e=>e.type==="NullValue"],["string",e=>e.type==="StringValue"],["number",e=>e instanceof de||e instanceof Ie],["integer",e=>e instanceof de],["iterable",e=>e.type==="ArrayValue"||e.type==="StringValue"],["mapping",e=>e instanceof st],["sequence",e=>e instanceof we||e instanceof st||e instanceof ee],["lower",e=>{let t=e.value;return e.type==="StringValue"&&t===t.toLowerCase()}],["upper",e=>{let t=e.value;return e.type==="StringValue"&&t===t.toUpperCase()}],["none",e=>e.type==="NullValue"],["defined",e=>e.type!=="UndefinedValue"],["undefined",e=>e.type==="UndefinedValue"],["equalto",(e,t)=>e.value===t.value],["eq",(e,t)=>e.value===t.value]]));function wb(e){e.set("false",!1),e.set("true",!0),e.set("none",null),e.set("raise_exception",t=>{throw new Error(t)}),e.set("range",ub),e.set("strftime_now",cb),e.set("True",!0),e.set("False",!1),e.set("None",null)}function Ys(e){return e instanceof st||e instanceof $u}function eo(e){return e instanceof de||e instanceof Ie||e instanceof le}function to(e){return e instanceof le?Number(e.value):e.value}function Kp(e,t){let r=t.split("."),n=e;for(let a of r)if(Ys(n))n=n.value.get(a)??new qe;else if(n instanceof we){let s=parseInt(a,10);if(!isNaN(s)&&s>=0&&s<n.value.length)n=n.value[s];else return new qe}else return new qe;return n}function Ku(e,t,r=!1){if(e instanceof je&&t instanceof je)return 0;if(e instanceof je||t instanceof je)throw new Error(`Cannot compare ${e.type} with ${t.type}`);if(e instanceof qe&&t instanceof qe)return 0;if(e instanceof qe||t instanceof qe)throw new Error(`Cannot compare ${e.type} with ${t.type}`);if(eo(e)&&eo(t)){let n=to(e),a=to(t);return n<a?-1:n>a?1:0}if(e.type!==t.type)throw new Error(`Cannot compare different types: ${e.type} and ${t.type}`);if(e.type==="StringValue"){let n=e.value,a=t.value;return r||(n=n.toLowerCase(),a=a.toLowerCase()),n<a?-1:n>a?1:0}else throw new Error(`Cannot compare type: ${e.type}`)}function Uu(e){if(e.type==="Identifier")return{name:e.value};let t=e;return{name:t.key.value,defaultValue:t.value}}function Up(e){return e==="kwargs"||e==="varargs"}function $p(e,t){let r=new Set,n=new Set,a=c=>{Up(c)&&r.add(c)},s=c=>{if(c.type==="Identifier")a(c.value);else if(c.type==="TupleLiteral")for(let d of c.value)s(d)},o=c=>{a(Uu(c).name)},i=c=>{for(let d of c){let p=Uu(d);p.defaultValue&&l(p.defaultValue)}},u=c=>{c.type==="CallExpression"&&l(c.args)},l=c=>{if(Array.isArray(c)){for(let d of c)l(d);return}if(c instanceof Map){for(let[d,p]of c)l(d),l(p);return}if(c instanceof Et)switch(c.type){case"For":{let d=c;if(s(d.loopvar),d.iterable.type==="SelectExpression"){let p=d.iterable;l(p.lhs),l(d.body),l(d.defaultBlock),l(p.test);return}l(d.iterable),l(d.body),l(d.defaultBlock);return}case"Set":{let d=c;s(d.assignee),l(d.value),l(d.body);return}case"Macro":{let d=c;for(let p of d.args)o(p);i(d.args),l(d.body);return}case"CallStatement":{let d=c;l(d.call);for(let p of d.callerArgs??[])o(p);i(d.callerArgs??[]),l(d.body);return}case"FilterStatement":{let d=c;l(d.body),u(d.filter);return}case"Identifier":{let d=c.value;Up(d)&&!r.has(d)&&n.add(d);return}case"MemberExpression":{let d=c;l(d.object),d.computed&&l(d.property);return}case"FilterExpression":{let d=c;l(d.operand),u(d.filter);return}case"TestExpression":l(c.operand);return;case"SelectExpression":{let d=c;l(d.test),l(d.lhs);return}case"KeywordArgumentExpression":l(c.value);return;default:for(let d of Object.values(c))l(d)}};for(let c of e)o(c);return l(t),n}var $a=class{global;constructor(e){this.global=e??new Bt}run(e){return this.evaluate(e,this.global)}evaluateBinaryExpression(e,t){let r=this.evaluate(e.left,t);switch(e.operator.value){case"and":return r.__bool__().value?this.evaluate(e.right,t):r;case"or":return r.__bool__().value?r:this.evaluate(e.right,t)}let n=this.evaluate(e.right,t);switch(e.operator.value){case"==":return new le(r.value==n.value);case"!=":return new le(r.value!=n.value)}if(r instanceof qe||n instanceof qe){if(n instanceof qe&&["in","not in"].includes(e.operator.value))return new le(e.operator.value==="not in");throw new Error(`Cannot perform operation ${e.operator.value} on undefined values`)}else{if(r instanceof je||n instanceof je)throw new Error("Cannot perform operation on null values");if(e.operator.value==="~")return new ee(r.value.toString()+n.value.toString());if(e.operator.value==="**"&&eo(r)&&eo(n)){let a=to(r),s=to(n);if(a===0&&s<0)throw new Error("0.0 cannot be raised to a negative power");let o=a**s;if(!Number.isFinite(o))throw new Error("Exponentiation result is not a finite real number");return r instanceof Ie||n instanceof Ie||s<0?new Ie(o):new de(o)}else if((r instanceof de||r instanceof Ie)&&(n instanceof de||n instanceof Ie)){let a=r.value,s=n.value;switch(e.operator.value){case"+":case"-":case"*":{let o=e.operator.value==="+"?a+s:e.operator.value==="-"?a-s:a*s;return r instanceof Ie||n instanceof Ie?new Ie(o):new de(o)}case"/":return new Ie(a/s);case"//":{let o=Math.floor(a/s);return r instanceof Ie||n instanceof Ie?new Ie(o):new de(o)}case"%":{let o=a%s;return r instanceof Ie||n instanceof Ie?new Ie(o):new de(o)}case"<":return new le(a<s);case">":return new le(a>s);case">=":return new le(a>=s);case"<=":return new le(a<=s)}}else if(r instanceof we&&n instanceof we){if(e.operator.value==="+")return new we(r.value.concat(n.value))}else if(n instanceof we){let a=n.value.find(s=>s.value===r.value)!==void 0;switch(e.operator.value){case"in":return new le(a);case"not in":return new le(!a)}}}if((r instanceof ee||n instanceof ee)&&e.operator.value==="+")return new ee(r.value.toString()+n.value.toString());if(r instanceof ee&&n instanceof ee)switch(e.operator.value){case"in":return new le(n.value.includes(r.value));case"not in":return new le(!n.value.includes(r.value))}if(r instanceof ee&&n instanceof st)switch(e.operator.value){case"in":return new le(n.value.has(r.value));case"not in":return new le(!n.value.has(r.value))}throw new SyntaxError(`Unknown operator "${e.operator.value}" between ${r.type} and ${n.type}`)}evaluateArguments(e,t){let r=[],n=new Map,a=(s,o)=>{if(n.has(s))throw new Error(`Got multiple values for keyword argument '${s}'`);n.set(s,o)};for(let s of e)if(s.type==="SpreadExpression"){let o=s,i=this.evaluate(o.argument,t);if(!(i instanceof we))throw new Error(`Cannot unpack non-iterable type: ${i.type}`);for(let u of i.value)r.push(u)}else s.type!=="KeywordArgumentExpression"&&s.type!=="KeywordSpreadExpression"&&r.push(this.evaluate(s,t));for(let s of e)if(s.type==="KeywordArgumentExpression"){let o=s;a(o.key.value,this.evaluate(o.value,t))}else if(s.type==="KeywordSpreadExpression"){let o=s,i=this.evaluate(o.argument,t);if(!(i instanceof st))throw new Error(`Argument after ** must be a mapping, not ${i.type}`);for(let[u,l]of i.value)a(u,l)}return[r,n]}applyFilter(e,t,r){if(t.type==="Identifier"){let n=t;if(n.value==="safe")return e;if(n.value==="tojson")return new ee(zr(e,{}));if(e instanceof we)switch(n.value){case"list":return e;case"first":return e.value[0];case"last":return e.value[e.value.length-1];case"length":return new de(e.value.length);case"reverse":return new we(e.value.slice().reverse());case"sort":return new we(e.value.slice().sort((a,s)=>Ku(a,s,!1)));case"join":return new ee(e.value.map(a=>a.value).join(""));case"string":return new ee(zr(e,{},0,!1));case"unique":{let a=new Set,s=[];for(let o of e.value)a.has(o.value)||(a.add(o.value),s.push(o));return new we(s)}default:throw new Error(`Unknown ArrayValue filter: ${n.value}`)}else if(e instanceof ee)switch(n.value){case"length":case"upper":case"lower":case"title":case"capitalize":{let a=e.builtins.get(n.value);if(a instanceof Oe)return a.value([],r);if(a instanceof de)return a;throw new Error(`Unknown StringValue filter: ${n.value}`)}case"trim":return new ee(e.value.trim());case"indent":return new ee(e.value.split(`
`).map((a,s)=>s===0||a.length===0?a:"    "+a).join(`
`));case"join":case"string":return e;case"int":{let a=parseInt(e.value,10);return new de(isNaN(a)?0:a)}case"float":{let a=parseFloat(e.value);return new Ie(isNaN(a)?0:a)}default:throw new Error(`Unknown StringValue filter: ${n.value}`)}else if(e instanceof de||e instanceof Ie)switch(n.value){case"abs":return e instanceof de?new de(Math.abs(e.value)):new Ie(Math.abs(e.value));case"int":return new de(Math.floor(e.value));case"float":return new Ie(e.value);case"string":return new ee(e.toString());default:throw new Error(`Unknown NumericValue filter: ${n.value}`)}else if(e instanceof st)switch(n.value){case"items":return new we(Array.from(e.value.entries()).map(([a,s])=>new we([new ee(a),s])));case"length":return new de(e.value.size);default:{let a=e.builtins.get(n.value);if(a)return a instanceof Oe?a.value([],r):a;throw new Error(`Unknown ObjectValue filter: ${n.value}`)}}else if(e instanceof le)switch(n.value){case"bool":return new le(e.value);case"int":return new de(e.value?1:0);case"float":return new Ie(e.value?1:0);case"string":return new ee(e.value?"true":"false");default:throw new Error(`Unknown BooleanValue filter: ${n.value}`)}throw new Error(`Cannot apply filter "${n.value}" to type: ${e.type}`)}else if(t.type==="CallExpression"){let n=t;if(n.callee.type!=="Identifier")throw new Error(`Unknown filter: ${n.callee.type}`);let a=n.callee.value;if(a==="tojson"){let[,s]=this.evaluateArguments(n.args,r),o=s.get("indent")??new je;if(!(o instanceof de||o instanceof je))throw new Error("If set, indent must be a number");let i=s.get("ensure_ascii")??new le(!1);if(!(i instanceof le))throw new Error("If set, ensure_ascii must be a boolean");let u=s.get("sort_keys")??new le(!1);if(!(u instanceof le))throw new Error("If set, sort_keys must be a boolean");let l=s.get("separators")??new je,c=null;if(l instanceof we||l instanceof zp){if(l.value.length!==2)throw new Error("separators must be a tuple of two strings");let[d,p]=l.value;if(!(d instanceof ee)||!(p instanceof ee))throw new Error("separators must be a tuple of two strings");c=[d.value,p.value]}else if(!(l instanceof je))throw new Error("If set, separators must be a tuple of two strings");return new ee(zr(e,{indent:o.value,ensureAscii:i.value,sortKeys:u.value,separators:c}))}else if(a==="join"){let s;if(e instanceof ee)s=Array.from(e.value);else if(e instanceof we)s=e.value.map(l=>l.value);else throw new Error(`Cannot apply filter "${a}" to type: ${e.type}`);let[o,i]=this.evaluateArguments(n.args,r),u=o.at(0)??i.get("separator")??new ee("");if(!(u instanceof ee))throw new Error("separator must be a string");return new ee(s.join(u.value))}else if(a==="int"||a==="float"){let[s,o]=this.evaluateArguments(n.args,r),i=s.at(0)??o.get("default")??(a==="int"?new de(0):new Ie(0));if(e instanceof ee){let u=a==="int"?parseInt(e.value,10):parseFloat(e.value);return isNaN(u)?i:a==="int"?new de(u):new Ie(u)}else{if(e instanceof de||e instanceof Ie)return e;if(e instanceof le)return a==="int"?new de(e.value?1:0):new Ie(e.value?1:0);throw new Error(`Cannot apply filter "${a}" to type: ${e.type}`)}}else if(a==="default"){let[s,o]=this.evaluateArguments(n.args,r),i=s[0]??new ee(""),u=s[1]??o.get("boolean")??new le(!1);if(!(u instanceof le))throw new Error("`default` filter flag must be a boolean");return e instanceof qe||u.value&&!e.__bool__().value?i:e}if(e instanceof we){switch(a){case"sort":{let[s,o]=this.evaluateArguments(n.args,r),i=s.at(0)??o.get("reverse")??new le(!1);if(!(i instanceof le))throw new Error("reverse must be a boolean");let u=s.at(1)??o.get("case_sensitive")??new le(!1);if(!(u instanceof le))throw new Error("case_sensitive must be a boolean");let l=s.at(2)??o.get("attribute")??new je;if(!(l instanceof ee||l instanceof de||l instanceof je))throw new Error("attribute must be a string, integer, or null");let c=d=>{if(l instanceof je)return d;let p=l instanceof de?String(l.value):l.value;return Kp(d,p)};return new we(e.value.slice().sort((d,p)=>{let f=c(d),m=c(p),h=Ku(f,m,u.value);return i.value?-h:h}))}case"selectattr":case"rejectattr":{let s=a==="selectattr";if(e.value.some(d=>!Ys(d)))throw new Error(`\`${a}\` can only be applied to array of objects`);if(n.args.some(d=>d.type!=="StringLiteral"))throw new Error(`arguments of \`${a}\` must be strings`);let[o,i,u]=n.args.map(d=>this.evaluate(d,r)),l;if(i){let d=r.tests.get(i.value);if(!d)throw new Error(`Unknown test: ${i.value}`);l=d}else l=(...d)=>d[0].__bool__().value;let c=e.value.filter(d=>{let p=d.value.get(o.value),f=p?l(p,u):!1;return s?f:!f});return new we(c)}case"map":{let[,s]=this.evaluateArguments(n.args,r);if(s.has("attribute")){let o=s.get("attribute");if(!(o instanceof ee))throw new Error("attribute must be a string");let i=s.get("default"),u=e.value.map(l=>{if(!Ys(l))throw new Error("items in map must be an object");let c=Kp(l,o.value);return c instanceof qe?i??new qe:c});return new we(u)}else throw new Error("`map` expressions without `attribute` set are not currently supported.")}}throw new Error(`Unknown ArrayValue filter: ${a}`)}else if(e instanceof ee){switch(a){case"indent":{let[s,o]=this.evaluateArguments(n.args,r),i=s.at(0)??o.get("width")??new de(4);if(!(i instanceof de))throw new Error("width must be a number");let u=s.at(1)??o.get("first")??new le(!1),l=s.at(2)??o.get("blank")??new le(!1),c=e.value.split(`
`),d=" ".repeat(i.value),p=c.map((f,m)=>!u.value&&m===0||!l.value&&f.length===0?f:d+f);return new ee(p.join(`
`))}case"replace":{let s=e.builtins.get("replace");if(!(s instanceof Oe))throw new Error("replace filter not available");let[o,i]=this.evaluateArguments(n.args,r);return s.value([...o,new An(i)],r)}}throw new Error(`Unknown StringValue filter: ${a}`)}else if(e instanceof st){let s=e.builtins.get(a);if(s&&s instanceof Oe){let[o,i]=this.evaluateArguments(n.args,r);return i.size>0&&o.push(new An(i)),s.value(o,r)}throw new Error(`Unknown ObjectValue filter: ${a}`)}else throw new Error(`Cannot apply filter "${a}" to type: ${e.type}`)}throw new Error(`Unknown filter: ${t.type}`)}evaluateFilterExpression(e,t){let r=this.evaluate(e.operand,t);return this.applyFilter(r,e.filter,t)}evaluateTestExpression(e,t){let r=this.evaluate(e.operand,t),n=t.tests.get(e.test.value);if(!n)throw new Error(`Unknown test: ${e.test.value}`);let a=n(r);return new le(e.negate?!a:a)}evaluateSelectExpression(e,t){return this.evaluate(e.test,t).__bool__().value?this.evaluate(e.lhs,t):new qe}evaluateUnaryExpression(e,t){let r=this.evaluate(e.argument,t);switch(e.operator.value){case"not":return new le(!r.value);case"+":case"-":{let n=e.operator.value==="-"?-1:1;if(r instanceof de||r instanceof Ie||r instanceof le){let a=r instanceof le?r.value?1:0:r.value,s=n*a;return r instanceof Ie?new Ie(s):new de(s)}throw new SyntaxError(`Unknown operator "${e.operator.value}" for ${r.type}`)}default:throw new SyntaxError(`Unknown operator: ${e.operator.value}`)}}evaluateTernaryExpression(e,t){return this.evaluate(e.condition,t).__bool__().value?this.evaluate(e.trueExpr,t):this.evaluate(e.falseExpr,t)}evalProgram(e,t){return this.evaluateBlock(e.body,t)}evaluateBlock(e,t){let r="";for(let n of e){let a=this.evaluate(n,t);a.type!=="NullValue"&&a.type!=="UndefinedValue"&&(r+=a.toString())}return new ee(r)}evaluateIdentifier(e,t){return t.lookupVariable(e.value)}evaluateCallExpression(e,t){let[r,n]=this.evaluateArguments(e.args,t);n.size>0&&r.push(new An(n));let a=this.evaluate(e.callee,t);if(a.type!=="FunctionValue")throw new Error(`Cannot call something that is not a function: got ${a.type}`);return a.value(r,t)}evaluateSliceExpression(e,t,r){if(!(e instanceof we||e instanceof ee))throw new Error("Slice object must be an array or string");let n=this.evaluate(t.start,r),a=this.evaluate(t.stop,r),s=this.evaluate(t.step,r);if(!(n instanceof de||n instanceof qe))throw new Error("Slice start must be numeric or undefined");if(!(a instanceof de||a instanceof qe))throw new Error("Slice stop must be numeric or undefined");if(!(s instanceof de||s instanceof qe))throw new Error("Slice step must be numeric or undefined");return e instanceof we?new we(Fp(e.value,n.value,a.value,s.value)):new ee(Fp(Array.from(e.value),n.value,a.value,s.value).join(""))}evaluateMemberExpression(e,t){let r=this.evaluate(e.object,t),n;if(e.computed){if(e.property.type==="SliceExpression")return this.evaluateSliceExpression(r,e.property,t);n=this.evaluate(e.property,t)}else e.property.type==="IntegerLiteral"?n=new de(e.property.value):n=new ee(e.property.value);let a;if(Ys(r)){if(!(n instanceof ee))throw new Error(`Cannot access property with non-string: got ${n.type}`);a=r.value.get(n.value)??(r instanceof st?r.builtins.get(n.value):void 0)}else if(r instanceof we||r instanceof ee)if(n instanceof de)a=r.value.at(n.value),r instanceof ee&&(a=new ee(r.value.at(n.value)));else if(n instanceof ee)a=r.builtins.get(n.value);else throw new Error(`Cannot access property with non-string/non-number: got ${n.type}`);else{if(!(n instanceof ee))throw new Error(`Cannot access property with non-string: got ${n.type}`);a=r.builtins.get(n.value)}return a instanceof Ut?a:new qe}evaluateSet(e,t){let r=e.value?this.evaluate(e.value,t):this.evaluateBlock(e.body,t);if(e.assignee.type==="Identifier"){let n=e.assignee.value;t.setVariable(n,r)}else if(e.assignee.type==="TupleLiteral"){let n=e.assignee;if(!(r instanceof we))throw new Error(`Cannot unpack non-iterable type in set: ${r.type}`);let a=r.value;if(a.length!==n.value.length)throw new Error(`Too ${n.value.length>a.length?"few":"many"} items to unpack in set`);for(let s=0;s<n.value.length;++s){let o=n.value[s];if(o.type!=="Identifier")throw new Error(`Cannot unpack to non-identifier in set: ${o.type}`);t.setVariable(o.value,a[s])}}else if(e.assignee.type==="MemberExpression"){let n=e.assignee,a=this.evaluate(n.object,t);if(!(a instanceof $u))throw new Error("cannot assign attribute on non-namespace object");if(n.property.type!=="Identifier")throw new Error("Cannot assign to member with non-identifier property");a.value.set(n.property.value,r)}else throw new Error(`Invalid LHS inside assignment expression: ${JSON.stringify(e.assignee)}`);return new je}evaluateIf(e,t){let r=this.evaluate(e.test,t);return this.evaluateBlock(r.__bool__().value?e.body:e.alternate,t)}evaluateFor(e,t){let r=new Bt(t),n,a;if(e.iterable.type==="SelectExpression"){let l=e.iterable;a=this.evaluate(l.lhs,r),n=l.test}else a=this.evaluate(e.iterable,r);if(!(a instanceof we||a instanceof st))throw new Error(`Expected iterable or object type in for loop: got ${a.type}`);a instanceof st&&(a=a.keys());let s=[],o=[];for(let l=0;l<a.value.length;++l){let c=new Bt(r),d=a.value[l],p;if(e.loopvar.type==="Identifier")p=f=>f.setVariable(e.loopvar.value,d);else if(e.loopvar.type==="TupleLiteral"){let f=e.loopvar;if(d.type!=="ArrayValue")throw new Error(`Cannot unpack non-iterable type: ${d.type}`);let m=d;if(f.value.length!==m.value.length)throw new Error(`Too ${f.value.length>m.value.length?"few":"many"} items to unpack`);p=h=>{for(let g=0;g<f.value.length;++g){if(f.value[g].type!=="Identifier")throw new Error(`Cannot unpack non-identifier type: ${f.value[g].type}`);h.setVariable(f.value[g].value,m.value[g])}}}else throw new Error(`Invalid loop variable(s): ${e.loopvar.type}`);n&&(p(c),!this.evaluate(n,c).__bool__().value)||(s.push(d),o.push(p))}let i="",u=!0;for(let l=0;l<s.length;++l){let c=new Map([["index",new de(l+1)],["index0",new de(l)],["revindex",new de(s.length-l)],["revindex0",new de(s.length-l-1)],["first",new le(l===0)],["last",new le(l===s.length-1)],["length",new de(s.length)],["previtem",l>0?s[l-1]:new qe],["nextitem",l<s.length-1?s[l+1]:new qe]]);r.setVariable("loop",new st(c)),o[l](r);try{let d=this.evaluateBlock(e.body,r);i+=d.value}catch(d){if(d instanceof jp)continue;if(d instanceof Qp)break;throw d}u=!1}if(u){let l=this.evaluateBlock(e.defaultBlock,r);i+=l.value}return new ee(i)}bindMacroArguments(e,t,r,n,a){let s=n.slice(),o=new Map;s.at(-1)instanceof An&&(o=new Map(s.pop().value));let i=[];for(let u=0;u<t.length;++u){let{name:l,defaultValue:c}=Uu(t[u]),d=s[u];d===void 0&&o.has(l)&&(d=o.get(l),o.delete(l)),d===void 0&&c!==void 0&&i.push([l,c]),a.setVariable(l,d??new qe)}if(r.has("kwargs"))a.setVariable("kwargs",new st(o));else if(o.size>0)throw new Error(`macro ${e} takes no keyword argument '${o.keys().next().value}'`);if(r.has("varargs"))a.setVariable("varargs",new we(s.slice(t.length)));else if(s.length>t.length)throw new Error(`macro ${e} takes not more than ${t.length} argument(s)`);for(let[u,l]of i)a.setVariable(u,this.evaluate(l,a))}evaluateMacro(e,t){let r=$p(e.args,e.body);return t.setVariable(e.name.value,new Oe((n,a)=>{let s=new Bt(a);return this.bindMacroArguments(`'${e.name.value}'`,e.args,r,n,s),this.evaluateBlock(e.body,s)})),new je}evaluateCallStatement(e,t){let r=e.callerArgs??[],n=$p(r,e.body),a=new Oe(l=>{let c=new Bt(t);return this.bindMacroArguments("None",r,n,l,c),this.evaluateBlock(e.body,c)}),[s,o]=this.evaluateArguments(e.call.args,t);s.push(new An(o));let i=this.evaluate(e.call.callee,t);if(i.type!=="FunctionValue")throw new Error(`Cannot call something that is not a function: got ${i.type}`);let u=new Bt(t);return u.setVariable("caller",a),i.value(s,u)}evaluateFilterStatement(e,t){let r=this.evaluateBlock(e.body,t);return this.applyFilter(r,e.filter,t)}evaluate(e,t){if(!e)return new qe;switch(e.type){case"Program":return this.evalProgram(e,t);case"Set":return this.evaluateSet(e,t);case"If":return this.evaluateIf(e,t);case"For":return this.evaluateFor(e,t);case"Macro":return this.evaluateMacro(e,t);case"CallStatement":return this.evaluateCallStatement(e,t);case"Break":throw new Qp;case"Continue":throw new jp;case"IntegerLiteral":return new de(e.value);case"FloatLiteral":return new Ie(e.value);case"StringLiteral":return new ee(e.value);case"ArrayLiteral":return new we(e.value.map(r=>this.evaluate(r,t)));case"TupleLiteral":return new zp(e.value.map(r=>this.evaluate(r,t)));case"ObjectLiteral":{let r=new Map;for(let[n,a]of e.value){let s=this.evaluate(n,t);if(!(s instanceof ee))throw new Error(`Object keys must be strings: got ${s.type}`);r.set(s.value,this.evaluate(a,t))}return new st(r)}case"Identifier":return this.evaluateIdentifier(e,t);case"CallExpression":return this.evaluateCallExpression(e,t);case"MemberExpression":return this.evaluateMemberExpression(e,t);case"UnaryExpression":return this.evaluateUnaryExpression(e,t);case"BinaryExpression":return this.evaluateBinaryExpression(e,t);case"FilterExpression":return this.evaluateFilterExpression(e,t);case"FilterStatement":return this.evaluateFilterStatement(e,t);case"TestExpression":return this.evaluateTestExpression(e,t);case"SelectExpression":return this.evaluateSelectExpression(e,t);case"Ternary":return this.evaluateTernaryExpression(e,t);case"Comment":return new je;default:throw new SyntaxError(`Unknown node type: ${e.type}`)}}};function Js(e){switch(typeof e){case"number":return Number.isInteger(e)?new de(e):new Ie(e);case"string":return new ee(e);case"boolean":return new le(e);case"undefined":return new qe;case"object":return e===null?new je:Array.isArray(e)?new we(e.map(Js)):new st(new Map(Object.entries(e).map(([t,r])=>[t,Js(r)])));case"function":return new Oe((t,r)=>{let n=e(...t.map(a=>a.value))??null;return Js(n)});default:throw new Error(`Cannot convert to runtime value: ${e}`)}}var Ve=`
`,bb="{%- ",vb=" -%}",Ke=Object.freeze({CONDITIONAL:0,LOGICAL_OR:1,LOGICAL_AND:2,LOGICAL_NOT:3,COMPARISON:4,ADDITIVE:5,MULTIPLICATIVE:6,EXPONENTIATION:7,TEST:8,FILTER:9,UNARY_SIGN:10,ATOM:11});function Vp(e){switch(e.operator.type){case"ExponentiationBinaryOperator":return Ke.EXPONENTIATION;case"MultiplicativeBinaryOperator":return Ke.MULTIPLICATIVE;case"AdditiveBinaryOperator":return Ke.ADDITIVE;case"ComparisonBinaryOperator":return Ke.COMPARISON;case"Identifier":return e.operator.value==="and"?Ke.LOGICAL_AND:e.operator.value==="in"||e.operator.value==="not in"?Ke.COMPARISON:Ke.LOGICAL_OR}return Ke.LOGICAL_OR}function Zp(e){switch(e.type){case"SelectExpression":case"Ternary":return Ke.CONDITIONAL;case"BinaryExpression":return Vp(e);case"UnaryExpression":return e.operator.value==="not"?Ke.LOGICAL_NOT:Ke.UNARY_SIGN;case"TestExpression":return Ke.TEST;case"FilterExpression":return Ke.FILTER;default:return Ke.ATOM}}function zt(e,t){let r=Ne(e);return Zp(e)<t?`(${r})`:r}function _b(e,t="	"){let r=typeof t=="number"?" ".repeat(t):t;return Kt(e.body,0,r).replace(/\n$/,"")}function et(...e){return bb+e.join(" ")+vb}function Kt(e,t,r){return e.map(n=>yb(n,t,r)).join(Ve)}function ro(e){return e.map(t=>Ne(t)).join(", ")}function yb(e,t,r){let n=r.repeat(t);switch(e.type){case"Program":return Kt(e.body,t,r);case"If":return Ab(e,t,r);case"For":return kb(e,t,r);case"Set":return Sb(e,t,r);case"Macro":return xb(e,t,r);case"Break":return n+et("break");case"Continue":return n+et("continue");case"CallStatement":return Tb(e,t,r);case"FilterStatement":return Cb(e,t,r);case"Comment":return n+"{# "+e.value+" #}";default:return n+"{{- "+Ne(e)+" -}}"}}function Ab(e,t,r){let n=r.repeat(t),a=[],s=e;for(;s&&(a.push({test:s.test,body:s.body}),s.alternate.length===1&&s.alternate[0].type==="If");)s=s.alternate[0];let o=n+et("if",Ne(a[0].test))+Ve+Kt(a[0].body,t+1,r);for(let i=1;i<a.length;++i)o+=Ve+n+et("elif",Ne(a[i].test))+Ve+Kt(a[i].body,t+1,r);return s&&s.alternate.length>0&&(o+=Ve+n+et("else")+Ve+Kt(s.alternate,t+1,r)),o+=Ve+n+et("endif"),o}function kb(e,t,r){let n=r.repeat(t),a="";if(e.iterable.type==="SelectExpression"){let o=e.iterable;a=`${Ne(o.lhs)} if ${Ne(o.test)}`}else a=Ne(e.iterable);let s=n+et("for",Ne(e.loopvar),"in",a)+Ve+Kt(e.body,t+1,r);return e.defaultBlock.length>0&&(s+=Ve+n+et("else")+Ve+Kt(e.defaultBlock,t+1,r)),s+=Ve+n+et("endfor"),s}function Sb(e,t,r){let n=r.repeat(t),a=Ne(e.assignee),s=e.value?Ne(e.value):"",o=n+et("set",`${a}${e.value?" = "+s:""}`);return e.body.length===0?o:o+Ve+Kt(e.body,t+1,r)+Ve+n+et("endset")}function xb(e,t,r){let n=r.repeat(t),a=ro(e.args);return n+et("macro",`${e.name.value}(${a})`)+Ve+Kt(e.body,t+1,r)+Ve+n+et("endmacro")}function Tb(e,t,r){let n=r.repeat(t),a=e.callerArgs&&e.callerArgs.length>0?`(${ro(e.callerArgs)})`:"",s=Ne(e.call),o=n+et(`call${a}`,s)+Ve;return o+=Kt(e.body,t+1,r)+Ve,o+=n+et("endcall"),o}function Cb(e,t,r){let n=r.repeat(t),a=e.filter.type==="Identifier"?e.filter.value:Ne(e.filter),s=n+et("filter",a)+Ve;return s+=Kt(e.body,t+1,r)+Ve,s+=n+et("endfilter"),s}function Ne(e){switch(e.type){case"SpreadExpression":return`*${Ne(e.argument)}`;case"KeywordSpreadExpression":return`**${Ne(e.argument)}`;case"Identifier":return e.value;case"IntegerLiteral":return`${e.value}`;case"FloatLiteral":{let t=e.value;return Object.is(t,-0)?"-0.0":t%1===0?t.toFixed(1):t.toString()}case"StringLiteral":return JSON.stringify(e.value);case"BinaryExpression":{let t=e,r=Vp(t),n=zt(t.left,r),a=zt(t.right,r+1);return`${n} ${t.operator.value} ${a}`}case"UnaryExpression":{let t=e,r=t.argument.type==="UnaryExpression"?Zp(t):Ke.ATOM;return t.operator.value+(t.operator.value==="not"?" ":"")+zt(t.argument,r)}case"CallExpression":{let t=e,r=ro(t.args);return`${Ne(t.callee)}(${r})`}case"MemberExpression":{let t=e,r=zt(t.object,Ke.ATOM),n=Ne(t.property);return!t.computed&&t.property.type!=="Identifier"&&t.property.type!=="IntegerLiteral"&&(n=`(${n})`),t.computed?`${r}[${n}]`:`${r}.${n}`}case"FilterExpression":{let t=e,r=zt(t.operand,Ke.FILTER);return t.filter.type==="CallExpression"?`${r} | ${Ne(t.filter)}`:`${r} | ${t.filter.value}`}case"SelectExpression":{let t=e;return`${zt(t.lhs,Ke.LOGICAL_OR)} if ${zt(t.test,Ke.LOGICAL_OR)}`}case"TestExpression":{let t=e;return`${zt(t.operand,Ke.TEST)} is${t.negate?" not":""} ${t.test.value}`}case"ArrayLiteral":case"TupleLiteral":{let t=ro(e.value),r=e.type==="ArrayLiteral"?"[]":"()";return`${r[0]}${t}${r[1]}`}case"ObjectLiteral":return`{${Array.from(e.value.entries()).map(([r,n])=>`${Ne(r)}: ${Ne(n)}`).join(", ")}}`;case"SliceExpression":{let t=e,r=t.start?Ne(t.start):"",n=t.stop?Ne(t.stop):"",a=t.step?`:${Ne(t.step)}`:"";return`${r}:${n}${a}`}case"KeywordArgumentExpression":{let t=e;return`${t.key.value}=${Ne(t.value)}`}case"Ternary":{let t=e;return`${zt(t.trueExpr,Ke.LOGICAL_OR)} if ${zt(t.condition,Ke.LOGICAL_OR)} else ${zt(t.falseExpr,Ke.CONDITIONAL)}`}default:throw new Error(`Unknown expression type: ${e.type}`)}}var Xp=class{parsed;constructor(e){let t=no(e,{lstrip_blocks:!0,trim_blocks:!0});this.parsed=ao(t)}render(e){let t=new Bt;if(wb(t),e)for(let[a,s]of Object.entries(e))t.set(a,s);return new $a(t).run(this.parsed).value}format(e){return _b(this.parsed,e?.indent||"	")}};var so=new Set(["repeat","tunables","dtypes","logicalDtypes","tensorDtypes","ranks","shapes","present","args","attrs","device","env","constants","derive","source","DISPATCH_FOLD_WIDTH","f16Ok","subgroupMatrixConfigAvailable","ceil","floor","min","max","pow","ceilDiv","pow2ceil","log2ceil","numel","dim","rows","outer","inner","prefix","suffix","fill","moveAxis","permuteShape","reversedAxes","broadcastShape","matmulShape","broadcastable","sameShape","hasAxis","axisValue","isUniqueIntList","has","dtypeBytes","onnxDtypeCode","pick","true","false","none","True","False","None","range","loop","namespace","dict"]);function Tr(e){switch(typeof e){case"string":return Yp.test(e)?`"${e}"`:JSON.stringify(e);case"number":return Number.isNaN(e)?"NaN":e===Number.POSITIVE_INFINITY?"Infinity":e===Number.NEGATIVE_INFINITY?"-Infinity":e===0?Object.is(e,-0)?"-0":"0":`${e}`;case"object":{if(e===null)return"null";if(Array.isArray(e)){let a="[";for(let s=0;s<e.length;s++)s>0&&(a+=","),a+=Tr(e[s])??"";return`${a}]`}let t=e,r=Pb(t),n="{";for(let a=0;a<r.length;a++){let s=r[a];a>0&&(n+=","),n+=`${Yp.test(s)?`"${s}"`:JSON.stringify(s)}:${Tr(t[s])}`}return`${n}}`}default:return JSON.stringify(e)}}var Yp=/^[\x20\x21\x23-\x5b\x5d-\x7e]*$/;function Pb(e){let t=Object.keys(e);if(t.length>16)return t.sort();for(let r=1;r<t.length;r++){let n=t[r],a=r-1;for(;a>=0&&t[a]>n;)t[a+1]=t[a],a--;t[a+1]=n}return t}var Ib=Object.freeze(new Set(Object.getOwnPropertyNames(Object.prototype)));function oo(e){return!Ib.has(e)}function Hu(e,t=""){if(Array.isArray(e)){for(let r=0;r<e.length;r+=1){let n=e[r];if(n===null||typeof n!="object")continue;let a=Hu(n,`${t}[${r}]`);if(a!==void 0)return a}return}if(!(e===null||typeof e!="object"))for(let[r,n]of Object.entries(e)){let a=t===""?r:`${t}.${r}`;if(!oo(r))return a;if(n===null||typeof n!="object")continue;let s=Hu(n,a);if(s!==void 0)return s}}function Jp(e,t){let r=Hu(e);if(r!==void 0)throw new Error(`${t}: key "${r}" shadows an Object.prototype member; rename it`)}var ef=[{code:1,label:"F32",logical:"float32",tensor:"tensor(float)"},{code:2,label:"U8",logical:"uint8",tensor:"tensor(uint8)"},{code:3,label:"I8",logical:"int8",tensor:"tensor(int8)"},{code:5,label:"I16",logical:"int16",tensor:"tensor(int16)"},{code:6,label:"I32",logical:"int32",tensor:"tensor(int32)"},{code:7,label:"I64",logical:"int64",tensor:"tensor(int64)"},{code:9,label:"BOOL",logical:"bool",tensor:"tensor(bool)"},{code:10,label:"F16",logical:"float16",tensor:"tensor(float16)"},{code:12,label:"U32",logical:"uint32",tensor:"tensor(uint32)"}],Bb=new Map(ef.map(e=>[e.code,e])),tf=new Map,Eb=new Map;for(let e of ef)tf.set(e.logical,e),Eb.set(e.tensor,e.logical);function rf(e){return Bb.get(e)}function io(e){let t=tf.get(e);if(t===void 0)throw new Error(`Unsupported ONNX logical dtype: ${String(e)}`);return t.code}function uo(e,t=0){return`${(e/(1024*1024)).toFixed(t)} MiB`}function lr(e,t,r){return e===1?t:r!==void 0?r:/(s|x|z|ch|sh)$/.test(t)?`${t}es`:`${t}s`}var a7=Object.freeze([]);function Rb(e){return e!==null&&typeof e=="object"&&typeof e.type=="string"}function kn(e){if(!Rb(e))throw new Error("expected a typed Jinja AST node");return e}function be(e,t){try{return kn(e[t])}catch{throw new Error(`Jinja ${e.type??"node"}.${t} is not an AST node`)}}function tt(e,t){let r=e[t];if(!Array.isArray(r))throw new Error(`Jinja ${e.type??"node"}.${t} is not an AST node array`);for(let n of r)kn(n);return r}function ot(e){if(e.type!=="Identifier"||typeof e.value!="string")throw new Error(`Jinja ${e.type??"node"} is not an Identifier`);return e.value}function nf(e){return e.type==="KeywordArgumentExpression"?ot(be(e,"key")):ot(e)}function Vu(e,t){if(e.type==="Identifier"){t(ot(e));return}if(e.type==="TupleLiteral"){for(let r of tt(e,"value"))Vu(r,t);return}throw new Error(`Jinja assignment target ${e.type??"node"} is not a lexical binding`)}function Sn(e){if(e.type==="Identifier")return[ot(e)];if(e.type!=="MemberExpression")return;let t=Sn(be(e,"object"));if(t===void 0)return;let r=be(e,"property");return e.computed===!0?r.type==="StringLiteral"&&typeof r.value=="string"?[...t,r.value]:void 0:r.type==="Identifier"?[...t,ot(r)]:void 0}function ia(e){return e.type==="Identifier"?ot(e):e.type==="MemberExpression"?ia(be(e,"object")):void 0}function Rt(e,t){if(!e||typeof e!="object")return;if(Array.isArray(e)){for(let a of e)Rt(a,t);return}if(e instanceof Map){for(let[a,s]of e)Rt(a,t),Rt(s,t);return}let r=e;if(t(r))return;let n=r.type==="MemberExpression"&&r.computed!==!0;for(let a of Object.keys(r))a==="type"||n&&a==="property"||Rt(r[a],t)}function co(e,t){let r=!1;return Rt(e,n=>r?!0:(r=t(n),r)),r}function af(e){return co(e,t=>t.type==="Break"||t.type==="Continue")}function sf(e){if(e.type==="ArrayLiteral"||e.type==="TupleLiteral")return tt(e,"value").length===0;if(e.type==="StringLiteral")return e.value==="";if(e.type==="ObjectLiteral"){if(!(e.value instanceof Map))throw new Error("Jinja ObjectLiteral.value is not a Map");return e.value.size===0}return!1}var lo=class e{bindings;parent;constructor(t,r){this.parent=t,this.bindings=new Map(r)}child(){return new e(this)}fork(){return new e(this.parent,this.bindings)}define(t,r){this.bindings.set(t,r)}resolve(t){return this.bindings.get(t)??this.parent?.resolve(t)}replaceResolved(t,r){return this.bindings.has(t)?(this.bindings.set(t,r),!0):this.parent?.replaceResolved(t,r)??!1}localEntries(){return this.bindings}};var of=Object.freeze({}),Ha=Object.freeze({dynamicPossible:!0});function fo(e,t){let r=be(e,"object");r.type==="MemberExpression"&&fo(r,t),e.computed===!0&&t(be(e,"property"))}function po(e,t,r=Ha){Vu(e,n=>t.define(n,r))}function uf(e,t){if(e.type!=="Identifier")return Ha;let r=t.resolve(ot(e));if((r?.macros?.length??0)===0)return Ha;let n=r?.externalPossible===!0||r?.dynamicPossible===!0;return Object.freeze({...n?{dynamicPossible:!0}:{},macros:r.macros})}function lf(e,t){return co(e,r=>r.type==="Identifier"&&(t.resolve(ot(r))?.macros?.length??0)>0)}function cf(e,t,r){let n=new Set([...t.localEntries().keys(),...r.localEntries().keys()]);for(let a of n){let s=e.resolve(a),o=t.localEntries().get(a)??s,i=r.localEntries().get(a)??s;if(o===i&&o!==void 0){e.define(a,o);continue}let u=[...new Set([...o?.macros??[],...i?.macros??[]])],l=o===void 0||i===void 0||o.externalPossible===!0||i.externalPossible===!0,c=o?.dynamicPossible===!0||i?.dynamicPossible===!0;e.define(a,Object.freeze({...l?{externalPossible:!0}:{},...c?{dynamicPossible:!0}:{},...u.length>0?{macros:Object.freeze(u)}:{}}))}}function df(e){let t=kn(e);if(t.type!=="Program")throw new Error("Jinja template dependency analysis expected a Program AST");let r=new Set,n=new Map,a=new Set,s=(p,f)=>{if(r.add(p),f===void 0){a.add(p),n.delete(p);return}if(a.has(p))return;let m=n.get(p);m===void 0&&(m=new Set,n.set(p,m)),m.add(f)},o=new Set,i=!1,u,l=(p,f,m,h)=>{if(o.has(p)){i=!0;return}o.add(p);try{let g=f.child();m!==void 0&&g.define("caller",Object.freeze({macros:Object.freeze([m])}));let w=[],b=new Map,v=h===void 0;for(let y of h??[])y.type==="KeywordArgumentExpression"?b.set(ot(be(y,"key")),be(y,"value")):y.type==="SpreadExpression"?v=!0:w.push(y);for(let y=0;y<p.args.length;y+=1){let S=p.args[y],C=nf(S),k=w[y],B=f;if(S.type==="KeywordArgumentExpression"&&(k??=b.get(C),k===void 0)){let P=be(S,"value");c(P,g),k=v?void 0:P,B=g}g.define(C,k===void 0?Ha:uf(k,B))}u(p.body,g)}finally{o.delete(p)}},c=(p,f,m,h)=>{switch(p.type){case"Identifier":{let g=ot(p),w=f.resolve(g);if((w===void 0||w.externalPossible===!0)&&s(g),h!==void 0)for(let b of w?.macros??[])l(b,f,m,h);return}case"IntegerLiteral":case"FloatLiteral":case"StringLiteral":return;case"MemberExpression":{let g=Sn(p),w=g?.[0]??ia(p),b=w===void 0?void 0:f.resolve(w);if(w!==void 0&&(b===void 0||b.externalPossible===!0)){g!==void 0&&g.length>=2?s(w,g[1]):s(w),fo(p,v=>c(v,f));return}c(be(p,"object"),f),p.computed===!0&&c(be(p,"property"),f);return}case"CallExpression":{let g=be(p,"callee"),w=tt(p,"args"),b=g.type==="MemberExpression"?ia(g):void 0,v=b===void 0?void 0:f.resolve(b),y=g.type==="Identifier"?f.resolve(ot(g)):void 0,S=g.type==="Identifier"?y!==void 0&&(y.dynamicPossible===!0||(y.macros?.length??0)===0):b!==void 0&&v!==void 0,C=g.type==="Identifier"?y===void 0||y.externalPossible===!0:b!==void 0&&(v===void 0||v.externalPossible===!0),k=g.type!=="Identifier"&&!(g.type==="MemberExpression"&&b!==void 0);(S||k||C&&w.some(B=>lf(B,f)))&&(i=!0),b!==void 0&&(v===void 0||v.externalPossible===!0)?(s(b),fo(g,B=>c(B,f))):c(g,f,m,w);for(let B of w)c(B,f);return}case"ArrayLiteral":case"TupleLiteral":for(let g of tt(p,"value"))c(g,f);return;case"ObjectLiteral":if(!(p.value instanceof Map))throw new Error("Jinja ObjectLiteral.value is not a Map");for(let[g,w]of p.value)c(kn(g),f),c(kn(w),f);return;case"BinaryExpression":c(be(p,"left"),f),c(be(p,"right"),f);return;case"FilterExpression":{c(be(p,"operand"),f);let g=be(p,"filter");if(g.type==="CallExpression")for(let w of tt(g,"args"))c(w,f);return}case"TestExpression":{let g=be(p,"operand");c(g,f);return}case"SelectExpression":c(be(p,"test"),f),c(be(p,"lhs"),f);return;case"UnaryExpression":case"SpreadExpression":c(be(p,"argument"),f);return;case"SliceExpression":for(let g of["start","stop","step"])p[g]!==void 0&&c(be(p,g),f);return;case"KeywordArgumentExpression":c(be(p,"value"),f);return;case"Ternary":c(be(p,"condition"),f),c(be(p,"trueExpr"),f),c(be(p,"falseExpr"),f);return;default:{let g=!0;Rt(p,w=>{if(g)return g=!1,!1;if(w.type==="Identifier"&&typeof w.value=="string"){let b=f.resolve(w.value);(b===void 0||b.externalPossible===!0)&&s(w.value);for(let v of b?.macros??[])l(v,f);return!0}return!1})}}};if(u=(p,f)=>{for(let m of p)switch(m.type){case"Comment":case"Break":case"Continue":break;case"If":{c(be(m,"test"),f);let h=f.fork(),g=f.fork();u(tt(m,"body"),h),u(tt(m,"alternate"),g),cf(f,h,g);break}case"For":{let h=be(m,"iterable"),g=be(m,"loopvar");if(h.type==="SelectExpression"){c(be(h,"lhs"),f);let y=f.child();po(g,y),c(be(h,"test"),y)}else c(h,f);let w=f.child();po(g,w),w.define("loop",of);let b=tt(m,"body");u(b,w),b.some(y=>co(y,S=>S.type==="Macro"))&&(i=!0);let v=f.child();b.some(af)&&!sf(h)&&cf(v,w,f.child()),u(tt(m,"defaultBlock"),v);break}case"Set":{let h=be(m,"assignee"),g=m.value===null?void 0:be(m,"value");g===void 0?u(tt(m,"body"),f):c(g,f),h.type==="MemberExpression"?(c(be(h,"object"),f),h.computed===!0&&c(be(h,"property"),f),g!==void 0&&lf(g,f)&&(i=!0)):po(h,f,g===void 0?Ha:uf(g,f));break}case"Macro":{let h=ot(be(m,"name")),g=Object.freeze({args:tt(m,"args"),body:tt(m,"body")});f.define(h,Object.freeze({macros:Object.freeze([g])}));break}case"CallStatement":{let h=be(m,"call"),g=Array.isArray(m.callerArgs)?m.callerArgs.map(kn):[],w=Object.freeze({args:Object.freeze(g),body:tt(m,"body")});c(h,f,w);let b=be(h,"callee"),v=b.type==="Identifier"?f.resolve(ot(b)):void 0;if(v===void 0||v.externalPossible===!0||v.dynamicPossible===!0||(v.macros?.length??0)===0){i=!0;let S=f.child();S.define("caller",of);for(let C of g)po(C,S);u(w.body,S)}break}case"FilterStatement":{u(tt(m,"body"),f);let h=be(m,"filter");if(h.type==="CallExpression")for(let g of tt(h,"args"))c(g,f);break}default:c(m,f)}},u(tt(t,"body"),new lo),i){let p=f=>{if(f.type==="CallExpression"){let m=be(f,"callee"),h=m.type==="MemberExpression"?ia(m):void 0;return h!==void 0&&s(h),!1}if(f.type==="MemberExpression"){let m=Sn(f),h=m?.[0]??ia(f);return h!==void 0&&(m!==void 0&&m.length>=2?s(h,m[1]):s(h)),fo(f,g=>Rt(g,p)),!0}return f.type==="Identifier"&&typeof f.value=="string"?(s(f.value),!0):!1};Rt(t,p)}let d=new Map;for(let p of r)d.set(p,a.has(p)?null:Object.freeze([...n.get(p)??[]].sort()));return Object.freeze({roots:r,namespaceReads:d})}var Gb;function mo(){return Gb}var Lb=8e6,ua=new Map,Zu=0,qb=0;function la(e){return e==null?e:Array.isArray(e)?e.map(la):e instanceof Map?Object.fromEntries([...e].map(([t,r])=>[t,la(r)])):typeof e=="object"&&"value"in e?e.type==="NullValue"?null:la(e.value):e}function go(e,t){if(typeof e=="function")return!0;if(e===null||typeof e!="object")return!1;if(e.type==="FunctionValue")return!0;if(Array.isArray(e)){let r=t??new Set;return r.has(e)?!1:(r.add(e),e.some(n=>go(n,r)))}if(e instanceof Map){let r=t??new Set;if(r.has(e))return!1;r.add(e);for(let n of e.values())if(go(n,r))return!0;return!1}if("value"in e){let r=e.value;if(r===null||typeof r!="object"&&typeof r!="function")return!1;let n=t??new Set;return n.has(e)?!1:(n.add(e),go(r,n))}return!1}function Pe(e){return((...t)=>e(...t.map(la)))}function gf(e,t){let r=Math.max(e.length,t.length),n=new Array(r);for(let a=0;a<r;++a){let s=a-(r-e.length),o=a-(r-t.length),i=s<0?1:e[s],u=o<0?1:t[o];if(i!==u&&i!==1&&u!==1)throw new Error(`Shapes [${e}] and [${t}] are not broadcast-compatible`);n[a]=i===1?u:i}return n}function Ob(e,t){if(e.length===0||t.length===0)throw new Error("MatMul inputs must each have rank >= 1");let r=e.length===1,n=t.length===1,a=e.at(-1),s=n?t[0]:t.at(-2);if(a!==s)throw new Error(`MatMul inner dimensions do not match (${a} != ${s})`);let o=gf(r?[]:e.slice(0,-2),n?[]:t.slice(0,-2));return r&&n?o:r?[...o,t.at(-1)]:n?[...o,e.at(-2)]:[...o,e.at(-2),t.at(-1)]}function Db(e,t,r){if(!Array.isArray(e)||e.length===0)throw new Error("moveAxis() expects a non-empty shape");if(!Number.isInteger(t)||!Number.isInteger(r))throw new Error("moveAxis() axes must be integers");let n=e.length;if(t<-n||t>=n||r<-n||r>=n)throw new Error(`moveAxis() axes must be in [-${n}, ${n-1}]`);let a=t<0?t+n:t,s=r<0?r+n:r,o=[...e],[i]=o.splice(a,1);return o.splice(s,0,i),o}function Mb(e,t){if(!Array.isArray(e))throw new Error("permuteShape() expects a shape");let r=e.length;if(!Array.isArray(t)||t.length!==r)throw new Error(`permuteShape() expects one axis per shape entry, got ${Array.isArray(t)?t.length:"a non-list"} for rank ${r}`);let n=new Set,a=new Array(r);for(let s=0;s<r;++s){let o=t[s];if(!Number.isInteger(o)||o<-r||o>=r)throw new Error(`permuteShape() axes must be in [-${r}, ${r-1}]`);let i=o<0?o+r:o;if(n.has(i))throw new Error(`permuteShape() axes must be unique, saw ${i} twice`);n.add(i),a[s]=e[i]}return a}var wo=Object.freeze({ceil:Pe(e=>Math.ceil(e)),floor:Pe(e=>Math.floor(e)),min:Pe((...e)=>Math.min(...e)),max:Pe((...e)=>Math.max(...e)),pow:Pe((e,t)=>Math.pow(e,t)),ceilDiv:Pe((e,t)=>Math.ceil(e/t)),pow2ceil:Pe(e=>e<=1?1:2**Math.ceil(Math.log2(e))),log2ceil:Pe(e=>e<=1?0:Math.ceil(Math.log2(e))),numel:Pe(e=>e.reduce((t,r)=>t*r,1)),dim:Pe((e,t)=>{let r=(t%e.length+e.length)%e.length;return e[r]}),rows:Pe((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.reduce((n,a,s)=>s===r?n:n*a,1)}),outer:Pe((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.slice(0,r).reduce((n,a)=>n*a,1)}),inner:Pe((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.slice(r+1).reduce((n,a)=>n*a,1)}),prefix:Pe((e,t)=>e.slice(0,Math.max(0,t))),suffix:Pe((e,t)=>e.slice(Math.max(0,t))),fill:Pe((e,t)=>{if(!Number.isInteger(t)||t<0)throw new Error("fill() count must be a non-negative integer");return Array.from({length:t},()=>e)}),moveAxis:Pe(Db),permuteShape:Pe(Mb),reversedAxes:Pe(e=>{if(!Number.isInteger(e)||e<0)throw new Error("reversedAxes() expects a non-negative integer rank");return Array.from({length:e},(t,r)=>e-1-r)}),broadcastShape:Pe(gf),matmulShape:Pe(Ob),broadcastable:Pe((e,t)=>{if(e.length>t.length)return!1;let r=t.length-e.length;return e.every((n,a)=>n===1||n===t[a+r])}),sameShape:Pe((e,t)=>e.length===t.length&&e.every((r,n)=>r===t[n])),hasAxis:Pe((e,t,r)=>{let n=(t%r+r)%r;return e.some(a=>(a%r+r)%r===n)}),axisValue:Pe((e,t,r,n,a)=>{let s=(r%n+n)%n,o=t.findIndex(i=>(i%n+n)%n===s);return o>=0&&o<e.length?e[o]:a}),isUniqueIntList:Pe((e,t,r,n=0)=>{if(!Array.isArray(e)||!Number.isInteger(t)||!Number.isInteger(r)||r<t||!Number.isInteger(n)||n<0)return!1;let a=new Set;for(let s of e){if(!Number.isInteger(s)||s<t||s>=r)return!1;let o=n===0?s:(s%n+n)%n;if(a.has(o))return!1;a.add(o)}return!0}),has:Pe((e,t)=>e instanceof Map?e.has(t):!!(e&&Object.prototype.hasOwnProperty.call(e,t))),dtypeBytes:Pe(e=>{let t=Qe[bo(e)];if(!t)throw new Error(`Unsupported dtype for dtypeBytes(): ${e}`);return t.storageByteSize}),onnxDtypeCode:Pe(e=>io(e==="int64"?e:Le(e))),pick:Pe((e,t)=>{if(!Array.isArray(e))throw new Error("pick() expects a list of [condition, value] pairs as its first argument");for(let r of e){if(!Array.isArray(r)||r.length<2)throw new Error("pick() entries must be [condition, value] pairs");if(r[0])return r[1]}return t})});function hf(e){let t=Qe[bo(e)];if(!t)throw new Error(`Unsupported WebGPU dtype: ${e}`);return t.wgslScalar}function bo(e){return e==="f32"?"float32":e==="f16"?"float16":e==="u32"?"uint32":e==="i32"?"int32":e==="vec2<u32>"?"int64":Le(e)}var wf=["f16Ok","subgroupMatrixConfigAvailable"],bf=["DISPATCH_FOLD_WIDTH"],vf=["repeat","tunables","derive","dtypes","logicalDtypes","tensorDtypes","ranks","shapes","present","args","attrs","device"];for(let e of[...vf,...wf,...bf,...Object.keys(wo),"env"])if(!so.has(e))throw new Error(`reserved scope name "${e}" is missing from the reserved list`);var pf=new WeakMap,Wb=new WeakSet;function Va(e){let t=Ue(e),r=pf.get(t);return r===void 0&&(r={features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,limits:t.limits,adapterInfo:Ru(t)},pf.set(t,r),Wb.add(r)),r}var Ju=[["true",!0],["false",!1],["none",null],["True",!0],["False",!1],["None",null]],Fb=[...Ju,["range",Qb],["raise_exception",e=>{throw new Error(e)}]];function Qb(e,t,r=1){if(t===void 0&&(t=e,e=0),r===0)throw new Error("range() step must not be zero");let n=[];if(r>0)for(let a=e;a<t;a+=r)n.push(a);else for(let a=e;a>t;a+=r)n.push(a);return n}var Xu=class{environment;interpreter;cachedBindings=new Map;activeNames=[];nextNames=[];initialNames;evaluating=!1;constructor(){this.environment=new Bt;for(let[t,r]of Ju)this.environment.set(t,r);this.interpreter=new $a(this.environment),this.initialNames=new Set(this.environment.variables.keys())}get isEvaluating(){return this.evaluating}evaluate(t,r,n){let a=!1;this.evaluating=!0;try{let s=_f(t,r,n);if(r.statement===void 0)throw r.parseError??new Error(`Empty WebGPU expression: ${t}`);a=nv(r,s);let o=r.lookupIdentifiers,i=r.freeIdentifiers.length,u=this.nextNames;u.length=0;for(let d=0;d<o.length;d+=1){let p=o[d],f=s[d];d<i||f.present?(this.bind(p,f.value),u.push(p)):this.remove(p,!0)}let l=this.activeNames;for(let d=0;d<l.length;d+=1){let p=l[d];u.includes(p)||this.remove(p,!1)}l.length=0;for(let d=0;d<u.length;d+=1)l.push(u[d]);let c=this.interpreter.evaluate(r.statement,this.environment);return go(c)&&(a=!1),la(c)}catch(s){throw this.reset(),s}finally{this.evaluating=!1,a||this.reset()}}bind(t,r){if(this.initialNames.has(t)){this.environment.set(t,r);return}let n=this.cachedBindings.get(t);if(n!==void 0&&Object.is(n.raw,r)){this.environment.setVariable(t,n.runtime);return}this.environment.variables.delete(t),this.cachedBindings.delete(t);let a=this.environment.set(t,r);this.cachedBindings.set(t,{raw:r,runtime:a})}remove(t,r){this.initialNames.has(t)||this.environment.variables.delete(t),r&&this.cachedBindings.delete(t)}reset(){for(let t of this.environment.variables.keys())this.initialNames.has(t)||this.environment.variables.delete(t);this.cachedBindings.clear(),this.activeNames.length=0}},ho;function el(e){let t=ho;ho=new Xu;try{return e()}finally{ho=t}}function St(e,t={}){if(typeof e!="string")return e;let r=Sf(e),n=ho;if(n!==void 0&&!n.isEvaluating)return n.evaluate(e,r,t);let a=_f(e,r,t);if(r.statement===void 0)throw r.parseError??new Error(`Empty WebGPU expression: ${e}`);return jb(r.statement,r,a)}function jb(e,t,r){let n=new Bt;for(let[o,i]of Ju)n.set(o,i);let a=t.lookupIdentifiers,s=t.freeIdentifiers.length;for(let o=0;o<a.length;o+=1){let i=r[o];(o<s||i.present)&&n.set(a[o],i.value)}return la(new $a(n).evaluate(e,n))}function _f(e,t,r){let n=t.lookupIdentifiers,a=new Array(n.length),s=t.freeIdentifiers.length,o;for(let i=0;i<n.length;i+=1){let u=yf(r,n[i]);a[i]=u,i<s&&!u.present&&(o??=[]).push(n[i])}if(o!==void 0)throw new Error(`Unknown ${lr(o.length,"identifier")} ${o.map(i=>`"${i}"`).join(", ")} in WebGPU expression: ${e}
Identifiers resolve against the expression scope. Namespaces: ${[...vf].sort().join(", ")} (plus the bare names declared in \`derive\` and the constants ${bf.join(", ")}).
device sub-fields: features, wgslLanguageFeatures, limits, adapterInfo.
Helper functions: ${[...Object.keys(wo),...wf].join(", ")}.
If a string literal was intended, quote it (e.g. '"${o[0]}"' not '${o[0]}').`);return a}function Nb(e,t,r){let n=e.lookupIdentifiers.indexOf(r);return n<0?void 0:t[n]}var zb=Object.freeze({present:!1});function Yu(e,t){return e!==void 0&&Object.hasOwn(e,t)&&Object.prototype.propertyIsEnumerable.call(e,t)?{present:!0,value:e[t]}:null}function yf(e,t,r=!1){let n=e;if(t==="env"&&n.env!==void 0)return{present:!0,value:n.env};let a=Yu(e.derived,t);if(a)return a;switch(t){case"repeat":return{present:!0,value:e.repeat??{}};case"tunables":return{present:!0,value:e.tunables??{}};case"derive":return{present:!0,value:e.derived??{}};case"DISPATCH_FOLD_WIDTH":return{present:!0,value:xr}}switch(t){case"dtypes":return{present:!0,value:e.dtypes??{}};case"logicalDtypes":return{present:!0,value:e.logicalDtypes??{}};case"tensorDtypes":return{present:!0,value:e.tensorDtypes??{}};case"ranks":return{present:!0,value:e.ranks??{}};case"shapes":return{present:!0,value:e.shapes??{}};case"present":return{present:!0,value:e.present??{}};case"args":return{present:!0,value:e.args??{}};case"attrs":return{present:!0,value:e.attrs??{}};case"device":return{present:!0,value:Va(e.device),sessionCallRoot:"device-view"};case"f16Ok":{let i=Va(e.device).features.has("shader-f16");return{present:!0,sessionCallRoot:"pure-function",value:Pe(u=>u!=="f16"&&u!=="float16"||i)}}case"subgroupMatrixConfigAvailable":{let o=Ru(Ue(e.device)).subgroupMatrixConfigs;return{present:!0,sessionCallRoot:"pure-function",value:Pe((i,u,l,c,d)=>o===void 0||o.some(p=>p.componentType===i&&(u==null||p.resultComponentType===u)&&p.M===l&&p.N===c&&p.K===d))}}}let s=Yu(wo,t);return s?{...s,sessionCallRoot:"pure-function"}:zb}function tl(e,t={},r){let n=rl(e),a=n.template,s={};for(let u of Ub(n)){let l=yf(t,u,!0);l.present&&(s[u]=l.value)}let o=Yb(n,s,Vb(n),t),i=mo();return i===void 0?Jb(a,s,o):i(Object.freeze({source:e,parsed:a.parsed,context:s,...o===void 0?{}:{memoKey:o},...r===void 0?{}:{provenance:r},render:()=>a.render(s)}))}function rl(e){let t=ua.get(e);if(t!==void 0)return ua.delete(e),ua.set(e,t),t;t={id:qb++,template:Kb(e)},ua.set(e,t),Zu+=e.length;for(let r of ua.keys()){if(Zu<=Lb||r===e)break;ua.delete(r),Zu-=r.length}return t}function Kb(e){let t=ao(no(e,{lstrip_blocks:!0,trim_blocks:!0}));return{parsed:t,render(r){let n=new Bt;for(let[a,s]of Fb)n.set(a,s);for(let[a,s]of Object.entries(r))n.set(a,s);return new $a(n).run(t).value}}}function Ub(e){if(e.referenced!==void 0)return e.referenced;let t=new Set;return Rt(e.template.parsed,r=>(r.type==="Identifier"&&typeof r.value=="string"&&t.add(r.value),!1)),e.referenced=t,t}function Af(e){let t=e.externalNames;return t===void 0&&(t=df(e.template.parsed),e.externalNames=t),t}function $b(e){return Af(e).roots}function kf(e){return $b(rl(e))}function vo(e,t){let r=rl(e),n=Af(r).namespaceReads;return n?.has(t)?n.get(t):Object.freeze([])}function Hb(e,t=!1){let r=e.callee;if(r===void 0||r.type!=="Identifier"||ot(r)!=="has")return;let n=e.args;if(!Array.isArray(n)||n.length!==2)return;let[a,s]=n,o=Sn(a),i=o?.[0]==="device"?o.slice(1):t&&o?.[0]==="env"&&o[1]==="device"?o.slice(2):void 0;if(!(i===void 0||i.length===0)&&!(s.type!=="StringLiteral"||typeof s.value!="string"))return[...i,s.value]}function Vb(e){if(e.deviceReads!==void 0)return e.deviceReads;let t=new Set,r=!1;Rt(e.template.parsed,a=>{if(r)return!0;if(a.type==="CallExpression"){let s=Hb(a,!0);return s!==void 0?(t.add(s.join(".")),!0):!1}if(a.type==="MemberExpression"){let s=Sn(a);if(s===void 0||s[0]!=="device"&&s[0]!=="env")return!1;let o=s[0]==="device"?s.slice(1):s[1]==="device"?s.slice(2):void 0;return o!==void 0&&(o.length===0?r=!0:t.add(o.join("."))),!0}return a.type==="Identifier"&&(a.value==="device"||a.value==="env")?(r=!0,!0):!1});let n=r?"opaque":Object.freeze([...t].sort());return e.deviceReads=n,n}var Zb=2048,xn=new Map;function Xb(e,t){let r=t.split("."),n=e,a=e;for(let s of r){if(n===null||typeof n!="object"&&typeof n!="function")return n;a=n,n=n[s]}return typeof n=="function"?a:n}function Cr(e,t){if(e===null)return"null";let r=typeof e;if(r==="string")return JSON.stringify(e);if(r==="number")return Tr(e);if(r==="boolean"||r==="bigint")return String(e);if(r==="undefined")return"undef";if(r!=="object")return;let n=e;if(!t.has(n)){t.add(n);try{if(Array.isArray(n)){let i=[];for(let u of n){let l=Cr(u,t);if(l===void 0)return;i.push(l)}return`[${i.join(",")}]`}if(n instanceof Map||ArrayBuffer.isView(n))return;let a=n;if(typeof a.has=="function"&&typeof a[Symbol.iterator]=="function"){let i=[];for(let l of n){let c=Cr(l,t);if(c===void 0)return;i.push(c)}let u=[];for(let l of Object.keys(n).sort()){let c=n[l];if(typeof c=="function")continue;let d=Cr(c,t);if(d===void 0)return;u.push(`${JSON.stringify(l)}:${d}`)}return`S[${i.sort().join(",")}]{${u.join(",")}}`}let s=n,o=[];for(let i of Object.keys(s).sort()){let u=Cr(s[i],t);if(u===void 0)return;o.push(`${JSON.stringify(i)}:${u}`)}return`{${o.join(",")}}`}finally{t.delete(n)}}}function ff(e,t,r){if(t==="opaque")return Cr(e,r);let n=[];for(let a of t){let s=Cr(Xb(e,a),r);if(s===void 0)return;n.push(`${JSON.stringify(a)}:${s}`)}return`D{${n.join(",")}}`}function Yb(e,t,r,n){let a=!1;for(let u of Object.keys(t)){let l=t[u];if(typeof l=="function"&&Yu(wo,u)?.value!==l){a=!0;break}}let s=a?"opaque":r,o=new Set,i=[];for(let u of Object.keys(t).sort()){let l=t[u],c;if(typeof l=="function")c=`fn:${u}`;else if(u==="device")c=ff(l,s,o);else if(u==="env"&&l!==null&&typeof l=="object"){let d=l,p=[];for(let f of Object.keys(d).sort()){let m=f==="device"?ff(d[f],s,o):Cr(d[f],o);if(m===void 0)return;p.push(`${JSON.stringify(f)}:${m}`)}c=`{${p.join(",")}}`}else c=Cr(l,o);if(c===void 0)return;i.push(`${JSON.stringify(u)}:${c}`)}if(a){let u=Cr(Va(n.device),o);if(u===void 0)return;i.push(`@device:${u}`)}return`${e.id} ${i.join(",")}`}function Jb(e,t,r){if(r===void 0)return e.render(t);let n=xn.get(r);if(n!==void 0)return xn.delete(r),xn.set(r,n),n;let a=e.render(t);if(xn.set(r,a),xn.size>Zb){let s=xn.keys().next().value;s!==void 0&&xn.delete(s)}return a}var mf=new Map,ev=Object.freeze({lstrip_blocks:!0,trim_blocks:!0}),tv=new Set(["true","false","none","True","False","None"]);function Sf(e){let t=mf.get(e);if(t!==void 0)return t;let r,n;try{r=ao(no(`{{ ${e} }}`,ev))}catch(a){n=a}return t=Object.freeze({statement:r?.body[0],...n!==void 0?{parseError:n}:{},...rv(r)}),mf.set(e,t),t}function rv(e){let t=new Set,r=new Set,n=[],a=s=>{if(!s||typeof s!="object")return;if(Array.isArray(s)){for(let i of s)a(i);return}if(s instanceof Map){for(let[i,u]of s)a(i),a(u);return}let o=s;switch(o.type){case"Identifier":tv.has(o.value)||t.add(o.value);return;case"MemberExpression":a(o.object),o.computed&&a(o.property);return;case"BinaryExpression":a(o.left),a(o.right);return;case"UnaryExpression":a(o.argument);return;case"FilterExpression":{a(o.operand);let i=o.filter;i?.type==="CallExpression"&&a(i.args);return}case"TestExpression":{let i=o.test,u=o.operand;(i?.value==="defined"||i?.value==="undefined")&&u?.type==="Identifier"?r.add(u.value):a(o.operand);return}case"CallExpression":{n.push([...xf(o.callee)??[]]),a(o.callee),a(o.args);return}case"KeywordArgumentExpression":a(o.value);return;default:{for(let[i,u]of Object.entries(o))i!=="type"&&a(u);return}}};a(e);for(let s of t)r.delete(s);return{freeIdentifiers:Object.freeze([...t]),probedIdentifiers:Object.freeze([...r]),lookupIdentifiers:Object.freeze([...t,...r]),callPaths:Object.freeze(n.map(s=>Object.freeze(s)))}}function nv(e,t){for(let r of e.callPaths){let n=r[0],a=n===void 0?void 0:Nb(e,t,n);if(!(r.length===1&&a?.sessionCallRoot==="pure-function")&&!(a?.sessionCallRoot==="device-view"&&r.length===3&&r[0]==="device"&&(r[1]==="features"||r[1]==="wgslLanguageFeatures")&&r[2]==="has"))return!1}return!0}function xf(e){if(!e||typeof e!="object")return null;let t=e;if(t.type==="Identifier"&&typeof t.value=="string")return[t.value];if(t.type==="MemberExpression"&&t.computed!==!0){let r=xf(t.object);if(!r)return null;let n=t.property;if(n?.type==="Identifier"&&typeof n.value=="string")return[...r,n.value]}return null}function Tf(e){let t=Sf(e);return{free:t.freeIdentifiers,probed:t.probedIdentifiers,parseError:t.parseError!==void 0}}function Cf(e,t){return Object.fromEntries(Object.entries(e??{}).map(([r,n])=>[r,$t(n,t)]))}function $t(e,t){return typeof e=="string"?St(e,t):Array.isArray(e)?e.map(r=>$t(r,t)):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([r,n])=>[r,$t(n,t)])):e}function _o(e){let t=new WeakMap,r=(n,a)=>{if(n===null||typeof n!="object")return n;if(t.has(n))return t.get(n);if(t.set(n,n),Array.isArray(n)){let u=n.map((c,d)=>r(c,`${a}[${d}]`)),l=u.every((c,d)=>c===n[d])?n:u;return t.set(n,l),l}let s=[],o=!1;for(let[u,l]of Object.entries(n)){if(u==="$comment"){if(typeof l!="string")throw new Error(`${a}.$comment must be a string`);o=!0;continue}let c=r(l,`${a}.${u}`);o||=c!==l,s.push([u,c])}let i=o?Object.fromEntries(s):n;return t.set(n,i),i};return r(e,"manifest")}var av=new Set(["read-only-storage","storage","uniform"]),Ht=/^[A-Za-z_][A-Za-z0-9_]*$/,Of=/^[A-Za-z0-9][A-Za-z0-9_]*$/,sv=new Set(["u32","i32","f32","bool","string"]);function Cn(e,t){let r=e instanceof Map?new Map(e):new Map(Object.entries(e)),n=new Map(t);return Object.freeze({readText(a){let s=r.get(a);if(s===void 0)throw new Error(`WebGPU template asset is missing: ${a}`);return s},has(a){return r.has(a)},...n.size===0?{}:{assumes:a=>n.get(a)}})}var Pf=new WeakSet;function il(e){e=_o(e),it(e,"WebGPU manifest"),Pf.has(e)||(Jp(e,"WebGPU manifest"),Pf.add(e));let t=e.domain??"ai.onnx";Qv(t,"WebGPU manifest domain");let r=e.name;if(typeof r!="string"||r.length===0)throw new Error("WebGPU manifest requires a non-empty name");let n=`${t}.${r}`;if(e.conformance!==void 0&&e.conformance!=="internal-lowering"&&e.conformance!=="legacy-default-domain")throw new Error(`WebGPU manifest ${n} has invalid conformance ${JSON.stringify(e.conformance)}`);for(let v of Object.keys(e))if(!ov.has(v))throw new Error(`WebGPU manifest ${n} has unknown top-level key "${v}"${If[v]?` (${If[v]})`:""}`);if(!Gt(e.inputs))throw new Error(`WebGPU manifest ${n} requires an inputs map`);if(!Gt(e.outputs))throw new Error(`WebGPU manifest ${n} requires an outputs map`);let a=uv(e.typeConstraints,n),s=Bf(e.inputs,"input",n,a),o=Bf(e.outputs,"output",n,a),i=lv(e.args,n),u=dv(s,o,i,n);if(e.description!==void 0&&typeof e.description!="string")throw new Error(`WebGPU manifest ${n} description must be a string`);let{defaults:l,names:c}=pv(e.attributes,n),d=fv(e.attributeConstraints,l,new Set(c),n),p=hv(s,o,a),f=bv(e.bindings,n,p);Uf(e.derive,`WebGPU manifest ${n} derive`);let m=new Set([...so,...Object.keys(a)]),h=ll(e.derive,`WebGPU manifest ${n} derive`,m),g=jf(e.when,`manifest ${n}`),w=e.requires===void 0?null:Nf(e.requires,`WebGPU manifest ${n}`),b=yv(e,n,{entries:f,ports:p,reservedDeriveNames:m},g,w);return Object.freeze({domain:t,name:r,id:n,...e.conformance===void 0?{}:{conformance:e.conformance},sinceVersion:e.sinceVersion,inputs:Object.freeze(s),outputs:Object.freeze(o),args:u,attributes:l,attributeNames:c,attributeConstraints:d,derive:h,typeConstraints:a,tunables:Df(e.tunables,`WebGPU manifest ${n} tunables`),variants:Object.freeze(b)})}var ov=new Set(["$schema","domain","name","description","conformance","sinceVersion","inputs","outputs","args","attributes","attributeConstraints","typeConstraints","tunables","derive","when","requires","bindings","variants","onnx","aten"]),If={attributeDescriptions:"an attribute's prose is `attributes.<name>.description`",tunableDescriptions:"a tunable's prose is `tunables.<name>.description`",attributeNames:"a default-less attribute is an `attributes` entry without `default`",constants:"`derive` is the one value channel; every derive is template-visible and a pass is keyed by the derives its template reads",variantFamilies:"a family is a `variants` member: { cases, variants }",copy:"a copy is expanded where the manifest is read, by expandSharedLibraries given a copy source",bindingSets:"spec 2.0 declares binding ENTRIES under `bindings` and passes list entry handles",bindingEntries:"spec 2.0 declares binding entries under `bindings`"},ul=new Set(Object.values(wn)),iv=new Set(["description","onnx","dtype","rank","optional","layout","shape","storage","narrowing"]);function uv(e,t){let r={};for(let[n,a]of Object.entries(e??{})){if(!Array.isArray(a)||a.length===0)throw new Error(`${t}.typeConstraints.${n} must be a non-empty array`);for(let s of a)if(!ul.has(s))throw new Error(`${t}.typeConstraints.${n} has unsupported logical dtype ${String(s)}`);r[n]=Object.freeze([...a])}return Object.freeze(r)}function Bf(e,t,r,n){let a=[];for(let[s,o]of Object.entries(e)){let i=`${r}.${t}s.${s}`;if(!Ht.test(s))throw new Error(`${i} is not a WGSL-compatible tensor name`);it(o,i);for(let l of Object.keys(o))if(!iv.has(l))throw new Error(`${i} has unknown key "${l}"`);if(o.onnx!==void 0&&(typeof o.onnx!="string"||o.onnx.length===0))throw new Error(`${i}.onnx must be a non-empty string`);if(o.description!==void 0&&typeof o.description!="string")throw new Error(`${i}.description must be a string`);if(o.dtype!==void 0&&typeof o.dtype!="string")throw new Error(`${i}.dtype must be a string`);if(o.optional!==void 0&&typeof o.optional!="boolean")throw new Error(`${i}.optional must be a boolean`);let u=o.dtype===void 0?void 0:ul.has(o.dtype)?[o.dtype]:n[o.dtype];if(o.storage!==void 0){if(!Object.hasOwn(Qe,o.storage))throw new Error(`${i}.storage ${String(o.storage)} is not a WebGPU storage dtype`);if(u===void 0)throw new Error(`${i} declares a storage dtype but its dtype ${JSON.stringify(o.dtype)} is not a known logical dtype or type variable`);if(u.length===1&&u[0]==="int64"){if(o.storage!=="int32"&&o.storage!=="uint32"&&o.storage!=="int64")throw new Error(`${i}: an int64 port requires int64 storage or an int32/uint32 projection`)}else if(u.length!==1||!bn(u[0],o.storage))throw new Error(`${i}.storage ${o.storage} does not exactly match logical dtype [${u.join(", ")}]; only int64 -> int32/uint32 projection is supported`)}if(o.narrowing!==void 0){if(o.narrowing!=="checked"&&o.narrowing!=="saturating")throw new Error(`${i}.narrowing must be checked or saturating`);if(t!=="input"||u?.length!==1||u[0]!=="int64"||o.storage!=="int32"&&o.storage!=="uint32")throw new Error(`${i}.narrowing is valid only on a projected logical int64 input`)}else if(t==="input"&&u?.length===1&&u[0]==="int64"&&(o.storage==="int32"||o.storage==="uint32"))throw new Error(`${i}: a projected int64 input must declare narrowing as checked or saturating`);if(o.layout!==void 0){if(t==="output")throw new Error(`${i} cannot declare a storage layout`);if(o.layout!=="packed")throw new Error(`${i} has unsupported storage layout ${JSON.stringify(o.layout)}`);if(!(u!==void 0&&u.length>0&&u.every(c=>Object.hasOwn(Qe,c)&&Lu(c))))throw new Error(`${i} declares the packed storage layout but its dtype ${JSON.stringify(o.dtype)} does not resolve to uint8 only`)}a.push(Object.freeze({...o,role:s}))}return a}function lv(e,t){let r={};for(let[n,a]of Object.entries(e??{})){let s=`${t}.args.${n}`;if(!Ht.test(n))throw new Error(`${s} is not a valid arg name`);if(it(a,s),!sv.has(a.kind))throw new Error(`${s} has unsupported kind ${String(a.kind)}${a.kind==="tensor"?" (tensors are inputs/outputs ports in spec 2.0)":""}`);for(let o of Object.keys(a))if(!["kind","description","onnx","required","oneOf"].includes(o))throw new Error(`${s} has unknown key "${o}"`);if(a.description!==void 0&&typeof a.description!="string")throw new Error(`${s}.description must be a string`);if(a.oneOf!==void 0&&(!Array.isArray(a.oneOf)||a.oneOf.length===0||!a.oneOf.every(o=>cv(a.kind,o))))throw new Error(`${s}.oneOf must list values of its kind: strings for a string arg, integers in range for a u32 or i32 arg`);r[n]=a}return r}function cv(e,t){return e==="string"?typeof t=="string":typeof t!="number"||!Number.isInteger(t)?!1:e==="u32"?t>=0&&t<=4294967295:e==="i32"&&t>=-2147483648&&t<=2147483647}function dv(e,t,r,n){let a=[],s=new Map,o=new Set(t.map(i=>i.role));for(let[i,u]of[["input",e],["output",t]])for(let l of u){if(s.has(l.role)){if(i==="output"){let c=e[s.get(l.role)];if(c.dtype!==l.dtype||c.storage!==l.storage)throw new Error(`${n}: inout tensor "${l.role}" must declare the same dtype and storage under inputs and outputs`);continue}throw new Error(`${n}: tensor name "${l.role}" is declared twice`)}s.set(l.role,a.length),a.push([l.role,Object.freeze({kind:"tensor",semantic:l.role,role:i==="input"&&o.has(l.role)?"inout":i,...l.onnx!==void 0?{onnx:l.onnx}:{},...l.storage!==void 0?{dtype:l.storage}:{},...l.narrowing!==void 0?{narrowing:l.narrowing}:{},...l.optional?{required:!1}:{}})])}for(let[i,u]of Object.entries(r)){if(s.has(i))throw new Error(`${n}: arg "${i}" collides with a tensor of the same name`);s.set(i,a.length),a.push([i,Object.freeze({...u})])}return Object.freeze(Object.fromEntries(a))}function pv(e,t){if(e===void 0)return{defaults:Object.freeze({}),names:Object.freeze([])};it(e,`WebGPU manifest ${t} attributes`);let r={},n=[];for(let[a,s]of Object.entries(e)){let o=`WebGPU manifest ${t} attributes.${a}`;if(a.length===0)throw new Error(`${o}: attribute names must be non-empty`);if(!Gt(s))throw new Error(`${o} must be an object { default?, description? } (a bare value is the spec 2.0 draft spelling; write { "default": ${JSON.stringify(s)} })`);for(let i of Object.keys(s))if(i!=="default"&&i!=="description")throw new Error(`${o} has unknown key "${i}"`);if(s.description!==void 0&&typeof s.description!="string")throw new Error(`${o}.description must be a string`);if(Object.hasOwn(s,"default")){if(s.default===void 0)throw new Error(`${o}.default must be a JSON value`);r[a]=s.default}n.push(a)}return{defaults:Object.freeze(r),names:Object.freeze(n.sort())}}function Df(e,t){if(e===void 0)return Object.freeze({});it(e,t);let r={};for(let[n,a]of Object.entries(e)){let s=`${t}.${n}`;if(!Ht.test(n))throw new Error(`${s}: tunable names must be identifiers`);if(!Gt(a))throw new Error(`${s} must be an object { default, description? }; got ${JSON.stringify(a)}`);for(let i of Object.keys(a))if(i!=="default"&&i!=="description")throw new Error(`${s} has unknown key "${i}"`);let o=a.default;if(typeof o!="number"&&typeof o!="string"&&typeof o!="boolean")throw new Error(`${s}.default must be a number, string or boolean`);if(typeof o=="number"&&!Number.isFinite(o))throw new Error(`${s}.default must be finite`);if(a.description!==void 0&&typeof a.description!="string")throw new Error(`${s}.description must be a string`);r[n]=o}return Object.freeze(r)}function ll(e,t,r){let n=jv(e,t);for(let a of Object.keys(n)){if(!Ht.test(a))throw new Error(`${t}.${a}: derive names must be identifiers`);if(r.has(a))throw new Error(`${t}.${a} would shadow a scope name templates and expressions already bind; rename the derive`)}return Object.freeze(n)}function fv(e,t,r,n){if(e===void 0)return Object.freeze({});it(e,`WebGPU manifest ${n} attributeConstraints`);let a=[];for(let[s,o]of Object.entries(e)){let i=`WebGPU manifest ${n} attributeConstraints.${s}`;if(!r.has(s))throw new Error(`${i} does not name a declared attribute`);it(o,i);let u=Object.keys(o).filter(d=>d!=="required"&&d!=="values");if(u.length>0)throw new Error(`${i} has unknown ${lr(u.length,"field")}: ${u.join(", ")}`);let l=o.required??!1;if(typeof l!="boolean")throw new Error(`${i}.required must be a boolean`);if(l&&Object.hasOwn(t,s))throw new Error(`${i}.required is redundant because the attribute has a default`);let c=null;if(o.values!==void 0){if(!Array.isArray(o.values)||o.values.length===0)throw new Error(`${i}.values must be a non-empty array`);let d=[];for(let p of o.values){if(p!==null&&typeof p!="string"&&typeof p!="boolean"&&!(typeof p=="number"&&Number.isFinite(p)))throw new Error(`${i}.values must contain only finite JSON scalars`);if(d.some(f=>So(f,p)))throw new Error(`${i}.values contains duplicate ${JSON.stringify(p)}`);d.push(p)}c=Object.freeze(d)}if(!l&&c===null)throw new Error(`${i} must declare required: true and/or values`);if(Object.hasOwn(t,s)&&c!==null&&!c.some(d=>So(d,t[s])))throw new Error(`WebGPU manifest ${n} default attribute ${s}=${JSON.stringify(t[s])} is not in attributeConstraints.${s}.values`);a.push([s,Object.freeze({required:l,values:c})])}return Object.freeze(Object.fromEntries(a))}function So(e,t){return typeof e=="number"&&typeof t=="number"?Number.isFinite(e)&&Number.isFinite(t)&&Math.fround(e)===Math.fround(t):e===t}function Mf(e){let t=new Map;for(let[r,n]of Object.entries(e))n.kind==="tensor"&&t.set(r,[r]);return t}function Wf(e){let t=new Map;for(let[r,n]of Object.entries(e))n.kind==="tensor"&&n.dtype!==void 0&&t.set(r,n.dtype);return t}function cl(e,t){let r=new Map;for(let n of t)n.layout!==void 0&&r.set(n.role,n.layout);return r}function Ff(e){return t=>t}function Qf(e){return e.map(t=>Xa(t)?Qu(t.name,Zs(t.struct.name,t.struct.fields.map(Fv))):Fu(t.name,{arg:t.arg,access:t.buffer.type==="storage"?"read_write":"read",elementType:t.elementType,...t.length!==void 0?{length:t.length}:{}}))}var mv="main",gv=new Set(["buffer","name","arg","scratch","elementType","length","structName","struct"]);function hv(e,t,r){let n=o=>o.storage==="int64"||o.storage===void 0&&(o.dtype==="int64"||o.dtype!==void 0&&r[o.dtype]?.length===1&&r[o.dtype][0]==="int64"),a=new Map,s=new Set(t.map(o=>o.role));for(let o of e)a.set(o.role,{port:o,opaqueInt64:n(o),direction:s.has(o.role)?"inout":"input"});for(let o of t)a.has(o.role)||a.set(o.role,{port:o,opaqueInt64:n(o),direction:"output"});return a}function wv(e,t){let{port:r}=e;if(r.storage!==void 0)return Qe[r.storage].wgslScalar;if(r.dtype===void 0)throw new Error(`${t} binds port "${r.role}", which declares no dtype; write the entry's elementType`);if(ul.has(r.dtype)){let n=Object.hasOwn(Qe,r.dtype)?Qe[r.dtype]:void 0;if(n===void 0)throw new Error(`${t} binds port "${r.role}" of logical dtype ${r.dtype}, which has no storage scalar; write the entry's elementType`);return n.wgslScalar}return`$${r.dtype}`}function bv(e,t,r){let n=new Map;if(e===void 0)return n;it(e,`WebGPU manifest ${t} bindings`);for(let[a,s]of Object.entries(e)){if(!Ht.test(a))throw new Error(`WebGPU manifest ${t} binding handle "${a}" must be an identifier`);n.set(a,Ao(s,a,`WebGPU manifest ${t} bindings.${a}`,r,null))}return n}function Ao(e,t,r,n,a){it(e,r);for(let p of Object.keys(e))if(!gv.has(p))throw new Error(`${r} has unknown key "${p}"${p==="semantic"?" (spec 2.0 bindings resolve by arg/name and name scratch explicitly)":""}`);for(let p of["name","arg","scratch","elementType","structName"])if(e[p]!==void 0&&(typeof e[p]!="string"||e[p].length===0))throw new Error(`${r}.${p} must be a non-empty string`);let s=e.name??t??e.arg??e.scratch;if(s==null)throw new Error(`${r}: an inline binding entry needs a name (name, arg or scratch)`);if(!Ht.test(s))throw new Error(`${r}.name must be a WGSL identifier`);for(let[p,f]of[["name",s],["arg",e.arg],["scratch",e.scratch]])if(typeof f=="string"&&!oo(f))throw new Error(`${r}.${p} "${f}" is a JavaScript prototype property name`);let o={name:s,...e.arg!==void 0&&e.arg!==s?{arg:e.arg}:{},...e.scratch!==void 0&&e.scratch!==s?{scratch:e.scratch}:{}},i=e.scratch??(a?.has(s)?s:void 0),u=i!==void 0?a?.get(i):void 0,l=i===void 0?n.get(e.arg??s):void 0,c=e.buffer??(e.struct!==void 0?"uniform":i!==void 0?"storage":l!==void 0?l.direction==="input"?"read-only-storage":"storage":void 0);if(c===void 0)throw new Error(`${r} binds "${e.arg??s}", which is neither a declared tensor nor an intermediate, so its buffer type cannot default; write buffer`);if(!av.has(c))throw new Error(`${r}.buffer must be read-only-storage, storage or uniform`);if(c==="uniform"){if(e.elementType!==void 0||e.length!==void 0)throw new Error(`${r}: a uniform binding takes struct fields, not elementType/length`);if(!Array.isArray(e.struct)||e.struct.length===0)throw new Error(`${r}: a uniform binding requires a non-empty struct field list`);return _v(e.struct,r),Object.freeze({...o,buffer:{type:c},struct:Object.freeze({name:e.structName??"Params",fields:Object.freeze([...e.struct])})})}if(e.struct!==void 0||e.structName!==void 0)throw new Error(`${r}: a storage binding takes elementType, not struct fields`);let d=e.elementType;if(d===void 0)if(u!==void 0)d=Qe[u.dtype].wgslScalar;else if(l!==void 0)d=wv(l,r);else throw i!==void 0?new Error(`${r} names scratch "${i}" outside a variant, so its elementType cannot default; write elementType`):new Error(`${r}: a storage binding requires elementType`);if(l?.opaqueInt64&&!d.startsWith("$")&&d.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${r}: an int64 tensor binding must use vec2<u32>`);if(e.length!==void 0){let p=e.length,f=typeof p=="number"&&Number.isSafeInteger(p)&&p>0,m=typeof p=="string"&&p.startsWith("$")&&p.length>1;if(!f&&!m)throw new Error(`${r}.length must be a positive integer or a $-prefixed derive reference`)}return Object.freeze({...o,buffer:{type:c},elementType:d,...e.length!==void 0?{length:e.length}:{}})}function vv(e,t,r,n,a){let s=`WebGPU pass ${r} bindings[${t}]`;if(typeof e=="string"){let o=n.entries.get(e);if(o!==void 0)return o;if(n.ports.has(e))return Ao({},e,s,n.ports,a);if(a.has(e))return Ao({scratch:e},e,s,n.ports,a);throw new Error(`${s} "${e}" is neither a binding entry handle, a declared tensor nor an intermediate of the variant`)}if(Gt(e))return Ao(e,null,s,n.ports,a);throw new Error(`${s} must be a handle, a tensor or intermediate name, or an inline binding entry`)}function _v(e,t){let r=new Set;for(let[n,a]of e.entries()){if(it(a,`${t} field ${n}`),typeof a.name!="string"||!Ht.test(a.name))throw new Error(`${t} field ${n} name must be an identifier`);if(!oo(a.name))throw new Error(`${t} field ${n} name "${a.name}" is a JavaScript prototype property name`);if(r.has(a.name))throw new Error(`${t} repeats field ${a.name}`);if(r.add(a.name),!["u32","i32","f32"].includes(a.type))throw new Error(`${t} field ${a.name} has invalid type ${String(a.type)}`);for(let s of Object.keys(a))if(!["name","type","value","required","internal","default"].includes(s))throw new Error(`${t} field ${a.name} has unknown key "${s}"`);if(a.internal!==void 0&&typeof a.internal!="boolean")throw new Error(`${t} field ${a.name} internal must be a boolean`)}}function Xa(e){return e.buffer.type==="uniform"}function yv(e,t,r,n,a){let s=pl(e,t);if(s.length===0)throw new Error(`WebGPU manifest ${t} requires a non-empty variants list`);let o=s.map((u,l)=>kv(u,l,r,n,a)),i=new Set;for(let u of o){if(i.has(u.id))throw new Error(`WebGPU manifest ${t} has duplicate expanded variant id "${u.id}"`);i.add(u.id)}return xv(o,t),o}function dl(e,t){return t.priority-e.priority}var Av=new Set(["id","description","priority","when","demoteWhen","supersededBy","requires","tunables","derive","intermediates","passes"]);function kv(e,t,r,n,a){it(e,"WebGPU variant");let s=e.id??`variant_${t}`;if(typeof s!="string"||!Of.test(s))throw new Error(`WebGPU variant id must be stable, got ${String(s)}`);for(let p of Object.keys(e))if(!Av.has(p))throw new Error(`WebGPU variant ${s} has unknown key "${p}"${p==="bindings"?" (spec 2.0 lists bindings on each pass)":p==="constants"?" (`derive` is the one value channel)":""}`);if(e.description!==void 0&&typeof e.description!="string")throw new Error(`WebGPU variant ${s} description must be a string`);let o=e.priority??0;if(typeof o!="number"||!Number.isFinite(o))throw new Error(`WebGPU variant ${s} priority must be a finite number`);if(!Array.isArray(e.passes)||e.passes.length===0)throw new Error(`WebGPU variant ${s} requires passes`);Uf(e.derive,`WebGPU variant ${s} derive`);let i=Bv(e.intermediates,s),u=new Map(i.map(p=>[p.id,p]));for(let p of u.keys())if(r.ports.has(p))throw new Error(`WebGPU variant ${s} intermediate "${p}" is also a tensor name`);let l=zf(e.passes,r,u,s),c=Kf(l);Sv(c,u,r.ports,s);let d=e.requires===void 0?null:Nf(e.requires,`WebGPU variant ${s}`);return Object.freeze({id:s,priority:o,when:Object.freeze([...n,...jf(e.when,`variant ${s}`)]),demoteWhen:Ef(e.demoteWhen,`WebGPU variant ${s} demoteWhen`),supersededBy:Ef(e.supersededBy,`WebGPU variant ${s} supersededBy`),requires:Pv(a,d,s),tunables:Df(e.tunables,`WebGPU variant ${s} tunables`),derive:ll(e.derive,`WebGPU variant ${s} derive`,r.reservedDeriveNames),intermediates:Object.freeze(i),passes:Object.freeze(c),passSequence:Object.freeze(l)})}function Sv(e,t,r,n){let a=new Set(t.keys()),s=new Set(r.keys());for(let o of e)for(let i of o.bindings){if(i.buffer.type==="uniform")continue;if(i.scratch!==void 0){if(!a.has(i.scratch))throw new Error(`WebGPU variant ${n} pass ${o.id} binding "${i.name}" names scratch "${i.scratch}", which the variant does not declare as an intermediate`);continue}if(a.has(i.name))continue;let u=i.arg??i.name;if(!s.has(u))throw new Error(`WebGPU variant ${n} pass ${o.id} binding "${i.name}" names request tensor "${u}", which is neither an input, an output nor an intermediate of the variant`)}}function jf(e,t){if(e===void 0)return Object.freeze([]);if(!Array.isArray(e))throw new Error(`WebGPU ${t} when must be a list of expression strings${typeof e=="string"?" (spec 2.0 has no single-string form: wrap it in a list)":""}`);if(e.length===0)throw new Error(`WebGPU ${t} when must not be an empty list (omit the key for an always-eligible variant)`);for(let r of e)if(typeof r!="string"||r.length===0)throw new Error(`WebGPU ${t} when[] entries must be non-empty expression strings, got ${JSON.stringify(r)}`);return Object.freeze([...e])}function Ef(e,t){if(e===void 0)return null;if(!Array.isArray(e)||e.length===0)throw new Error(`${t} must be a non-empty string[]${typeof e=="string"?" (spec 2.0 has no single-string form: wrap it in a list)":""}`);for(let r of e)if(typeof r!="string"||r.length===0)throw new Error(`${t} entries must be non-empty strings, got ${JSON.stringify(r)}`);return Object.freeze([...e])}function xv(e,t){let r=new Map(e.map(a=>[a.id,a])),n=new Map(e.map((a,s)=>[a.id,s]));for(let a of e)for(let s of a.supersededBy??[]){if(s===a.id)throw new Error(`WebGPU manifest ${t} variant "${a.id}" lists itself in supersededBy`);let o=r.get(s);if(!o)throw new Error(`WebGPU manifest ${t} variant "${a.id}" supersededBy names unknown variant "${s}"`);if((dl(o,a)||(n.get(s)??0)-(n.get(a.id)??0))>=0)throw new Error(`WebGPU manifest ${t} variant "${a.id}" (priority ${a.priority}) is supersededBy "${s}" (priority ${o.priority}), which does not outrank it. supersededBy removes a variant where a HIGHER-ranked sibling already covers the shape; to express the reverse, put the guard on the other variant.`)}}var Rf=new Set(["features","limits","subgroupMinSize","subgroupMatrixConfigs"]),Tv=new Set(jr),Cv=new Set(Na);function Nf(e,t){if(!Gt(e))throw new Error(`${t} requires must be an object`);for(let l of Object.keys(e))if(!Rf.has(l))throw new Error(`${t} requires has unknown key "${l}"; expected one of ${[...Rf].join(", ")}`);let{features:r,limits:n,subgroupMinSize:a,subgroupMatrixConfigs:s}=e,o=Ev(r,`${t} requires.features`);for(let l of o)if(!Tv.has(l))throw new Error(`${t} requires.features has unknown feature "${l}"; expected one of ${jr.join(", ")}`);if(n!==void 0&&!Gt(n))throw new Error(`${t} requires.limits must be an object`);for(let[l,c]of Object.entries(n??{})){if(!Cv.has(l))throw new Error(`${t} requires.limits has unknown limit "${l}"; not a canonical WebGPU limit name`);if(typeof c!="number"||!Number.isFinite(c))throw new Error(`${t} requires.limits.${l} must be a finite number`)}let i=Iv(s,t);if(a!==void 0&&(!Number.isInteger(a)||a<1))throw new Error(`${t} requires.subgroupMinSize must be a positive integer`);let u={...o.length>0?{features:o}:{},...n&&Object.keys(n).length>0?{limits:n}:{},...a!==void 0?{subgroupMinSize:a}:{},...i!==null?{subgroupMatrixConfigs:i}:{}};return Object.keys(u).length===0?null:u}function Pv(e,t,r){if(e===null)return t;if(t===null)return e;let n=[...new Set([...e.features??[],...t.features??[]])],a={...e.limits??{}};for(let[i,u]of Object.entries(t.limits??{}))u!==void 0&&(a[i]=Math.max(a[i]??-1/0,u));let s=t.subgroupMinSize!==void 0||e.subgroupMinSize!==void 0?Math.max(t.subgroupMinSize??0,e.subgroupMinSize??0):void 0;if(e.subgroupMatrixConfigs!==void 0&&t.subgroupMatrixConfigs!==void 0)throw new Error(`WebGPU variant ${r} and the manifest both declare requires.subgroupMatrixConfigs; declare the pattern list once`);let o=t.subgroupMatrixConfigs??e.subgroupMatrixConfigs;return{...n.length>0?{features:n}:{},...Object.keys(a).length>0?{limits:a}:{},...s!==void 0?{subgroupMinSize:s}:{},...o!==void 0?{subgroupMatrixConfigs:o}:{}}}function Iv(e,t){if(e===void 0)return null;if(!Array.isArray(e)||e.length===0)throw new Error(`${t} requires.subgroupMatrixConfigs must be a non-empty array of config patterns`);let r=Ks,n=e.map((a,s)=>{let o=`${t} requires.subgroupMatrixConfigs[${s}]`;if(!Gt(a))throw new Error(`${o} must be an object with optional componentType/resultComponentType/M/N/K`);let i=Object.keys(a).filter(l=>!["componentType","resultComponentType","M","N","K"].includes(l));if(i.length>0)throw new Error(`${o} has unknown ${i.length===1?"key":"keys"} ${i.join(", ")}`);for(let l of["componentType","resultComponentType"]){let c=a[l];if(c!==void 0&&(typeof c!="string"||!r.includes(c)))throw new Error(`${o}.${l} must be one of ${r.join(", ")}, got ${JSON.stringify(c)}`)}for(let l of["M","N","K"]){let c=a[l];if(c!==void 0&&(!Number.isInteger(c)||c<1))throw new Error(`${o}.${l} must be a positive integer, got ${JSON.stringify(c)}`)}let u={...a.componentType!==void 0?{componentType:a.componentType}:{},...a.resultComponentType!==void 0?{resultComponentType:a.resultComponentType}:{},...a.M!==void 0?{M:a.M}:{},...a.N!==void 0?{N:a.N}:{},...a.K!==void 0?{K:a.K}:{}};if(Object.keys(u).length===0)throw new Error(`${o} must constrain at least one field (an empty pattern matches everything)`);return Object.freeze(u)});return Object.freeze(n)}function Bv(e,t){if(e===void 0)return[];if(!Array.isArray(e))throw new Error(`WebGPU variant ${t} intermediates must be an array`);return e.map((r,n)=>{if(it(r,`WebGPU variant ${t} intermediate ${n}`),typeof r.id!="string"||r.id.length===0)throw new Error(`WebGPU variant ${t} intermediate ${n} id must be a non-empty string`);if(typeof r.dtype!="string"||!Object.hasOwn(Qe,r.dtype))throw new Error(`WebGPU variant ${t} intermediate ${r.id} dtype must be a concrete WebGPU storage dtype`);if(r.dtype==="int64")throw new Error(`WebGPU variant ${t}: opaque int64 intermediates are unsupported`);if(r.zeroInit!==void 0&&r.zeroInit!==!0)throw new Error(`WebGPU variant ${t} intermediate ${r.id} zeroInit must be true when present (omit the key when the kernel does not require a zeroed buffer)`);return Object.freeze({id:r.id,dtype:r.dtype,shape:r.shape,...r.zeroInit===!0?{zeroInit:!0}:{}})})}function Ev(e,t){if(e===void 0)return[];if(!Array.isArray(e)||![...e].every(r=>typeof r=="string"))throw new Error(`${t} must be an array of strings`);return[...e]}var Rv=new Set(["id","name","shader","derive","bindings","dispatch","profile","subgroupCollectivesWidth","submitAfter","viewAlias"]);function zf(e,t,r,n,a=new Set){return e.map((s,o)=>{if(!Gv(s))return Lv(s,o,t,r);let i=s.id??`repeat_${o}`;if(typeof i!="string"||!Of.test(i))throw new Error(`WebGPU variant ${n} repeat group id must be stable, got ${String(i)}`);it(s.repeat,`WebGPU repeat group ${i}.repeat`);let u=s.repeat.index;if(typeof u!="string"||!Ht.test(u))throw new Error(`WebGPU repeat group ${i} index must be an identifier, got ${String(u)}`);if(u==="index"||u==="count")throw new Error(`WebGPU repeat group ${i} index cannot use reserved repeat member ${u}`);if(a.has(u))throw new Error(`WebGPU repeat group ${i} index ${u} shadows an outer repeat index`);if(s.repeat.count===void 0)throw new Error(`WebGPU repeat group ${i} requires repeat.count`);if(!Array.isArray(s.passes)||s.passes.length===0)throw new Error(`WebGPU repeat group ${i} requires passes`);let l=new Set(a);return l.add(u),Object.freeze({kind:"repeat",id:i,count:s.repeat.count,index:u,passes:Object.freeze(zf(s.passes,t,r,n,l))})})}function Gv(e){return Gt(e)&&"repeat"in e}function Kf(e){let t=[];for(let r of e)"kind"in r&&r.kind==="repeat"?t.push(...Kf(r.passes)):t.push(r);return t}function Lv(e,t,r,n){it(e,`WebGPU pass ${t}`);let a=e.id??`pass_${t}`;if(typeof a!="string"||a.length===0)throw new Error(`WebGPU pass ${t} id must be a non-empty string`);for(let d of Object.keys(e))if(!Rv.has(d))throw new Error(`WebGPU pass ${a} has unknown key "${d}"${d==="source"||d==="constants"?" (a pass spells `shader` and pass-level `derive`)":""}`);if(e.name!==void 0&&(typeof e.name!="string"||e.name.length===0))throw new Error(`WebGPU pass ${a} name must be a non-empty string`);if(typeof e.shader!="string"||e.shader.length===0)throw new Error(`WebGPU pass ${a} requires shader`);if(e.submitAfter!==void 0&&typeof e.submitAfter!="boolean")throw new Error(`WebGPU pass ${a} submitAfter must be a boolean`);let s=qv(e.subgroupCollectivesWidth,a);if(!e.dispatch)throw new Error(`WebGPU pass ${a} requires dispatch`);it(e.dispatch,`WebGPU pass ${a} dispatch`);let o="threads"in e.dispatch?Object.freeze({divisor:e.dispatch.workgroupSize}):void 0,i=Dv(e.dispatch,a);if(!Array.isArray(e.bindings)||e.bindings.length===0)throw new Error(`WebGPU pass ${a} requires a non-empty bindings list`);let u=Object.freeze(e.bindings.map((d,p)=>vv(d,p,a,r,n))),l=new Set;for(let d of u){if(l.has(d.name))throw new Error(`WebGPU pass ${a} binds the WGSL name "${d.name}" twice`);l.add(d.name)}let c=ll(e.derive,`WebGPU pass ${a} derive`,r.reservedDeriveNames);return Object.freeze({id:a,...e.name?{name:e.name}:{},shader:e.shader,entryPoint:mv,bindings:u,derive:c,dispatch:i,...o!==void 0?{dispatchWorkgroupContract:o}:{},profile:Object.freeze({...e.profile??{}}),subgroupCollectivesWidth:s,submitAfter:e.submitAfter===!0,...e.viewAlias?{viewAlias:Ov(e.viewAlias,u,`pass ${a}`)}:{}})}function qv(e,t){if(e===void 0)return null;if(e==="portable"||typeof e=="number"&&Number.isInteger(e)&&e>=4&&e<=128&&(e&e-1)===0)return e;throw new Error(`WebGPU pass ${t} subgroupCollectivesWidth must be "portable" or a power of two in [4, 128]`)}function Ov(e,t,r){if(!Array.isArray(e)||e.length===0)throw new Error(`WebGPU ${r} viewAlias must be a non-empty array of { input, output } binding-name pairs`);let n=a=>t.find(s=>s.name===a)?.buffer.type;return Object.freeze(e.map(a=>{if(typeof a?.input!="string"||typeof a?.output!="string")throw new Error(`WebGPU ${r} viewAlias pair must be { "input": <binding>, "output": <binding> }`);if(n(a.input)!=="read-only-storage")throw new Error(`WebGPU ${r} viewAlias.input "${a.input}" must name a read-only-storage binding of the pass`);if(n(a.output)!=="storage")throw new Error(`WebGPU ${r} viewAlias.output "${a.output}" must name a storage (read_write) binding of the pass`);return Object.freeze({input:a.input,output:a.output})}))}var Tn=xr;function Dv(e,t){let r=e,n="threads"in r,a="workgroups"in r;if("gridStride"in r)throw new Error(`WebGPU pass ${t} dispatch uses gridStride, which spec 2.0 removed: write the clamp explicitly, x = min(<count>, device.limits.maxComputeWorkgroupsPerDimension)`);if(!n&&!a){for(let o of Object.keys(r))if(o!=="x"&&o!=="y"&&o!=="z")throw new Error(`WebGPU pass ${t} dispatch has unknown key "${o}"`);if(r.x===void 0)throw new Error(`WebGPU pass ${t} dispatch requires x`);for(let o of["x","y","z"])r[o]!==void 0&&yo(r[o],t);return e}if(n&&a)throw new Error(`WebGPU pass ${t} dispatch must use exactly one of 'threads'+'workgroupSize' or 'workgroups'`);let s=n?["threads","workgroupSize"]:["workgroups"];for(let o of Object.keys(r))if(!s.includes(o))throw new Error(`WebGPU pass ${t} dispatch shorthand has unknown key "${o}"${["x","y","z"].includes(o)?" (spec 2.0 has no pinned axes; write the explicit x/y/z grid)":""}`);if(n){if(r.workgroupSize===void 0)throw new Error(`WebGPU pass ${t} dispatch shorthand requires both 'threads' and 'workgroupSize'`);return Gf(`ceilDiv((${yo(r.threads,t)}), (${yo(r.workgroupSize,t)}))`)}return Gf(yo(r.workgroups,t))}function Gf(e){return Object.freeze({x:`min(${e}, ${Tn})`,y:`ceilDiv(${e}, ${Tn})`,z:1})}function yo(e,t){if(typeof e=="string")return e;if(typeof e=="number"&&Number.isSafeInteger(e)&&e>=0)return String(e);throw new Error(`WebGPU pass ${t} dispatch sizes must be expression strings or non-negative safe integers, got ${JSON.stringify(e)}`)}function pl(e,t){if(e=_o(e),e.variantFamilies!==void 0)throw new Error(`WebGPU manifest ${t} has a top-level variantFamilies list; a family is a \`variants\` member { cases, variants }`);let r=[];for(let[n,a]of(e.variants??[]).entries()){if(!Gt(a))throw new Error(`WebGPU manifest ${t}: variants[${n}] must be an object`);if("use"in a)throw new Error(`WebGPU manifest ${t} variants[${n}] is an unexpanded shared fragment reference ("use": ${JSON.stringify(a.use)}); expand the shared libraries before normalizing`);if(!("cases"in a)&&!("axes"in a)&&!("exclude"in a)){r.push(a);continue}let s=a;for(let i of Object.keys(s))if(i!=="cases"&&i!=="variants"&&i!=="description"&&i!=="axes"&&!(i==="exclude"&&"axes"in s))throw new Error(`WebGPU manifest ${t}: variants[${n}] has unknown key "${i}"`);if(s.description!==void 0&&typeof s.description!="string")throw new Error(`WebGPU manifest ${t}: variants[${n}] description must be a string`);if(!Array.isArray(s.variants)||s.variants.length===0)throw new Error(`WebGPU manifest ${t}: a family requires at least one variant template`);let{cases:o}=Mv(s,`WebGPU manifest ${t}: variants[${n}]`);for(let[i,{values:u}]of o.entries())for(let[l,c]of s.variants.entries()){let d=`WebGPU manifest ${t}: variants[${n}] case ${i} template ${l}`,p=Wv(c,u,d);ol(p,d,`variants[${n}].variants[${l}]`),r.push(p)}}return r}function Lf(e,t){if(!Za(e)||Object.keys(e).length===0)throw new Error(`${t} must be a non-empty record`);for(let[r,n]of Object.entries(e)){if(!Ht.test(r))throw new Error(`${t} key "${r}" must be an identifier`);if(!al(n))throw new Error(`${t}.${r} must be a finite JSON value`)}}function Mv(e,t){e=_o(e);let r=[],n=(d,p,f)=>{Object.keys(d).length&&Lf(d,`${t} ${f===void 0?"case":`axis ${f}`} ${p}`);let m=new Map;for(let h of Object.keys(d)){let g={case:p,...f===void 0?{}:{axis:f},key:h};r.push(g),m.set(h,g)}return{values:d,fields:m}};if(!("axes"in e)){if(!Array.isArray(e.cases)||e.cases.length===0)throw new Error(`${t}: family cases must be a non-empty array of case records`);return{cases:e.cases.map((d,p)=>(Lf(d,`${t} case ${p}`),n(d,p))),fields:r}}if(!Za(e.axes)||Object.keys(e.axes).length!==2)throw new Error(`${t}: axes must have exactly two named axes`);let a=Object.keys(e.axes);for(let d of a)if(!Ht.test(d))throw new Error(`${t}: axis name "${d}" must be an identifier`);let s=new Map,o=Object.entries(e.axes).map(([d,p])=>{if(!Array.isArray(p)||p.length===0)throw new Error(`${t}: axis ${d} must be a non-empty array`);let f=new Set;return p.map((m,h)=>{if(!Za(m)||typeof m.$key!="string"||!Ht.test(m.$key))throw new Error(`${t}: axis ${d}[${h}] requires an identifier $key`);if(f.has(m.$key))throw new Error(`${t}: duplicate $key "${m.$key}" in axis ${d}`);f.add(m.$key);let g=Object.fromEntries(Object.entries(m).filter(([w])=>w!=="$key"));for(let w of Object.keys(g)){if(a.includes(w))throw new Error(`${t}: value "${w}" collides with an axis selector`);let b=s.get(w);if(b!==void 0&&b!==d)throw new Error(`${t}: value "${w}" belongs to both axes`);s.set(w,d)}return{key:m.$key,...n(g,h,d)}})}),i=(d,p,f)=>{if(!Za(d))throw new Error(`${p} must be a selector record`);for(let[m,h]of a.entries())if(typeof d[h]!="string"||!o[m].some(g=>g.key===d[h]))throw new Error(`${p} must select an existing ${h} key`);if(f&&Object.keys(d).some(m=>!a.includes(m)))throw new Error(`${p} must name exactly both axes`);return JSON.stringify(a.map(m=>d[m]))},u=new Set;if(e.exclude!==void 0&&!Array.isArray(e.exclude))throw new Error(`${t}: exclude must be an array`);for(let[d,p]of(e.exclude??[]).entries()){let f=i(p,`${t} exclude[${d}]`,!0);if(u.has(f))throw new Error(`${t}: duplicate excluded cell ${f}`);u.add(f)}if(e.cases!==void 0&&!Array.isArray(e.cases))throw new Error(`${t}: cases overrides must be an array`);let l=new Map;for(let[d,p]of(e.cases??[]).entries()){let f=i(p,`${t} case ${d}`,!1);if(l.has(f))throw new Error(`${t}: duplicate cell override ${f}`);if(u.has(f))throw new Error(`${t}: override targets excluded cell ${f}`);let m=Object.fromEntries(Object.entries(p).filter(([h])=>!a.includes(h)));l.set(f,n(m,d))}let c=[];for(let d of o[0])for(let p of o[1]){let f=JSON.stringify([d.key,p.key]);if(u.has(f))continue;let m=l.get(f);c.push({values:{...d.values,...p.values,...m?.values},fields:new Map([...d.fields,...p.fields,...m?.fields??[]])})}if(c.length===0)throw new Error(`${t}: axes must retain at least one cell`);return{cases:c,fields:r}}function al(e,t=new Set){if(e===null||typeof e=="string"||typeof e=="boolean")return!0;if(typeof e=="number")return Number.isFinite(e);if(typeof e!="object"||t.has(e))return!1;t.add(e);let r;if(Array.isArray(e)){r=!0;for(let n=0;n<e.length;n+=1)if(!Object.hasOwn(e,n)||!al(e[n],t)){r=!1;break}}else Za(e)?r=Object.values(e).every(n=>al(n,t)):r=!1;return t.delete(e),r}function Za(e){if(e===null||typeof e!="object"||Array.isArray(e))return!1;let t=Object.getPrototypeOf(e);return t===Object.prototype||t===null}var qf=/^\{([A-Za-z_]\w*)\}$/,sl=Symbol("variantFamily.omit");function Wv(e,t,r){let n=ko(e,t,[],r);if(n===sl)throw new Error(`${r}: a case cannot delete the whole variant template`);return n}function ko(e,t,r,n){if(typeof e=="string"){let a=qf.exec(e);if(a&&Object.hasOwn(t,a[1])){let s=t[a[1]];if(s===null){let o=r.at(-1);if(typeof o=="number")throw new Error(`${n}: null substitution "{${a[1]}}" cannot delete the array element at ${nl(r)} (substitute an empty array to splice nothing)`);if(r.length===0||o==="id"||r.length===1&&r[0]==="passes")throw new Error(`${n}: null substitution "{${a[1]}}" cannot remove ${nl(r)}, which is required`);return sl}return xo(s)}return e.replace(/\{([A-Za-z_]\w*)\}/g,(s,o)=>{if(!Object.hasOwn(t,o))return s;let i=t[o];if(i===null||typeof i=="object")throw new Error(`${n}: substitution "{${o}}" at ${nl(r)} must occupy the whole value to take a ${i===null?"null":"structured"} case value`);return String(i)})}if(Array.isArray(e)){let a=[];for(let s=0;s<e.length;s+=1){let o=e[s],i=typeof o=="string"?qf.exec(o):null,u=i!==null&&Object.hasOwn(t,i[1])?t[i[1]]:void 0;if(Array.isArray(u)){for(let l of u)a.push(xo(l));continue}a.push(ko(o,t,[...r,s],n))}return a}if(e!==null&&typeof e=="object"){let a=[],s=new Set;for(let[o,i]of Object.entries(e)){let u=String(ko(o,t,[...r,o],n));if(s.has(u))throw new Error(`${n}: substitution produced duplicate object key "${u}"`);s.add(u);let l=ko(i,t,[...r,o],n);l!==sl&&a.push([u,l])}return Object.fromEntries(a)}return e}function nl(e){return e.length===0?"variant":`variant${e.map(t=>typeof t=="number"?`[${t}]`:`.${t}`).join("")}`}function xo(e){return Array.isArray(e)?e.map(xo):e!==null&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([t,r])=>[t,xo(r)])):e}function ol(e,t,r){if(typeof e=="string"){let n=/\{[A-Za-z_]\w*\}/.exec(e);if(n!==null)throw new Error(`${t} leaves unresolved placeholder "${n[0]}" at ${r}`);return}if(Array.isArray(e)){for(let n=0;n<e.length;n+=1)ol(e[n],t,`${r}[${n}]`);return}if(e!==null&&typeof e=="object")for(let[n,a]of Object.entries(e)){let s=/\{[A-Za-z_]\w*\}/.exec(n);if(s!==null)throw new Error(`${t} leaves unresolved placeholder "${s[0]}" in object key at ${r}`);ol(a,t,`${r}.${n}`)}}function Uf(e,t){if(Gt(e)&&Object.hasOwn(e,"$use"))throw new Error(`${t} still carries an unexpanded "$use" shared-derive splice \u2014 expand the shared libraries before normalizing`)}function Fv(e){if(e.type==="u32")return qu(e.name,e);if(e.type==="i32")return Ou(e.name,e);if(e.type==="f32")return Du(e.name,e);throw new Error(`Unsupported WebGPU uniform field type: ${e.type}`)}function Qv(e,t){if(!/^[a-z][a-z0-9]*(?:\.[A-Za-z][A-Za-z0-9_]*)*$/.test(e))throw new Error(`${t} must be a dotted canonical domain, got ${e}`)}function it(e,t){if(!Gt(e))throw new Error(`${t} must be an object`)}function jv(e,t){return e===void 0?{}:(it(e,t),{...e})}function Gt(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}var $f="@group(0) @binding(0) var<storage, read_write> slot: array<atomic<u32>>;",Vf=Object.freeze([{construct:"atomicCompareExchangeWeak",specifies:"WGSL 17.11.3 makes atomicCompareExchangeWeak a mandatory built-in for every atomic type in both the storage and workgroup address spaces, returning __atomic_compare_exchange_result<T> { old_value: T, exchanged: bool }. It is core WGSL: no feature flag guards it and no conformant implementation may reject it.",defect:`WebKit's WGSL-to-Metal translation emits
    template<typename T, typename U = bool>
    struct __atomic_compare_exchange_result { T old_value; U exchanged; };
instantiated with an address-space-qualified T, which Metal refuses: "field may not be qualified with an address space". Every pipeline containing the built-in fails to create, whatever the kernel does with the result.`,measured:["Safari on iOS, Apple A-series GPU"],unaffected:["Chrome on Apple M-series \u2014 the same GPU family, which is what makes this an engine fact rather than a hardware one","Chrome on Intel Gen9","Chrome on NVIDIA Pascal"],tracking:null,detection:{mode:"build",probe:`${$f}
@compute @workgroup_size(1)
fn main() {
  let swapped = atomicCompareExchangeWeak(&slot[0], 0u, 1u);
  if (swapped.exchanged) {
    atomicStore(&slot[0], swapped.old_value);
  }
}
`,control:`${$f}
@compute @workgroup_size(1)
fn main() {
  let previous = atomicLoad(&slot[0]);
  if (previous == 0u) {
    atomicStore(&slot[0], previous);
  }
}
`}},{construct:"subgroupExclusiveAdd",specifies:"WGSL 17.12 defines the subgroup arithmetic built-ins over the active invocations of a subgroup, and says nothing that would make their results unusable as operands. A value returned by subgroupAdd or subgroupExclusiveAdd is an ordinary u32.",defect:"Feeding the result of subgroupAdd(1u) or subgroupExclusiveAdd(1u) into an atomicAdd that hands out per-subgroup claims makes every subgroup read a claim of zero, so most destinations are never written and kernels that compact or scan through such claims return wrong results. Each collective is correct on its own, which makes this a miscompile of the composition rather than a broken built-in.",measured:["Chrome, NVIDIA Maxwell GPU"],unaffected:["every other tested device, across Apple, Intel and NVIDIA Pascal GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"Kernels take the subgroup width and the electing lane from the subgroup_size / subgroup_invocation_id BUILTINS rather than from the collectives' results. That is correct on the affected device and on every other, so no kernel is withheld anywhere."}},{construct:"num_subgroups",specifies:"WGSL's subgroups extension defines the num_subgroups built-in input as the number of subgroups in the current workgroup, and subgroup_id as each subgroup's index in [0, num_subgroups).",defect:"A workgroup that spans several subgroups reports num_subgroups = 1, so a cross-subgroup fold sized by it reads only the first subgroup's partial and returns a fraction of the reduction.",measured:["Chrome on Android, Qualcomm Adreno 6xx to 8xx GPUs"],unaffected:["Chrome on Apple M-series, Intel and NVIDIA GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"Cross-subgroup folds bound their loop over the subgroup partials by max(num_subgroups, ceil(workgroup_size / subgroup_size)), so an under-reported count can neither skip the fold nor shorten it, and keep indexing the partials by subgroup_id. On a conformant device the reported count always wins, so the fold is exact for any subgroup layout."}},{construct:"workgroupBarrier",specifies:"WGSL's memory model keeps one invocation's accesses to a workgroup location coherent in program order, and workgroupBarrier orders every invocation's workgroup-memory writes before the loads that follow it. A load after a barrier returns the latest value stored to that location, whether the loading invocation stored it or a peer did.",defect:"A load from a workgroup array at a compile-time-constant index returns the value an earlier load of the same element returned, although the array was stored to in between through an invocation-dependent index and the loads are separated by workgroupBarrier: a second inlined tree reduction over the same bank returns the first reduction's slot-0 total, and a serial top-k that marks its pick with a dynamic-index store selects the same element on every pick.",measured:["Chrome on Android, Qualcomm Adreno 810 and 830 GPUs"],unaffected:["Chrome on Android, Qualcomm Adreno 710 GPU","Chrome on AMD GPUs with 64-wide subgroups"],tracking:null,detection:{mode:"behavior",avoidedBy:"A kernel never reads back, at a constant index, a workgroup element it re-stored through an invocation-dependent index after an earlier read of the same element: each tree reduction in one kernel folds through its own bank, and selections rank from values that are not written again after the barrier that publishes them."}},{construct:"local_invocation_index",specifies:"WGSL's memory model keeps one invocation's accesses to a location coherent in program order, and storageBarrier / workgroupBarrier make an invocation's storage writes visible to the rest of its workgroup. Once the loading invocation, or a peer ordered before it by a barrier, has stored to a location, no later load in the dispatch can return the value it held before the dispatch. local_invocation_index is the linearized local_invocation_id.",defect:"A loop whose induction variable starts at local_invocation_index and whose constant stride is at least its constant bound, so that no invocation runs it twice, loads a storage location at a compile-time-constant address (after unrolling and bounds clamping) after a barrier, and receives the value the location held before the dispatch, even when the loading invocation stored to it itself. atomicLoad is affected the same way; read-modify-write atomics, workgroup memory and invocation-dependent addresses are not. The same kernel with the index taken from local_invocation_id, or with the loop written as an if, reads the stored value.",measured:["Dawn (Chrome's WebGPU engine) on macOS, Apple M-series GPU"],unaffected:["Safari on iOS, Apple A-series GPU","Dawn on Windows (D3D12), Intel Xe3-LPG GPU"],tracking:null,detection:{mode:"behavior",avoidedBy:"A kernel whose loop starts at the flat local index and re-reads storage the same dispatch wrote computes that index from local_invocation_id (x + y * width + z * width * height) instead of reading local_invocation_index. The value is the same by definition, and the loads then read the stored value on every implementation measured."}},{construct:"tanh",specifies:"WGSL bounds the f32 tanh built-in only through its definition: its accuracy is inherited from sinh(x) / cosh(x), which inherit from exp (3 + 2|x| ULP) and division (2.5 ULP). At x = -1.25 that allows about 8.7e-7 absolute error.",defect:"tanh is off by about 7.5e-6 relative (6.3e-6 absolute at x = -1.25, seven times the bound) while exp on the same device is within one ULP of the other engines. A recurrence that applies tanh at every step carries the error into its state.",measured:["Chrome on Windows (D3D12), NVIDIA Ampere, Ada Lovelace and Blackwell GPUs"],unaffected:["Chrome on Linux (Vulkan), NVIDIA Blackwell GPU, which makes this an engine fact rather than a hardware one","Chrome on Windows (D3D12), NVIDIA Pascal GPU","Chrome and Safari on Apple, AMD, Intel, Qualcomm and Arm GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"The RNN, GRU and LSTM cells compute tanh from exp: an odd polynomial below |x| = 0.625 and (1 - e) / (1 + e) with e = exp(-2|x|) above it, both evaluated and one selected, which saturates to exactly +/-1 for large and infinite input. Adapters that report vendor apple keep the built-in, which measured exact there and is faster; an adapter that reports no vendor takes the exp form. Elementwise Tanh, Mish, Gelu and the fused activations still call the built-in."}}]);function Nv(e){return Vf.find(t=>t.construct===e)}async function Zf(e,t={}){let r=[],n=async a=>{try{return await e(a)}catch(s){return s instanceof Error?s.message:String(s)}};for(let a of t.bugs??Vf){if(a.detection.mode!=="build"||await n(a.detection.probe)===null)continue;let o=await n(a.detection.control);if(o!==null){t.onInconclusive?.(a.construct,`the control shader failed too (${o}), so the probe says nothing about ${a.construct}`);continue}r.push(a.construct)}return r}function zv(e){let t="",r=0,n=0;for(;n<e.length;){if(r===0&&e.startsWith("//",n)){let a=e.indexOf(`
`,n);if(a===-1)break;t+=`
`,n=a+1;continue}if(e.startsWith("/*",n)){r+=1,n+=2;continue}if(r>0&&e.startsWith("*/",n)){r-=1,n+=2,t+=" ";continue}if(r>0){e[n]===`
`&&(t+=`
`),n+=1;continue}t+=e[n],n+=1}return t}function Kv(e){return new RegExp(`(?<![A-Za-z0-9_])${e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}(?![A-Za-z0-9_])`)}function Xf(e,t){let r=[...t];if(r.length===0)return[];let n=e.map(zv);return r.filter(a=>{let s=Kv(a);return n.some(o=>s.test(o))})}function Uv(e){let t=e.constructs.map(n=>{let a=Nv(n);if(a===void 0)return n;let s=a.tracking===null?"unfiled upstream":a.tracking;return`${n} (${a.measured.join("; ")} \u2014 ${s})`}),r=e.used===null?"and no other variant can serve this call":`so ${e.used} is used instead`;return`${e.opId}: this device cannot build ${t.join(", ")}, which ${e.skipped} needs, ${r}. This is a WebGPU implementation bug, not a missing feature \u2014 the construct is mandatory WGSL.`}var Hf=new Set;function Yf(e){let t=`${e.opId}|${e.skipped}|${e.used??""}|${e.constructs.join(",")}`;Hf.has(t)||(Hf.add(t),console.warn(Uv(e)))}var fl=class extends Map{limit;constructor(t){super(),this.limit=t}get(t){let r=super.get(t);return r!==void 0&&(super.delete(t),super.set(t,r)),r}set(t,r){if(super.delete(t),super.set(t,r),this.size>this.limit){let n=this.keys().next();n.done!==!0&&super.delete(n.value)}return this}};function Kr(e){return e===void 0?new Map:new fl(e)}var To=class{entries;hits=0;misses=0;constructor(t){this.entries=Kr(t)}get(t){let r=this.entries.get(t);if(r===void 0){this.misses+=1;return}return this.hits+=1,r}getOrCreate(t,r){let n=this.get(t);if(n!==void 0)return n;let a=r();return this.entries.set(t,a),a}clear(){this.entries.clear(),this.hits=0,this.misses=0}},Co=class extends To{},Po=class extends To{};function ml(e){return{attrs:e.attrs,dtypes:e.dtypes,logicalDtypes:e.logicalDtypes,tensorDtypes:e.tensorDtypes,ranks:e.ranks,tunables:e.tunables}}function tm(e,t,r,n=null){return`${dt({op:e.id,sinceVersion:e.sinceVersion,variant:t.id,...ml(r),specialization:n})}|device=${Hv(r.device)}|bindings=${$v(t)}`}var Jf=new WeakMap;function $v(e){let t=Jf.get(e);if(t!==void 0)return t;let r=e.passes.map(a=>a.bindings.map(s=>({name:s.name,binding:void 0,group:void 0,type:s.buffer.type,elementType:s.elementType,...s.length!==void 0?{length:s.length}:{},struct:Vv(s.struct)}))),n=dt(r);return Jf.set(e,n),n}var em=new WeakMap;function Hv(e){if(e===null||typeof e!="object")return dt(Nt(e));let t=e,r=em.get(e);if(r!==void 0&&r.features===t.features&&r.wgslLanguageFeatures===t.wgslLanguageFeatures&&r.limits===t.limits&&r.adapterInfo===t.adapterInfo&&r.wgslCompileFailures===t.wgslCompileFailures)return r.text;let n=dt(Nt(e));return em.set(e,{text:n,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,limits:t.limits,adapterInfo:t.adapterInfo,wgslCompileFailures:t.wgslCompileFailures}),n}function Vv(e){return e&&{name:e.name,fields:e.fields.map(t=>({name:t.name,type:t.type}))}}function rm(e){return dt({shapes:e.shapes,args:e.args})}function nm(e,t,r){return`${e}|shapes=${t}|plan=${Tr(r)}`}var dt=Tr;function Zv(e){let t=2166136261;for(let r=0;r<e.length;++r)t^=e.charCodeAt(r),t=Math.imul(t,16777619);return(t>>>0).toString(16).padStart(8,"0")}function Io(e){return`${Zv(e)}:${e.length}`}function am(e,t,r){let n=t.map((a,s)=>{let o=a.offset??0,i=a.size!==void 0&&a.size>0?a.size:"";return`${a.binding??s}:${r(a)}:${o}:${i}`});return`${e}|${n.join("|")}`}var Xv=/(?:^|;)[ \t]*enable\s+[^;]*\bf16\b[^;]*;/m,Yv=/\bf16\b/,Jv=/(?:^|;)[ \t]*enable\s+[^;]*\bsubgroups\b[^;]*;/m,e_=/(?:^|;)[ \t]*enable\s+[^;]*\bsubgroup_size_control\b[^;]*;/m,t_=new RegExp("\\b(?:subgroup(?:Add|ExclusiveAdd|InclusiveAdd|Mul|ExclusiveMul|InclusiveMul|And|Or|Xor|Min|Max|All|Any|Ballot|Elect|Broadcast|BroadcastFirst|Shuffle|ShuffleXor|ShuffleUp|ShuffleDown)|quad(?:Broadcast|SwapX|SwapY|SwapDiagonal))\\s*\\(|@builtin\\s*\\(\\s*(?:subgroup_invocation_id|subgroup_size|subgroup_id|num_subgroups)\\s*\\)"),r_=/@subgroup_size\s*\(/,n_=/(?:^|[^A-Za-z0-9_.])(?:0[xX][0-9a-fA-F]*\.?[0-9a-fA-F]*[pP][+-]?[0-9]+|[0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?|\.[0-9]+(?:[eE][+-]?[0-9]+)?)h\b/;function gl(e){let t="",r=0,n=e.indexOf("/"),a=e.length;for(;n!==-1&&n<a;){let s=e[n+1];if(s==="/"){t+=e.slice(r,n);let o=e.indexOf(`
`,n+2);n=o===-1?a:o,r=n}else if(s==="*"){t+=e.slice(r,n);let o=1;for(n+=2;n<a&&o>0;){if(e[n]==="/"&&e[n+1]==="*"){o+=1,n+=2;continue}if(e[n]==="*"&&e[n+1]==="/"){o-=1,n+=2;continue}e[n]===`
`&&(t+=`
`),n+=1}r=n}else n+=1;n=e.indexOf("/",n)}return t+e.slice(r)}var Bo=new Map,a_=4096;function s_(e){let t=[];return(Yv.test(e)||n_.test(e))&&!Xv.test(e)&&t.push("f16"),t_.test(e)&&!Jv.test(e)&&t.push("subgroups"),r_.test(e)&&!e_.test(e)&&t.push("subgroup_size_control"),t}var o_=!0;function sm(e){let t=Bo.get(e);if(t===void 0){let r=s_(gl(e));t={inferred:r.length===0?e:`${r.map(n=>`enable ${n};
`).join("")}${e}`,f16Only:r.includes("f16")?`enable f16;
${e}`:e},Bo.size>=a_&&Bo.clear(),Bo.set(e,t)}return o_?t.inferred:t.f16Only}function om(e,t){if(t.length===0)return;let r=gl(e),n=new Set(t),a="__opaque_int64_value__",s="";for(let i=0;i<r.length;){let u=/^([A-Za-z_]\w*)\s*\[/.exec(r.slice(i));if(!u){s+=r[i++];continue}let l=i+u[0].length,c=1;for(;l<r.length&&c>0;)r[l]==="["&&++c,r[l]==="]"&&--c,++l;s+=n.has(u[1])?a:`${u[1]}[index]`,i=l}r=s;for(let i=0;i<8;++i){let u=[...r.matchAll(new RegExp(`\\b(?:let|var)\\s+([A-Za-z_]\\w*)(?:\\s*:[^=;]+)?\\s*=\\s*${a}\\s*;`,"g"))].map(l=>l[1]).filter(l=>l!==a);if(u.length===0)break;for(let l of u)r=r.replace(new RegExp(`\\b${l}\\b`,"g"),a)}for(let i="";i!==r;)i=r,r=r.replace(new RegExp(`\\(\\s*${a}\\s*\\)`,"g"),a);let o="(?:==|!=|<=|>=|<<|>>|[+*/%<>&|^~!-])";if(new RegExp(`${a}\\s*(?:\\.|\\[|${o})|${o}\\s*${a}|\\b(?:min|max|abs|clamp|bitcast|f32|i32|u32)\\s*(?:<[^>]+>)?\\s*\\(\\s*${a}`).test(r))throw new Error("Opaque int64 storage permits whole-element copies and selection only; value arithmetic, comparisons, and word extraction are unsupported")}function Ur(e,t){return e.scratch!==void 0&&t.has(e.scratch)?e.scratch:t.has(e.name)?e.name:void 0}var im=new WeakMap;function i_(e,t){let r=im.get(t);return r===void 0&&(r=Io(t.source),im.set(t,r)),`webgpu:${e}:pass=${t.pipelineCacheId}:source=${r}`}function Ya(e,t){let r=new Set(e.plan.scratches.map(a=>a.id)),n=new Set;for(let a of e.program.passes)if(t.has(a.id))for(let s of a.bindingSpecs){let o=Ur(s,r);o!==void 0&&n.add(o)}return n}function u_(e){let t=0;for(let r of e){if(r.type==="uniform")continue;let n=r.tensor,a=typeof r.size=="number"?r.size:typeof n?.byteLength=="number"?n.byteLength:void 0;typeof a=="number"&&Number.isFinite(a)&&a>0&&(t+=a)}return t}function l_(e,t){return typeof e?.bytesMoved=="number"?{profile:e}:t<=0?e?{profile:e}:{}:{profile:{...e??{},bytesMoved:t}}}async function lm(e,t,r={}){let n=Ja(e,t);try{if(n.programs.length===0)return;if(typeof e.runProgramSequence=="function")await e.runProgramSequence(n.programs,r);else for(let a of n.programs)await e.runProgram(a,r)}finally{n.cleanup()}}function Ja(e,t,r={}){let n=m_(t);if(!n.some(Boolean))return{programs:[],zeroInitClears:[],constantPrograms:[],cleanup(){}};let a=g_(t,n),s=r.scratchResources,o=s===void 0,i=s===void 0?d_(e,t,a):f_(t,s,a),u=[],l=[],c=[],d=r.constantPassIds,p=o?[]:p_(t,i,a),f=!1,m=()=>{if(f)return;f=!0;let h;for(let g of u)try{g()}catch(w){h??=w}if(o)try{cm(i)}catch(g){h??=g}if(h!==void 0)throw h};try{for(let h=0;h<t.program.passes.length;++h){let g=t.program.passes[h],w=t.plan.passes[h];if(!n[h])continue;let b=`${t.program.op} (variant ${t.program.variant}, pass ${g.id})`,v=w_(t,g.bindingSpecs,w.uniforms,i),y=[];for(let C of g.bindingSpecs){let k=v[C.name];if(!wt(k)||k.dtype!=="int64")continue;y.push(C.name);let B=C.elementType,P=B?.startsWith("$")?g.derived[B.slice(1)]:B;if(typeof P!="string"||P.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${b}: int64 binding ${C.name} must use vec2<u32>`)}if(om(g.source,y),__(g,v))continue;b_(b,g.bindings,C=>v[C.name]);let S=Ep(e,g.bindings,v,{labelPrefix:g.name});u.push(S.cleanup),(d?.has(g.id)?c:l).push({name:g.name,op:t.program.op,kernelId:g.kernelId,source:g.source,entryPoint:g.entryPoint,cacheKey:i_(t.program.key,g),bindings:S.bindings,dispatchWorkgroups:w.dispatchWorkgroups,...g.submitAfter?{submitAfter:!0}:{},...l_({...w.profile??{},variant:t.program.variant},u_(S.bindings))})}return{programs:l,zeroInitClears:p,constantPrograms:c,cleanup:m}}catch(h){try{m()}catch(g){throw new AggregateError([h,g],"WebGPU plan materialization and cleanup both failed")}throw h}}function c_(e,t){return t[e.arg??""]??t[e.name]}function d_(e,t,r){let n={};if(r.size===0)return n;if(typeof e.empty!="function")throw new Error("WebGPU multi-pass plan requires a runtime with empty(dtype, shape)");try{for(let a of t.plan.scratches)r.has(a.id)&&(n[a.id]=e.empty(a.dtype,a.shape,`webgpu-scratch-${a.id}`))}catch(a){try{cm(n)}catch(s){throw new AggregateError([a,s],"WebGPU scratch allocation and cleanup both failed")}throw a}return n}function p_(e,t,r){let n=[];for(let a of e.plan.scratches){if(a.zeroInit!==!0||!r.has(a.id))continue;let s=t[a.id];if(s===void 0)continue;let o=typeof s.byteOffset=="number"?s.byteOffset:0,i=typeof s.byteLength=="number"?s.byteLength:a.byteLength;if(i!==0){if(o%4!==0||i%4!==0)throw new Error(`WebGPU scratch ${a.id} declares zeroInit but its buffer range (offset ${o}, length ${i}) is not 4-byte aligned; a zeroInit scratch must be 4-byte aligned in both offset and length (use a 4-byte dtype, or pad the declared shape)`);n.push({kind:"clear",dst:s,dstOffset:o,byteLength:i,name:`${e.program.op}:${a.id}-zero-init`})}}return n}function f_(e,t,r){let n={};for(let a of e.plan.scratches){if(!r.has(a.id))continue;let s=t[a.id];if(s==null)throw new Error(`Missing caller-owned WebGPU scratch resource ${a.id}`);let o=s;if(o.dtype!==a.dtype||!Array.isArray(o.shape)||!Vs(o.shape,a.shape)){let i=Array.isArray(o.shape)?`[${o.shape.join(",")}]`:String(o.shape);throw new Error(`Caller-owned WebGPU scratch ${a.id} must be ${a.dtype}[${a.shape.join(",")}], got ${String(o.dtype)}${i}`)}n[a.id]=s}return n}function m_(e){if(e.program.passes.length!==e.plan.passes.length)throw new Error(`WebGPU program/plan pass count mismatch: ${e.program.passes.length} program passes, ${e.plan.passes.length} plan passes`);return e.plan.passes.map((t,r)=>{let n=e.program.passes[r];if(n.id!==t.id)throw new Error(`WebGPU program/plan pass ${r} identity mismatch: program ${n.id}, plan ${t.id}`);return!h_(t.dispatchWorkgroups)})}function g_(e,t){let r=new Set(e.plan.scratches.map(a=>a.id)),n=new Set;for(let a=0;a<e.program.passes.length;a+=1)if(t[a])for(let s of e.program.passes[a].bindingSpecs){let o=Ur(s,r);o!==void 0&&n.add(o)}return n}function h_(e){return e.some(t=>t===0)}function cm(e){let t;for(let r of Object.values(e))if(typeof r.destroy=="function")try{r.destroy()}catch(n){t??=n}if(t!==void 0)throw t}function w_(e,t,r,n){let a={},s=new Set(Object.keys(n));for(let o of t){if(o.buffer.type==="uniform"){let i=um(e,o,n,s);a[o.name]=i??r[o.name];continue}a[o.name]=um(e,o,n,s)}return a}function um(e,t,r,n){let a=Ur(t,n);return(a===void 0?void 0:r[a])??c_(t,e.request.resources)}function b_(e,t,r){let n=null;for(let a of t){let s=v_(a);if(s===null)continue;let o=hl(r(a));if(o===null)continue;n===null&&(n=new Map);let i=n.get(o.buffer);if(i===void 0){n.set(o.buffer,{name:a.name,access:s,view:o});continue}if(i.access==="read"&&s==="read")continue;if(i.access==="read_write"&&s==="read_write"){if(!y_(i.view.offset,i.view.end,o.offset,o.end))continue;throw new Error(`${e}: storage bindings "${i.name}" and "${a.name}" alias overlapping writable ranges of the same GPU buffer; overlapping storage aliases are unsafe when either binding is writable`)}let[u,l]=s==="read_write"?[i.name,a.name]:[a.name,i.name];throw new Error(`${e}: storage aliasing hazard \u2014 binding "${u}" (read-only-storage) and binding "${l}" (storage, read_write) resolve to the same GPU buffer. WebGPU usage scopes cover the whole buffer, so this poisons the entire command buffer and corrupts downstream results. Bind it once as storage (read_write) for intentional in-place, or use distinct buffers.`)}}function v_(e){if("buffer"in e&&e.buffer!==void 0)switch(e.buffer.type){case"read-only-storage":return"read";case"storage":return"read_write";default:return null}let t=e;return t.kind==="storage"?t.access:null}function __(e,t){let r=e.viewAlias;if(r===void 0)return!1;for(let{input:n,output:a}of r){let s=hl(t[n]),o=hl(t[a]);if(s===null||o===null||!(s.buffer===o.buffer&&s.offset===o.offset&&s.end===o.end)||s.dtype!==o.dtype)return!1}return!0}function hl(e){if(e===null||typeof e!="object")return null;let t=e,r=t.buffer??e;if(r===null||typeof r!="object"&&typeof r!="function")return null;let n=typeof t.byteOffset=="number"?t.byteOffset:0,a=typeof t.byteLength=="number"?t.byteLength:typeof t.size=="number"?t.size:Number.POSITIVE_INFINITY,s=typeof t.dtype=="string"?t.dtype:void 0;return{buffer:r,offset:n,end:n+a,dtype:s}}function y_(e,t,r,n){return e<n&&r<t}var dm;function wl(e,t){return dm===void 0?void 0:dm(e,t)}function pm(e){let t=e,r=new Set(Object.keys(t.args??{}));for(let a of["inputs","outputs"]){let s=t[a];if(Array.isArray(s))for(let o of s)typeof o?.role=="string"&&r.add(o.role);else if(s&&typeof s=="object")for(let o of Object.keys(s))r.add(o)}let n=t.bindings;if(n&&typeof n=="object"&&!Array.isArray(n))for(let[a,s]of Object.entries(n))r.add(typeof s?.name=="string"?s.name:a),typeof s?.arg=="string"&&r.add(s.arg);for(let a of t.variants??[]){let s=o=>{for(let i of o){if(i?.repeat!==void 0){s(i.passes??[]);continue}for(let u of Array.isArray(i?.bindings)?i.bindings:[])u&&typeof u=="object"&&(typeof u.name=="string"&&r.add(u.name),typeof u.arg=="string"&&r.add(u.arg))}};s(a?.passes??[])}return r}function fm(e,t,r=!1){return e.map(n=>{let a=r?[0]:A_($t(n.shape,t),n.id),s=bo(n.dtype);return Object.freeze({id:n.id,dtype:s,shape:a,byteLength:Ye(s,ke(a)),...n.zeroInit===!0?{zeroInit:!0}:{}})})}function A_(e,t){let r=Array.isArray(e)?e:[e];if(r.length===0)throw new Error(`WebGPU scratch ${t} shape must not be empty`);return Object.freeze(r.map(n=>{if(!Number.isInteger(n)||Number(n)<0)throw new Error(`WebGPU scratch ${t} shape dimension must be a nonnegative integer, got ${String(n)}`);return Number(n)}))}var mm=65535,Eo=131072;function gm(e,t){let r=[];if(wm(e.passSequence,t,t.repeat,"","",!1,r),r.length===0)throw new Error(`WebGPU variant ${e.id} expands to zero passes; every eligible request must retain at least one pass (all-empty outputs use retained zero-dispatch passes)`);if(r.length>Eo)throw new Error(`WebGPU variant ${e.id} expands to ${r.length} passes; maximum is ${Eo}`);let n=new Set;for(let a of r){if(n.has(a.id))throw new Error(`WebGPU variant ${e.id} expands duplicate pass id ${a.id}`);n.add(a.id)}return Object.freeze(r)}function hm(e){return e.some(bm)}function wm(e,t,r,n,a,s,o){for(let i of e){if(!bm(i)){if(o.push(Object.freeze({pass:i,id:`${n}${i.id}`,pipelineBaseId:`${a}${i.id}`,repeat:r,repeated:s})),o.length>Eo)return;continue}let u=St(i.count,{...t,repeat:r});if(!Number.isInteger(u)||Number(u)<0||Number(u)>mm)throw new Error(`WebGPU repeat group ${i.id} count must resolve to an integer in [0, ${mm}], got ${String(u)}`);let l=Number(u);for(let c=0;c<l;++c){let d=Object.freeze({...r,index:c,count:l,[i.index]:c,[`${i.index}_count`]:l});if(wm(i.passes,t,d,`${n}${i.id}[${c}].`,`${a}${i.id}.`,!0,o),o.length>Eo)return}}}function bm(e){return"kind"in e&&e.kind==="repeat"}function k_(e){return"kind"in e&&e.kind==="repeat"}function vm(e,t,r){let n=new Set,a=new Map,s=!1,o=u=>{try{let l=a.get(u.shader);if(l===void 0&&(l=vo(r(u.shader),"tunables"),a.set(u.shader,l)),l===null)s=!0;else for(let c of l)n.add(c)}catch{s=!0}},i=u=>{for(let l of u)k_(l)?i(l.passes):o(l)};return i(t.passSequence),Object.freeze({tunables:s?null:Object.freeze([...n].sort())})}function _m(e,t){if(t===null)return e;let r={};for(let n of t)Object.hasOwn(e,n)&&(r[n]=e[n]);return r}var vl=Object.freeze([]);function S_(e,t){return t.size===0?vl:Xf(e.passes.map(r=>r.source),t)}function Bm(e){return typeof e=="string"?e:String(e)}function x_(e,t){let r=Bm(e);return{kind:"whenFailed",clause:r,value:String(t),message:`when clause failed: \`${r}\` \u2192 ${String(t)}`}}function T_(e){let t=0,r=0;for(let n of e.passes){let a=0,s=0;for(let o of n.bindings){let i=o.buffer?.type;i&&(i==="uniform"?s+=1:a+=1)}t=Math.max(t,a),r=Math.max(r,s)}return{storage:t,uniform:r}}function C_(e,t){let{storage:r,uniform:n}=T_(e),a=t.limits.maxStorageBuffersPerShaderStage;if(typeof a=="number"&&r>a)return`needs ${r} storage buffers per shader stage, device allows ${a} (maxStorageBuffersPerShaderStage)`;let s=t.limits.maxUniformBuffersPerShaderStage;return typeof s=="number"&&n>s?`needs ${n} uniform buffers per shader stage, device allows ${s} (maxUniformBuffersPerShaderStage)`:null}var ym=new WeakMap;function P_(e,t){let r=ym.get(e);r===void 0&&(r=new WeakMap,ym.set(e,r));let n=r.get(t);if(n!==void 0&&n.features===t.features&&n.limits===t.limits&&n.adapterInfo===t.adapterInfo)return n;let a={features:t.features,limits:t.limits,adapterInfo:t.adapterInfo,requirementReason:Eu(e.requires,t),budgetReason:C_(e,t)};return r.set(t,a),a}function I_(e,t){let r={};for(let n of Object.keys(e)){let a=e[n];a!==void 0&&(r[n]=es(a)?{dtype:a.dtype,shape:a.shape,...a.logicalDtype===void 0?{}:{logicalDtype:a.logicalDtype}}:a)}return dt({inputs:r,args:t.args??{},attrs:t.attrs??{}})}var Am=new WeakMap;function B_(e){let t=Am.get(e);if(t!==void 0)return t;let r=Object.entries(e.derive),n=new Set(r.map(([i])=>i)),a=new Set,s=i=>{if(typeof i=="string"){let u=Tf(i);for(let l of[...u.free,...u.probed])!n.has(l)||a.has(l)||(a.add(l),s(e.derive[l]));return}if(Array.isArray(i)){for(let u of i)s(u);return}if(i&&typeof i=="object")for(let u of Object.values(i))s(u)};for(let i of e.outputs)i.shape!==void 0&&s(i.shape);let o=Object.freeze(r.filter(([i])=>a.has(i)).map(([i,u])=>Object.freeze([i,u])));return Am.set(e,o),o}function km(e,t){let r=Object.freeze(Rp(aa(Qf(e.bindings)),t).map(a=>Object.freeze(a))),n=[Bp(r),ju(r,t)].filter(Boolean).join(`
`);return Object.freeze({kernelBindings:r,resourceDeclarations:n})}function E_(e,t){return e.derived===t.derived&&e.repeat===t.repeat&&e.tunables===t.tunables&&e.device===t.device&&e.dtypes===t.dtypes&&e.logicalDtypes===t.logicalDtypes&&e.tensorDtypes===t.tensorDtypes&&e.ranks===t.ranks&&e.shapes===t.shapes&&e.present===t.present&&e.args===t.args&&e.attrs===t.attrs&&e.env===t.env}var _l=class extends Error{name="WebGpuTemplateAssumptionError"},ts=class{manifest;assets;programCache;planCache;prepareCache;explainCache;selectedVariants;manifestDerivedCache;expandedTemplateCache=new Map;passKeyedReadsCache=new WeakMap;passDerivedBase=new WeakMap;passBindingsMemo=new WeakMap;passDerivedMemo=new WeakMap;variantProgramDependencies=new WeakMap;inferCache;scopeNamesByArg;storageDtypesByPortRole;layoutsByArg;acceptedArgKeys;acceptedResourceKeys;acceptedAttributeKeys;variantsById;renderedSourceAudit;requestCacheLimit;constructor(t,r={}){this.manifest=il(t),this.assets=L_(r.assets??{}),this.renderedSourceAudit=r.renderedSourceAudit;let n=r.requestCacheLimit;this.requestCacheLimit=n,this.programCache=r.programCache??new Co(n),this.planCache=r.planCache??new Po(n),this.prepareCache=Kr(n),this.explainCache=Kr(n),this.selectedVariants=Kr(n),this.manifestDerivedCache=Kr(n),this.inferCache=Kr(n),this.scopeNamesByArg=Mf(this.manifest.args),this.storageDtypesByPortRole=Wf(this.manifest.args),this.layoutsByArg=cl(this.manifest.args,this.manifest.inputs),this.variantsById=new Map(this.manifest.variants.map(a=>[a.id,a])),this.acceptedArgKeys=new Set(Object.keys(this.manifest.args)),this.acceptedAttributeKeys=new Set(this.manifest.attributeNames),this.acceptedResourceKeys=pm(this.manifest)}get prepareCacheSize(){return this.prepareCache.size}explain(t,r,n={}){let a=wl(this,this.manifest),s=this.requestScope(t.device,r,a),{baseScope:o,scopeKey:i}=s,u=this.pinVariant(i,n.variant),l=this.explainCache.get(u);if(l)return l;let c=a?.selectionRequest(s),d=this.applyManifestDeriveCached(i,o,c),p=new Map,f=this.variantCandidates(n.variant).map(g=>{let w=this.checkVariant(g,d,p,r,c);return w.ok?{id:g.id,ok:!0,demoted:w.demotions.length>0?Object.freeze([...w.demotions]):null}:{id:g.id,ok:!1,rejection:w.rejection}}),m=null;for(let g of f)if(g.ok){if(g.demoted===null){m=g.id;break}m??=g.id}let h={selected:m,candidates:f};return this.explainCache.set(u,h),h}selectVariantId(t,r){this.validateRequest(r);let{baseScope:n,scopeKey:a}=this.requestScope(t.device,r,void 0),s=this.prepareCache.get(a);if(s!==void 0)return s.program.variant;let o=this.selectedVariants.get(a);return o===void 0&&(o=this.selectVariant(this.applyManifestDeriveCached(a,n,void 0),void 0,r),this.selectedVariants.set(a,o)),o.id}prepare(t,r,n={}){let a=wl(this,this.manifest),s=a?.rememberedRequestScope(r,t.device);s?.validated!==!0&&this.validateRequest(r);let o=s??this.requestScope(t.device,r,a);o.validated=!0;let{baseScope:i,scopeKey:u}=o,l=this.pinVariant(u,n.variant),c=this.prepareCache.get(l);if(c)return{program:c.program,plan:c.plan,request:r};let d=a?.selectionRequest(o),p=this.applyManifestDeriveCached(u,i,d),f=n.variant===void 0?this.selectedVariants.get(l):void 0,{variant:m,scope:h}=f===void 0?this.selectVariantAndScope(p,r,n.variant,d):{variant:f,scope:this.applyVariantDerive(p,r,f,d)},{passInstances:g,programKey:w,program:b}=this.buildProgramFor(m,h),v=this.allOutputsEmpty(h);this.validateDispatchWorkgroupContracts(m,h,g,b,v);let y=this.buildPlanPayload(m,h,g,v),S=rm(h),C=nm(w,S,y),k=this.planCache.getOrCreate(C,()=>this.buildPlan(C,w,y));return this.prepareCache.set(l,{program:b,plan:k}),this.selectedVariants.delete(l),{program:b,plan:k,request:r}}clearPreparedState(){this.prepareCache.clear(),this.explainCache.clear(),this.selectedVariants.clear(),this.manifestDerivedCache.clear(),this.programCache.clear(),this.planCache.clear()}inferOutputs(t,r={}){let n=I_(t,r),a=this.inferCache.get(n);if(a)return a;this.rejectUnknownKeys(this.manifest.id,"binding",t,r.args);let s=Ff(this.manifest.args);for(let c of this.manifest.outputs){if(c.shape!==void 0||c.optional)continue;let d=s(c.role),p=t[d];if(p===void 0||!es(p))throw new Error(`${this.manifest.id}: output "${c.role}" has no shape expression \u2014 pass "${d}" as an explicit destination, or declare an outputs[].shape for it in the manifest.`)}let o={};for(let[c,d]of Object.entries(t))d!==void 0&&(o[c]=es(d)?q_(d):d);let i={resources:o,...r.args?{args:r.args}:{},...r.attrs?{attrs:r.attrs}:{}},u=this.applyManifestDeriveForInference(this.buildBaseScope(Ue(void 0),i)),l={};for(let c of this.manifest.outputs){let d=s(c.role),p=t[d];if(c.optional&&(p===void 0||!es(p)))continue;if(c.shape===void 0){if(p!==void 0&&es(p)){l[d]=this.inferredOutput(c,p.shape,u);continue}if(c.optional)continue;throw new Error(`${this.manifest.id}: output "${c.role}" has no shape expression \u2014 pass "${d}" as an explicit destination, or declare an outputs[].shape for it in the manifest.`)}let f=typeof c.shape=="string"?St(c.shape,u):c.shape.map(m=>$t(m,u));if(!Array.isArray(f)||!f.every(m=>Number.isInteger(m)&&m>=0))throw new Error(`${this.manifest.id}: output "${c.role}" shape evaluated to ${JSON.stringify(f)} (expected non-negative integers)`);l[d]=this.inferredOutput(c,f,u)}return this.inferCache.set(n,l),l}inferredOutput(t,r,n){let a=this.resolveOutputLogicalDtype(t,n),s=this.storageDtypesByPortRole.get(t.role)??(a!==void 0&&Object.hasOwn(Qe,a)?a:void 0);if(s===void 0)throw new Error(`${this.manifest.id}: output "${t.role}" resolves to logical dtype ${String(a)}, but has no concrete WebGPU storage dtype`);return{role:t.role,shape:[...r],dtype:s,...a===void 0||a===s?{}:{logicalDtype:a}}}resolveOutputLogicalDtype(t,r){if(t.dtype===void 0)return"float32";if(Object.hasOwn(wn,t.dtype))return t.dtype;let n=r.logicalDtypes[t.dtype];if(n!==void 0)return n;let a=this.manifest.typeConstraints[t.dtype];if(this.manifest.domain==="ai.onnx"&&this.manifest.attributeConstraints.to?.required){let o=r.attrs.to,i=typeof o=="number"?rf(o)?.logical:void 0;if(i===void 0)throw new Error(`${this.manifest.id}: required attribute to=${JSON.stringify(o)} is not a supported ONNX TensorProto dtype code`);if(a===void 0||!a.includes(i))throw new Error(`${this.manifest.id}: required attribute to resolves to ${i}, not allowed by output type constraint ${t.dtype} = [${a?.join(", ")??""}]`);return i}if(a?.length===1)return a[0];let s=r.tensorDtypes;for(let o of this.manifest.inputs){let i=s[o.role];if(typeof i=="string"&&(!Array.isArray(a)||a.some(u=>u===i))&&Object.hasOwn(wn,i))return i}throw new Error(`${this.manifest.id}: output "${t.role}" dtype "${t.dtype}" is a constraint var with no bound input \u2014 pass a dtype override or bind an input sharing the var.`)}async run(t,r,n={}){let a=this.prepare(t,r,n);return await lm(t,a,n.runOptions??{}),a}buildProgramFor(t,r){let n=gm(t,r),a=this.programDependencies(t),s=this.programSpecialization(t,r,n),o={...r,tunables:_m(r.tunables,a.tunables)},i=tm(this.manifest,t,o,s),u=this.programCache.getOrCreate(i,()=>this.buildProgram(i,t,r,n));return{passInstances:n,programKey:i,program:u}}wgslBuildBlockers(t,r,n,a){if(a.size===0)return vl;try{let s=this.applyVariantDerive(r,n,t);return S_(this.buildProgramFor(t,s).program,a)}catch{return vl}}buildProgram(t,r,n,a){let s=new Map,o=a.map(i=>this.buildProgramPass(r,i,n,s));return Object.freeze({key:t,op:this.manifest.id,variant:r.id,passes:Object.freeze(o)})}buildProgramPass(t,r,n,a){let s=r.pass,o={...n,repeat:r.repeat},i=this.passDerived(s,o);for(let f of s.bindings){if(f.buffer.type==="uniform"||n.tensorDtypes[f.arg??f.name]!=="int64")continue;let m=f.elementType,h=m?.startsWith("$")?i[m.slice(1)]:m;if(typeof h!="string"||h.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${this.manifest.id} variant ${t.id} pass ${r.id}: int64 binding ${f.name} must use vec2<u32>`)}let u={...o,derived:i},{kernelBindings:l,resourceDeclarations:c}=this.passBindings(s,i),d=this.buildTemplateSource(t,s,u,c),p=r.pipelineBaseId;if(r.repeated){let f=a.get(r.pipelineBaseId);f===void 0&&(f=new Map,a.set(r.pipelineBaseId,f));let m=f.get(d);m!==void 0?(d=m.source,p=m.pipelineCacheId):(p=`${r.pipelineBaseId}|source=${f.size}`,f.set(d,{source:d,pipelineCacheId:p}))}return Object.freeze({id:r.id,pipelineCacheId:p,...r.repeated?{repeatBaseId:r.pipelineBaseId}:{},kernelId:`${this.manifest.id}.${t.id}.${s.id}`,name:s.name??`${this.manifest.id}.${t.id}.${s.id}`,source:d,derived:i,entryPoint:s.entryPoint,bindings:l,bindingSpecs:s.bindings,...s.submitAfter?{submitAfter:!0}:{},...s.viewAlias?{viewAlias:s.viewAlias}:{}})}passBindings(t,r){let n=this.passBindingsMemo.get(t);if(n===void 0){let i=[];for(let u of t.bindings)if(!Xa(u))for(let l of[u.elementType,u.length])typeof l=="string"&&l.startsWith("$")&&i.push(l.slice(1));n={refs:i,built:Kr(this.requestCacheLimit)},this.passBindingsMemo.set(t,n)}let a=[];for(let i of n.refs){let u=r[i];if(u==null)return km(t,r);a.push(u)}let s=dt(a),o=n.built.get(s);return o===void 0&&(o=km(t,r),n.built.set(s,o)),o}expandTemplate(t,r){let n=`${r?"p":"s"}\0${t}`,a=this.expandedTemplateCache.get(n);if(a!==void 0)return a;let s=i=>this.assets.readText(i),o=r?R_(t,s(t),s,new Set):{source:Rm(t,s(t),s,new Set)};return this.expandedTemplateCache.set(n,o),o}programDependencies(t){let r=this.variantProgramDependencies.get(t);return r===void 0&&(r=vm(this.manifest,t,n=>this.expandTemplate(n,!1).source),this.variantProgramDependencies.set(t,r)),r}buildTemplateSource(t,r,n,a){let{source:s,provenance:o}=this.expandTemplate(r.shader,mo()!==void 0),i=sm(tl(s,{...n,env:{device:Va(n.device),wgsl:{resourceDeclarations:a}}},o));if(this.renderedSourceAudit!==void 0)for(let u of this.renderedSourceAudit.unsupported(i,r.entryPoint,n.device))throw new Error(`WebGPU op ${this.manifest.id} variant ${t.id} pass ${r.id} rendered WGSL ${u}`);return i}programSpecialization(t,r,n){return hm(t.passSequence)?{passes:n.map(a=>{let s=a.pass,o={...r,repeat:a.repeat};return{id:a.id,pipelineBaseId:a.pipelineBaseId,shader:s.shader,...this.keyedPassSpecialization(s,o),...s.submitAfter?{submitAfter:!0}:{}}})}:{passes:t.passes.map(a=>({id:a.id,shader:a.shader,...this.keyedPassSpecialization(a,r),...a.submitAfter?{submitAfter:!0}:{}}))}}keyedPassSpecialization(t,r){let n=this.passDerived(t,r),a=this.assets.assumes?.(t.shader);if(a!==void 0){for(let[u,l]of Object.entries(a))if(!Object.hasOwn(n,u)||dt(n[u])!==dt(l))throw new _l(`WebGPU op ${this.manifest.id} pass ${t.id}: ${t.shader} was specialized for ${u} = ${dt(l)}, but this request derives ${Object.hasOwn(n,u)?dt(n[u]):"no value"}; the library's model hints do not cover it`)}let s=this.passKeyedReads(t),o=(u,l)=>{if(l==="all")return u;let c={};for(let d of Object.keys(u))l.has(d)&&(c[d]=u[d]);return c},i={derived:o(n,s.derived)};return s.args!==null&&(i.args=o(r.args,s.args)),s.shapes!==null&&(i.shapes=o(r.shapes,s.shapes)),s.present!==null&&(i.present=o(r.present,s.present)),i}passKeyedReads(t){let r=this.passKeyedReadsCache.get(t);if(r!==void 0)return r;let n=this.expandTemplate(t.shader,!1).source,a=kf(n),s=a.has("derive")?"all":new Set(a);if(s!=="all")for(let u of t.bindings)for(let l of["elementType"in u?u.elementType:void 0,"length"in u?u.length:void 0])typeof l=="string"&&l.startsWith("$")&&s.add(l.slice(1));let o=u=>{if(!a.has(u))return null;let l=vo(n,u);return l===null?"all":new Set(l)},i=Object.freeze({derived:s,args:o("args"),shapes:o("shapes"),present:o("present")});return this.passKeyedReadsCache.set(t,i),i}buildPlanPayload(t,r,n,a){let s=fm(t.intermediates,r,a),o=n.map(i=>this.buildPlanPass(t,i,r,a));return Object.freeze({scratches:s,passes:Object.freeze(o)})}buildPlan(t,r,n){return Object.freeze({key:t,programKey:r,...n})}allOutputsEmpty(t){let r=0;for(let[n,a]of Object.entries(this.manifest.args)){if(a.role!=="output"&&a.role!=="inout")continue;let s=t.resources[n];if(!(s==null&&a.required===!1)&&(!wt(s)||(r+=1,s.shape.reduce((o,i)=>o*i,1)!==0)))return!1}return r>0}validateDispatchWorkgroupContracts(t,r,n,a,s){if(s)return;let o=this.renderedSourceAudit;if(o!==void 0)for(let i=0;i<n.length;++i){let u=n[i],l=u.pass.dispatchWorkgroupContract;if(l===void 0)continue;let c=a.passes[i];if(c===void 0)throw new Error(`WebGPU op ${this.manifest.id} variant ${t.id} dispatch/workgroup validation could not find program pass ${u.id}`);let d={...r,repeat:u.repeat},p={...d,derived:this.passDerived(u.pass,d)},f=`WebGPU op ${this.manifest.id} variant ${t.id} pass ${u.id} (base ${u.pass.id}, shader ${u.pass.shader}) dispatch threads workgroupSize divisor`,m=JSON.stringify(l.divisor)??String(l.divisor),h;try{h=St(l.divisor,p)}catch(b){throw new Error(`${f} ${m} could not be evaluated: ${b instanceof Error?b.message:String(b)}`)}if(typeof h!="number"||!Number.isSafeInteger(h)||h<=0)throw new Error(`${f} must resolve to a positive safe integer, got ${String(h)} from ${m}`);let g=o.workgroupSize(c.source,c.entryPoint);if(g===null)throw new Error(`${f} is ${h} (from ${m}), but selected entry ${JSON.stringify(c.entryPoint)} does not have a statically resolved @workgroup_size. Make the selected entry size resolvable, or express cooperative logical tiling with an explicit workgroups dispatch.`);if(g.some(b=>!Number.isSafeInteger(b)||b<=0))throw new Error(`${f} cannot compare against invalid resolved @workgroup_size(${g.join(", ")})`);let w=BigInt(g[0])*BigInt(g[1])*BigInt(g[2]);if(w>BigInt(Number.MAX_SAFE_INTEGER))throw new Error(`${f} cannot compare against invalid resolved @workgroup_size(${g.join(", ")})`);if(BigInt(h)!==w)throw new Error(`${f} is ${h} (from ${m}), but selected entry ${JSON.stringify(c.entryPoint)} renders @workgroup_size(${g.join(", ")}) with product ${w}. Count physical invocations in threads and use the same physical product as workgroupSize, or express cooperative logical tiling with an explicit workgroups dispatch.`)}}buildPlanPass(t,r,n,a){let s=r.pass,o={...n,repeat:r.repeat};if(a)return Object.freeze({id:r.id,dispatchWorkgroups:[0,0,0],uniforms:this.zeroFilledUniforms(s)});let i={...o,derived:this.passDerived(s,o)},u=s.bindings.filter(Xa).reduce((c,d)=>(c[d.name]=this.uniformValues(d,i),c),{}),l=Cf(s.profile,i);return Object.freeze({id:r.id,dispatchWorkgroups:this.passDispatchWorkgroups(s,i),uniforms:u,...Object.keys(l).length>0?{profile:l}:{}})}zeroFilledUniforms(t){let r={};for(let n of t.bindings)Xa(n)&&(r[n.name]=Object.fromEntries(n.struct.fields.map(a=>[a.name,0])));return r}passDispatchWorkgroups(t,r){let n=r.device.limits.maxComputeWorkgroupsPerDimension;return[St(t.dispatch.x,r),St(t.dispatch.y??1,r),St(t.dispatch.z??1,r)].map((a,s)=>{if(!Number.isInteger(a)||Number(a)<0)throw new Error(`WebGPU pass ${t.id} dispatch axis ${s} must resolve to a nonnegative integer, got ${String(a)}`);if(Number(a)>n){let o=Number(a)===Tn&&n<Tn;throw new Error(`WebGPU pass ${t.id} dispatch axis ${s} = ${a} exceeds maxComputeWorkgroupsPerDimension (${n}); `+(o?`this is the 2D fold width, which every WebGPU device must support \u2014 this adapter reports less than the required ${Tn}.`:`clamp the dispatch (min(..., ${n})) and grid-stride over the remainder, or fold into 2D/3D.`))}return Number(a)})}uniformValues(t,r){let n=t.struct,a={};for(let s of n.fields){let o=s.value??r.args[s.name]??s.default;if(o===void 0)continue;let i=St(o,r),u=s.type==="f32";if(typeof i!="number"||Number.isNaN(i)||!Number.isFinite(i)&&!u)throw new Error(`WebGPU uniform ${t.name}.${s.name} must resolve to a finite number`);a[s.name]=i}return a}passDerived(t,r){let n=this.passDerivedBase.get(r.derived);n===void 0&&(n=new WeakMap,this.passDerivedBase.set(r.derived,n));let a=n.get(r.dtypes);a===void 0&&(a=Object.freeze({...r.dtypes,...r.derived}),n.set(r.dtypes,a));let s=Object.entries(t.derive);if(s.length===0)return a;let o=this.passDerivedMemo.get(r.derived);o===void 0&&(o=new Map,this.passDerivedMemo.set(r.derived,o));let i=o.get(t);i===void 0&&(i=[],o.set(t,i));for(let l of i)if(E_(l.scope,r))return l.derived;let u={...a};for(let[l,c]of s)u[l]=$t(c,{...r,derived:u});return i.push({scope:r,derived:u}),u}selectVariantAndScope(t,r,n,a){let s=this.selectVariant(t,n,r,a),o=this.applyVariantDerive(t,r,s,a);return{variant:s,scope:o}}selectVariant(t,r,n,a){let s=[],o=[],i=null,u=new Map,l=c=>{for(let d of o)Yf({opId:this.manifest.id,skipped:d.id,used:c?.id??null,constructs:d.constructs})};for(let c of this.variantCandidates(r)){let d=this.checkVariant(c,t,u,n,a);if(!d.ok){s.push(`${c.id}: ${d.rejection.message}`),d.rejection.kind==="wgslBuildFailure"&&o.push({id:c.id,constructs:d.rejection.constructs});continue}if(d.demotions.length===0)return l(c),c;i??=c}if(i!==null)return l(i),i;throw l(null),new Error(`No supported WebGPU variant for ${this.manifest.id}; rejected ${s.join("; ")}`)}variantCandidates(t){if(t!==void 0){let r=this.manifest.variants.find(n=>n.id===t);if(!r)throw new Error(`WebGPU op ${this.manifest.id} has no variant ${t}`);return[r]}return[...this.manifest.variants].sort(dl)}checkVariant(t,r,n=new Map,a,s){let o=n.get(t.id);if(o)return o;let i=this.computeVariantCheck(t,r,n,a,s);return n.set(t.id,i),i}computeVariantCheck(t,r,n,a,s){let o=P_(t,r.device),i=o.requirementReason;if(i)return{ok:!1,rejection:{kind:"requires",message:i}};let u=o.budgetReason;if(u)return{ok:!1,rejection:{kind:"bufferBudget",message:u}};let l=r,c=s?.clauseReads(t.id),d=Array.isArray(t.when)?t.when:[t.when];for(let h=0;h<d.length;h+=1){let g=d[h],w;try{w=s===void 0?St(g,l):s.evalClause(g,l,c?.when[h])}catch(b){let v=Bm(g);return{ok:!1,rejection:{kind:"whenError",clause:v,message:`when eval error in \`${v}\`: ${b.message}`}}}if(w!==!0)return{ok:!1,rejection:x_(g,w)}}for(let h of t.supersededBy??[]){let g=this.variantsById.get(h);if(!g)continue;let w=this.checkVariant(g,r,n,a,s);if(w.ok&&w.demotions.length===0)return{ok:!1,rejection:{kind:"superseded",variant:h,message:`superseded by \`${h}\`, which is eligible here and takes precedence`}}}let p=Ue(r.device).wgslCompileFailures;if(p.size>0&&a!==void 0){let h=this.wgslBuildBlockers(t,r,a,p);if(h.length>0)return{ok:!1,rejection:{kind:"wgslBuildFailure",constructs:h,message:`renders ${h.join(", ")}, which this device rejects at pipeline creation`}}}let f=[],m=t.demoteWhen??[];for(let h=0;h<m.length;h+=1){let g=m[h],w;try{w=s===void 0?St(g,l):s.evalClause(g,l,c?.demoteWhen[h])}catch(b){return{ok:!1,rejection:{kind:"demoteError",clause:g,message:`demoteWhen eval error in \`${g}\`: ${b.message}`}}}w===!0&&f.push(g)}return{ok:!0,demotions:f}}buildBaseScope(t,r){let n={...r.resources},a={...this.manifest.attributes,...r.attrs??{}};this.validateAttributes(a);let s={...r.args??{}};for(let[h,g]of Object.entries(n))!wt(g)&&typeof g!="object"&&(s[h]=g);let o=Object.fromEntries(Object.keys({...this.manifest.args,...n}).map(h=>[h,n[h]!==void 0&&n[h]!==null])),i={},u={},l={},c={},d={},p={};for(let h of this.scopeNamesByArg.keys()){let g=n[h];if(wt(g)){for(let w of this.scopeNamesByArg.get(h)){i[w]=g.shape,u[w]=g.shape.length,l[w]=g.dtype;let b=O_(g);b!==void 0&&(c[w]=b)}this.validateInputLayout(h,g)}}let f=this.validateInputDtypes(l,c);for(let[h,g]of f)d[h]=g.logicalDtype,p[h]=hf(g.storageDtype);return{device:t,attrs:a,args:s,resources:n,present:o,shapes:i,ranks:u,tensorDtypes:l,logicalDtypes:d,dtypes:p,derived:{},tunables:{...this.manifest.tunables,...r.tunables??{}},repeat:{}}}validateAttributes(t){for(let r of Object.keys(t))if(!this.acceptedAttributeKeys.has(r))throw new Error(`WebGPU op ${this.manifest.id}: unknown attribute "${r}" \u2014 not declared by the manifest (known: ${this.manifest.attributeNames.join(", ")||"none"})`);for(let[r,n]of Object.entries(this.manifest.attributeConstraints)){if(n.required&&(!Object.hasOwn(t,r)||t[r]===void 0))throw new Error(`WebGPU op ${this.manifest.id} missing required attribute ${r}`);if(!Object.hasOwn(t,r)||n.values===null)continue;let a=t[r];if(!n.values.some(s=>So(s,a)))throw new Error(`WebGPU op ${this.manifest.id} attribute ${r} must be one of [${n.values.map(s=>JSON.stringify(s)).join(", ")}], got ${JSON.stringify(a)}`)}}validateInputLayout(t,r){if(r.runtime===null||r.runtime===void 0)return;let n=this.layoutsByArg.get(t);if(r.layout!==n)throw new Error(`${this.manifest.id}: input "${t}" is bound in the ${r.layout??"widened"} storage layout, but its port declares ${n??"widened"}; upload it in the declared layout`)}validateInputDtypes(t,r){let n=new Map;for(let a of[...this.manifest.inputs,...this.manifest.outputs]){let s=a.dtype;if(!s)continue;let o=t[a.role];if(o===void 0)continue;let i=this.manifest.inputs.includes(a)?"input":"output",u=this.storageDtypesByPortRole.get(a.role);if(u!==void 0&&o!==u)throw new Error(`${this.manifest.id}: ${i} "${a.role}" has WebGPU storage dtype ${o}, not the declared ${u} storage ABI`);let l=r[a.role],c;if(Object.hasOwn(wn,s)){c=s;let p=u??s;if(o!==p)throw new Error(`${this.manifest.id}: ${i} "${a.role}" has dtype ${o}, not the declared concrete dtype ${p}`)}else{let p=this.manifest.typeConstraints[s];if(p===void 0){if(l!==void 0&&l!==o)throw new Error(`${this.manifest.id}: ${i} "${a.role}" carries logical dtype ${l} over ${o} storage, but unconstrained type variable ${s} resolves to its physical dtype`);n.set(s,{logicalDtype:o,storageDtype:o,role:a.role});continue}if(p.length===1&&p[0]==="int64"){if(o!==(u??"int64"))throw new Error(`${this.manifest.id}: ${i} "${a.role}" requires ${u??"int64"} storage for logical int64, got ${o}`);c="int64"}else{if(!p.some(f=>f===o))throw new Error(`${this.manifest.id}: ${i} "${a.role}" has dtype ${o}, not allowed by type constraint ${s} = [${p.join(", ")}]`);c=o}}if(l!==void 0&&l!==c)throw new Error(`${this.manifest.id}: ${i} "${a.role}" carries logical dtype ${l} over ${o} storage, but this port resolves to logical dtype ${c} and does not declare the same storage projection`);let d=n.get(s);if(d!==void 0){if(d.logicalDtype!==c)throw new Error(`${this.manifest.id}: ${i} "${a.role}" resolves to logical dtype ${c}, but shared type variable ${s} already resolved to ${d.logicalDtype} from "${d.role}"`);if(d.storageDtype!==o)throw new Error(`${this.manifest.id}: ${i} "${a.role}" has storage dtype ${o}, but shared type variable ${s} already resolved to ${d.storageDtype} from "${d.role}"`)}n.set(s,{logicalDtype:c,storageDtype:o,role:a.role})}return n}applyManifestDerive(t,r){if(r!==void 0)return{...t,derived:r.manifestDerive(t)};let n={};for(let[a,s]of Object.entries(this.manifest.derive))n[a]=$t(s,{...t,derived:n});return{...t,derived:n}}prepareScopeKey(t,r){return dt(this.prepareScopeKeyFields(t,r))}prepareScopeKeyFields(t,r){return{d:Pm(t,r.device),...ml(r),args:r.args,shapes:r.shapes,present:r.present}}pinVariant(t,r){return r===void 0?t:`${t}\0${r}`}applyManifestDeriveCached(t,r,n){let a=this.manifestDerivedCache.get(t);if(a!==void 0)return{...r,derived:a};let s=this.applyManifestDerive(r,n);return this.manifestDerivedCache.set(t,s.derived),s}requestScope(t,r,n){let a=n?.rememberedRequestScope(r,t);if(a!==void 0)return a;let s=Ue(t),o=n?.movedRequestScope(r,t,s,Pm(t,s));if(o!==void 0)return o;let i=this.buildBaseScope(s,r);return n===void 0?{rawDevice:t,baseScope:i,scopeKey:this.prepareScopeKey(t,i),validated:!1}:n.rememberRequestScope(r,t,i,this.prepareScopeKeyFields(t,i))}applyManifestDeriveForInference(t){let r={};for(let[n,a]of B_(this.manifest))try{r[n]=$t(a,{...t,derived:r})}catch{}return{...t,derived:r}}applyVariantDerive(t,r,n,a){let s={...t,tunables:{...this.manifest.tunables,...n.tunables,...r.tunables??{}}},o={...t.derived};if(a!==void 0)a.variantDerive(n,s,o);else for(let[i,u]of Object.entries(n.derive))o[i]=$t(u,{...s,derived:o});return{...s,derived:o}}rejectUnknownKeys(t,r,n,a){for(let s of Object.keys(n))if(!this.acceptedResourceKeys.has(s))throw new Error(`${t}: unknown ${r} "${s}" \u2014 not a declared arg or binding name (known: ${[...this.acceptedResourceKeys].sort().join(", ")})`);for(let s of Object.keys(a??{}))if(!this.acceptedArgKeys.has(s))throw new Error(`${t}: unknown arg "${s}" \u2014 not declared in manifest.args (known: ${[...this.acceptedArgKeys].sort().join(", ")})`)}validateRequest(t){this.rejectUnknownKeys(`WebGPU op ${this.manifest.id}`,"resource",t.resources,t.args);for(let[r,n]of Object.entries(this.manifest.args)){let a=t.resources[r]??t.args?.[r];if(a==null){if(n.required!==!1)throw new Error(`WebGPU op ${this.manifest.id} missing required arg ${r}`);continue}G_(r,n,a)}}},Sm=/([ \t]*){%\s*include\s+["']([^"']+)["']\s*%}([ \t]*\r?\n)?/g;function xm(e,t,r){if(r===void 0)return!1;let n=t===0?`
`:e[t-1];return n===`
`||n==="\r"}var Em=Object.freeze([]);function Rm(e,t,r,n){return Ro(e,t,r,n)}function R_(e,t,r,n){let a={expandedOffset:0,spans:[]},s=rs(e),o=Ro(s,t,r,n,a,Em);return Object.freeze({source:o,provenance:Object.freeze({rootKey:s,spans:Object.freeze(a.spans)})})}function Ro(e,t,r,n,a,s=Em){let o=rs(e);if(n.has(o))throw new Error(`Circular WebGPU template include: ${[...n,o].join(" -> ")}`);if(n.add(o),a===void 0){let m=t.replace(Sm,(h,g,w,b,v)=>{let y=Tm(o,w),S=Ro(y,r(y),r,n);return xm(t,v,b)?S:`${g}${S}${b??""}`});return n.delete(o),m}let i=0,u=0,l=1,c=1,d=(m,h)=>{for(let g=m;g<h;g+=1){let w=t.charCodeAt(g);w===10||w===11||w===12||w===13||w===133||w===8232||w===8233?(l+=1,c=1,w===13&&g+1<h&&t.charCodeAt(g+1)===10&&(g+=1)):c+=1}},p=(m,h)=>{if(m===h)return;let g=h-m;a.spans.push(Object.freeze({expandedStart:a.expandedOffset,expandedEnd:a.expandedOffset+g,sourceKey:o,sourceStart:m,sourceEnd:h,sourceLine:l,sourceColumn:c,includeInstancePath:s})),a.expandedOffset+=g},f=t.replace(Sm,(m,h,g,w,b)=>{let v=xm(t,b,w),y=b+h.length,S=b+m.length,C=S-(w?.length??0);p(u,v?b:y);let k=v?S:C;d(u,k),u=k;let B=Tm(o,g),P=Object.freeze([...s,i]);i+=1;let _=Ro(B,r(B),r,n,a,P);return v?_:`${h}${_}${w??""}`});return p(u,t.length),n.delete(o),f}function Tm(e,t){if(t.startsWith(".")){let r=e.slice(0,Math.max(0,e.lastIndexOf("/")));return rs(`${r}/${t}`)}return rs(t)}function rs(e){let t=[];for(let r of e.replaceAll("\\","/").split("/"))if(!(r===""||r===".")){if(r===".."){if(t.length===0)throw new Error(`WebGPU template path escapes package root: ${e}`);t.pop();continue}t.push(r)}return t.join("/")}function G_(e,t,r){if(t.kind==="tensor"){if(!wt(r))throw new Error(`WebGPU arg ${e} must be a GPU tensor`);if(t.dtype&&r.dtype!==t.dtype)throw new Error(`WebGPU arg ${e} dtype ${r.dtype} does not match ${t.dtype}`);return}if(t.kind==="bool"){if(typeof r!="boolean")throw new Error(`WebGPU arg ${e} must be a boolean`);return}if(t.kind==="string"){if(typeof r!="string")throw new Error(`WebGPU arg ${e} must be a string`)}else{if(typeof r!="number"||!Number.isFinite(r))throw new Error(`WebGPU arg ${e} must be a finite number`);if(t.kind==="u32"&&(!Number.isInteger(r)||r<0||r>4294967295))throw new Error(`WebGPU arg ${e} must be a u32`);if(t.kind==="i32"&&(!Number.isInteger(r)||r<-2147483648||r>2147483647))throw new Error(`WebGPU arg ${e} must be an i32`)}if(t.oneOf&&!t.oneOf.includes(r))throw new Error(`WebGPU arg ${e} must be one of ${t.oneOf.join(", ")}`)}function L_(e){return typeof e.readText=="function"?e:Cn(e)}var Cm=new WeakMap;function Pm(e,t){if(e&&typeof e=="object"){let r=e,n=Cm.get(e);if(n!==void 0&&n.features===r.features&&n.wgslLanguageFeatures===r.wgslLanguageFeatures&&n.limits===r.limits&&n.adapterInfo===r.adapterInfo&&n.wgslCompileFailures===r.wgslCompileFailures)return n.serial;let a=Im(dt(Nt(t)));return Cm.set(e,{serial:a,features:r.features,wgslLanguageFeatures:r.wgslLanguageFeatures,limits:r.limits,adapterInfo:r.adapterInfo,wgslCompileFailures:r.wgslCompileFailures}),a}return Im(dt(Nt(t)))}var bl=new Map;function Im(e){let t=bl.get(e);if(t!==void 0)return t;let r=`d${bl.size}`;return bl.set(e,r),r}function es(e){return e!==null&&typeof e=="object"&&typeof e.dtype=="string"&&Array.isArray(e.shape)}function q_(e){let t=e.shape.reduce((r,n)=>r*n,1);return{dtype:e.dtype,...e.logicalDtype===void 0?{}:{logicalDtype:e.logicalDtype},shape:e.shape,size:t,byteLength:t*4,buffer:{},runtime:null}}function O_(e){if(e===null||typeof e!="object")return;let t=e.logicalDtype;if(t!==void 0){if(typeof t=="string"&&Object.hasOwn(wn,t))return t;throw new Error(`tensor has unsupported logical dtype ${String(t)}`)}}var Gm=new Map([["ops/_shared/flat-index-2d.812cc3cd16.wgsl.jinja",`{% macro flat_index_2d(workgroupSize, name="i", bound="params.count") %}{% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}let {{ name }}=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ wgTerm }};
{% if bound != "" %}if({{ name }}>={{ bound }}){return;}
{% endif %}{% endmacro %}`],["ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.812cc3cd16.wgsl.jinja" %}{% if vec4PerThread > 1 %}const ITEMS:u32={{ vec4PerThread }}u;
{% endif %}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){
{% if vec4PerThread > 1 %}{{ flat_index_2d(tunables.WORKGROUP_SIZE, "tid", "") }} let span=(params.count+ITEMS-1u)/ITEMS;if(tid>=span){return;}for(var j=0u;j<ITEMS;j=j+1u){let i=tid+j*span;if(i>=params.count){break;}
{% else %}{{ flat_index_2d(tunables.WORKGROUP_SIZE) }}
{% endif %}`],["ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja",`{% if vec4PerThread > 1 %}}
{% endif %}`],["ops/_shared/broadcast-offset.9034aa7baf.wgsl.jinja",`{% macro offset_fn(fn_name, opShape, opRank, op_same, op_numel, outShape, outRank, out_numel) %}fn {{ fn_name }}({% if out_numel != 0 and op_numel != 1 %}out_index: u32{% endif %}) -> u32 {
{% if out_numel == 0 %}return 0u;
{% elif op_numel == 1 %}return 0u;
{% elif op_same %}return out_index;
{% else %}var offset=0u;
{% for axis in range(outRank) %}{% set op_axis = axis - (outRank - opRank) %}{% if op_axis >= 0 and opShape[op_axis] != 1 %}{% set c_stride = namespace(value=1) %}{% for j in range(axis + 1, outRank) %}{% set c_stride.value = c_stride.value * outShape[j] %}{% endfor %}{% set op_stride = namespace(value=1) %}{% for j in range(op_axis + 1, opRank) %}{% set op_stride.value = op_stride.value * opShape[j] %}{% endfor %}{% if c_stride.value == 1 %}let coord{{ axis }}=out_index% {{ outShape[axis] }}u;
{% else %}let coord{{ axis }}=(out_index/{{ c_stride.value }}u)% {{ outShape[axis] }}u;
{% endif %}{% if op_stride.value == 1 %}offset=offset+coord{{ axis }};
{% else %}offset=offset+coord{{ axis }}*{{ op_stride.value }}u;
{% endif %}{% endif %}{% endfor %}return offset;
{% endif %}}{% endmacro %}{% macro broadcast_offset_call(fn_name, opShape, outShape, out_index) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{{ fn_name }}({% if out_numel.value != 0 and op_numel.value != 1 %}{{ out_index }}{% endif %}){% endmacro %}`],["ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja",`{% macro flat_index_2d(workgroupSize, name="i", bound="params.count") %}{% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}let {{ name }}=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ wgTerm }};if({{ name }}>={{ bound }}){return;}
{% endmacro %}`],["ops/_shared/binary-broadcast-operand.5796cb977c.wgsl.jinja",`{% macro periodic_geometry(g, shape, rank, same, count, outputShape, outputRank, crosses, capacity) -%}{%- for axis in range(outputRank) | reverse -%}{%- set operandAxis = axis - (outputRank - rank) -%}{%- if g.matching and operandAxis >= 0 and shape[operandAxis] == outputShape[axis] -%}{%- set g.period = g.period * outputShape[axis] -%}{%- else -%}{%- set g.matching = false -%}{%- endif -%}{%- endfor -%}{%- set g.active = not same and crosses and g.period >= 4 and g.period == count and g.period <= capacity -%}{%- endmacro %}{% macro periodic_declaration(name, g, acc) -%}{% if g.active %}var<workgroup>table_{{ name }}:array<vec4<{{ acc }}>,{{ g.period }}>;
{% endif %}{%- endmacro %}{% macro periodic_stage(name, g, acc) -%}{% if g.active %}if(lid<{{ g.period }}u){let base_{{ name }}=(lid*4u)% {{ g.period }}u;table_{{ name }}[lid]=vec4<{{ acc }}>(
{% for lane in range(4) %}      {{ acc }}({{ name }}[{% if lane == 0 %}base_{{ name }}{% else %}(base_{{ name }} + {{ lane }}u) % {{ g.period }}u{% endif %}]){{ "," if lane < 3 else "" }}
{% endfor %});}
{% endif %}{%- endmacro %}{% macro broadcast_vector_load(name, shape, same, mode, g, acc, crosses, outputShape) -%}{% if same %}let {{ name }}v=vec4<{{ acc }}>({{ name }}[i4]);
{% elif mode == "scalar" %}let {{ name }}v=vec4<{{ acc }}>({{ acc }}({{ name }}[0]));
{% elif g.active %}let {{ name }}v=table_{{ name }}[i4% {{ g.period }}u];
{% elif crosses %}let {{ name }}v=vec4<{{ acc }}>(
{% for lane in range(4) %}{{ acc }}({{ name }}[{{ broadcast_offset_call(name ~ "_offset", shape, outputShape, "base" if lane == 0 else "base + " ~ lane ~ "u") }}]){{ "," if lane < 3 else "" }}
{% endfor %});
{% elif mode == "splat" %}let {{ name }}v=vec4<{{ acc }}>({{ acc }}({{ name }}[{{ broadcast_offset_call(name ~ "_offset", shape, outputShape, "base") }}]));
{% else %}let {{ name }}o={{ broadcast_offset_call(name ~ "_offset", shape, outputShape, "base") }};let {{ name }}v=vec4<{{ acc }}>(
{% for lane in range(4) %}    {{ acc }}({{ name }}[{{ name }}o{% if lane != 0 %} + {{ lane }}u{% endif %}]){{ "," if lane < 3 else "" }}
{% endfor %});
{% endif %}{%- endmacro %}`],["ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja",`{% macro flat_tail_open() %}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let invocation=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ tunables.WORKGROUP_SIZE }}u;
{% set itemsPerInvocation = itemsPerInvocation if itemsPerInvocation is defined else 4 %}let begin=invocation*{{ itemsPerInvocation }}u;let end=min(begin+{{ itemsPerInvocation }}u,params.count);for(var i=begin;i<end;i=i+1u){
{% endmacro %}{% macro flat_tail_close() %}  }{% endmacro %}`],["ops/_shared/broadcast-offset.25bcd5874d.wgsl.jinja",`{% macro offset_fn(fn_name, opShape, opRank, op_same, op_numel, outShape, outRank, out_numel) %}fn {{ fn_name }}({% if out_numel != 0 and op_numel != 1 %}out_index: u32{% endif %}) -> u32 {
{% if out_numel == 0 %}return 0u;
{% elif op_numel == 1 %}return 0u;
{% elif op_same %}return out_index;
{% else %}var offset=0u;
{% for axis in range(outRank) %}{% set op_axis = axis - (outRank - opRank) %}{% if op_axis >= 0 and opShape[op_axis] != 1 %}{% set c_stride = namespace(value=1) %}{% for j in range(axis + 1, outRank) %}{% set c_stride.value = c_stride.value * outShape[j] %}{% endfor %}{% set op_stride = namespace(value=1) %}{% for j in range(op_axis + 1, opRank) %}{% set op_stride.value = op_stride.value * opShape[j] %}{% endfor %}{% if c_stride.value == 1 %}let coord{{ axis }}=out_index% {{ outShape[axis] }}u;
{% else %}let coord{{ axis }}=(out_index/{{ c_stride.value }}u)% {{ outShape[axis] }}u;
{% endif %}{% if op_stride.value == 1 %}offset=offset+coord{{ axis }};
{% else %}offset=offset+coord{{ axis }}*{{ op_stride.value }}u;
{% endif %}{% endif %}{% endfor %}return offset;
{% endif %}}{% endmacro %}{% macro broadcast_offset_call(fn_name, opShape, outShape, out_index) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{{ fn_name }}({% if out_numel.value != 0 and op_numel.value != 1 %}{{ out_index }}{% endif %}){% endmacro %}{% macro broadcast_offset_fn(fn_name, opShape, opRank, outShape, outRank) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{% set op_same = namespace(value=(opRank == outRank)) %}{% if op_same.value %}{% for axis in range(outRank) %}{% if opShape[axis] != outShape[axis] %}{% set op_same.value = false %}{% endif %}{% endfor %}{% endif %}{{ offset_fn(fn_name, opShape, opRank, op_same.value, op_numel.value, outShape, outRank, out_numel.value) }}{% endmacro %}{% macro binary_broadcast_offsets() %}{% set aShape = aShape | default([]) %}{% set aRank = aRank | default(0) %}{% set bShape = bShape | default([]) %}{% set bRank = bRank | default(0) %}{% set cShape = cShape | default([]) %}{% set cRank = cRank | default(0) %}{{ broadcast_offset_fn("a_offset", aShape, aRank, cShape, cRank) }}
{{ broadcast_offset_fn("b_offset", bShape, bRank, cShape, cRank) }}{% endmacro %}`],["ops/_shared/subgroup-matrix-enable.de53490305.wgsl.jinja",`enable subgroups;
{% if pinSubgroupSize32 %}enable subgroup_size_control;
{% endif %}enable chromium_experimental_subgroup_matrix;diagnostic(off,chromium.subgroup_matrix_uniformity);`],["ops/_shared/subgroup-matrix-topology.5c216ddcce.wgsl.jinja","var<workgroup>matrixRoleCount:atomic<u32>;var<workgroup>matrixRejected:atomic<u32>;var<workgroup>matrixAccepted:u32;fn claimMatrixSubgroupRole(width:u32,requiredWidth:u32)->u32{let population=subgroupBallot(true);let populationCount=countOneBits(population.x)+countOneBits(population.y)+countOneBits(population.z)+countOneBits(population.w);var claimed=0u;if(subgroupElect()){claimed=atomicAdd(&matrixRoleCount,1u);if(width!=requiredWidth||populationCount!=requiredWidth){atomicStore(&matrixRejected,1u);} }workgroupBarrier();return subgroupBroadcastFirst(claimed);}fn matrixTopologyAccepted(localIndex:u32,bankCount:u32)->bool{if(localIndex==0u){matrixAccepted=select(0u,1u,atomicLoad(&matrixRejected)==0u&&atomicLoad(&matrixRoleCount)==bankCount);}return workgroupUniformLoad(&matrixAccepted)!=0u;}"],["ops/_shared/subgroup-matrix-software-8x8.cedda9952e.wgsl.jinja",`var<private>matrixSoftwareLane:u32;fn matrixSoftwareMultiplyAccumulate(lhs:array<f32,8>,rhs:array<vec2<f32>,8>,acc:vec2<f32>)->vec2<f32>{var value=acc;for(var k=0u;k<8u;k+=1u){value=fma(vec2<f32>(lhs[k]),rhs[k],value);}return value;}
{% macro matrixType(software, kind, scalar) %}{% if software %}{% if kind == "left" %}array<f32, 8>{% elif kind == "right" %}array<vec2<f32>, 8>{% else %}vec2<{{ scalar }}>{% endif %}{% else %}subgroup_matrix_{{ kind }}<{{ scalar }}, 8, 8>{% endif %}{% endmacro %}{% macro matrixLoad(software, kind, scalar, major, pointer, offset, stride) %}{% if not software %}subgroupMatrixLoad<{{ matrixType(false, kind, scalar) }}, {{ major }}>({{ pointer }}, {{ offset }}, {{ stride }}){% else %}{% if kind == "left" %}array<f32, 8>(
{% for k in range(8) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}(matrixSoftwareLane / 4u) * ({{ stride }}) + {{ k }}u{% else %}{{ k }}u * ({{ stride }}) + matrixSoftwareLane / 4u{% endif %}]){{ "," if k < 7 else "" }}
{% endfor %})
{% else %}array<vec2<f32>, 8>(
{% for k in range(8) %}vec2<f32>(
{% for h in range(2) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}{{ k }}u * ({{ stride }}) + (matrixSoftwareLane % 4u) * 2u + {{ h }}u{% else %}((matrixSoftwareLane % 4u) * 2u + {{ h }}u) * ({{ stride }}) + {{ k }}u{% endif %}]){{ "," if h == 0 else "" }}
{% endfor %}){{ "," if k < 7 else "" }}
{% endfor %})
{% endif %}{% endif %}{% endmacro %}{% macro matrixMma(software, scalar, lhs, rhs, acc) %}{% if not software %}subgroupMatrixMultiplyAccumulate({{ lhs }}, {{ rhs }}, {{ acc }}){% else %}vec2<{{ scalar }}>(matrixSoftwareMultiplyAccumulate({{ lhs }}, {{ rhs }}, vec2<f32>({{ acc }}))){% endif %}{% endmacro %}{% macro matrixStore(software, major, pointer, offset, value, stride) %}{% if not software %}subgroupMatrixStore<{{ major }}>({{ pointer }}, {{ offset }}, {{ value }}, {{ stride }});{% else %}{
let matrixPointer={{ pointer }};let matrixValue={{ value }};
{% for h in range(2) %}  (*matrixPointer)[({{ offset }}) + {{ {"row_major": "(matrixSoftwareLane / 4u) * (" ~ stride ~ ") + (matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u", "col_major": "((matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u) * (" ~ stride ~ ") + matrixSoftwareLane / 4u"}[major] }}] = matrixValue.{{ "x" if h == 0 else "y" }};
{% endfor %}}{% endif %}{% endmacro %}`],["ops/_shared/subgroup-matrix-8mma-step.fac6af68e9.wgsl.jinja",`{% set directInputs = directMatrixInputs is defined and directMatrixInputs %}      let matrix_a_offset = {% if directInputs %}(a_global_base + subtile_idy * SUB_ROWS) * K + kidx + step{% else %}subtile_idy * SUB_ROWS * TILE_K + step{% endif %};
{% for r in range(2) %}var matA{{ r }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&" ~ ("w" if directInputs else "tile_A"), ("matrix_a_offset" ~ (" + 8u * " ~ ("K" if directInputs else "TILE_K") if r > 0 else "")), "K" if directInputs else "TILE_K") }};
{% endfor %}      let matrix_b_offset = {% if directInputs %}b_base + (kidx + step) * N + b_global_base + subtile_idx * SUB_COLS{% else %}subtile_idx * SUB_COLS * TILE_K + step{% endif %};
{% for c in range(4) %}      var matB{{ c }}: {{ matrixType(matrixSoftware, "right", operandScalar) }} = {% set stepBPointer %}&{{ "xm" if directInputs else "tile_B" }}{% endset %}{{ matrixLoad(matrixSoftware, "right", operandScalar, ("row_major" if directInputs else "col_major"), stepBPointer, ("matrix_b_offset" ~ (" + " ~ (c * 8) ~ "u" ~ (" * TILE_K" if not directInputs else "") if c > 0 else "")), "N" if directInputs else "TILE_K") }};
{% endfor %}matC00={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB0", "matC00") }};matC01={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB1", "matC01") }};matC02={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB2", "matC02") }};matC03={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB3", "matC03") }};matC10={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB0", "matC10") }};matC11={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB1", "matC11") }};matC12={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB2", "matC12") }};matC13={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB3", "matC13") }};`],["ops/_shared/conv-gemm-store-out.6930a83d54.wgsl.jinja",`{% if emitConvStoreOut | default(true) %}fn store_out(m: u32, n: u32, yBase: u32, raw: f32) {
if(m>=params.M||n>=params.N){return;}var v=raw;
{% if hasBias %}v=v+f32(bias[m]);
{% endif %}y[yBase+m*params.N+n]={{ T }}(v);}
{% endif %}`],["ops/_shared/conv-gemm-store-out.b342b63455.wgsl.jinja","fn store_out(m:u32,n:u32,yBase:u32,raw:f32){if(m>=params.M||n>=params.N){return;}var v=raw;v=v+f32(bias[m]);y[yBase+m*params.N+n]={{ T }}(v);}"],["ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja",`{% macro emit_precise_sincos_half_pi(width=1) %}{% set t = "vec4<f32>" if width == 4 else "f32" %}fn precise_sincos_half_pi{{ "4" if width == 4 else "" }}(x:{{ t }})->{{ "mat2x4<f32>" if width == 4 else "vec2<f32>" }} {let x2=x*x;var sinPolynomial={{ "vec4<f32>(" if width == 4 else "" }}1.6059043836821613e-10{{ ")" if width == 4 else "" }};sinPolynomial=fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-2.505210838544172e-8{{ ")" if width == 4 else "" }});sinPolynomial=fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}2.7557319223985893e-6{{ ")" if width == 4 else "" }});sinPolynomial=fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-1.984126984126984e-4{{ ")" if width == 4 else "" }});sinPolynomial=fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}8.333333333333333e-3{{ ")" if width == 4 else "" }});sinPolynomial=fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-1.6666666666666666e-1{{ ")" if width == 4 else "" }});let sine=x*fma(sinPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}1.0{{ ")" if width == 4 else "" }});var cosPolynomial={{ "vec4<f32>(" if width == 4 else "" }}2.08767569878681e-9{{ ")" if width == 4 else "" }};cosPolynomial=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-2.755731922398589e-7{{ ")" if width == 4 else "" }});cosPolynomial=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}2.48015873015873e-5{{ ")" if width == 4 else "" }});cosPolynomial=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-1.388888888888889e-3{{ ")" if width == 4 else "" }});cosPolynomial=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}4.1666666666666664e-2{{ ")" if width == 4 else "" }});cosPolynomial=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}-5.0e-1{{ ")" if width == 4 else "" }});let cosine=fma(cosPolynomial,x2,{{ "vec4<f32>(" if width == 4 else "" }}1.0{{ ")" if width == 4 else "" }});return {{ "mat2x4<f32>" if width == 4 else "vec2<f32>" }}(cosine,sine);}
{% endmacro %}`],["ops/_shared/precise-trig-f32.5a0bc5b040.wgsl.jinja",'{% include "ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja" %}{{ emit_precise_sincos_half_pi() }}'],["ops/_shared/unary-numeric-helpers.5eb0cb613d.wgsl.jinja",`{% include "ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja" %}{% macro emit_trig_reduction_support() %}{% include "ops/_shared/precise-trig-f32.5a0bc5b040.wgsl.jinja" %}const TWO_OVER_PI:array<u32,14>=array<u32,14>(0xa2f983u,0x6e4e44u,0x1529fcu,0x2757d1u,0xf534ddu,0xc0db62u,0x95993cu,0x439041u,0xfe5163u,0xabdebbu,0xc561b7u,0x246e3au,0x424dd2u,0xe00649u);const PIO2_F:f32=1.5707963267948966;const INV_PIO2_F:f32=0.6366197723675814;const PIO2_HI_F:f32=1.570796251296997;const PIO2_LO_F:f32=7.549789415861596e-8;const REDUCE_THRESHOLD:f32={{ "65536.0" if robustTanReduction | default(false) else "1.0e4" }};struct Pio2{octant:u32,r:f32};fn reduce_pio2_fast(ax:f32)->Pio2{let nFloat=floor(fma(ax,INV_PIO2_F,0.5));var residual=fma(-nFloat,PIO2_HI_F,ax);residual=fma(-nFloat,PIO2_LO_F,residual);var out:Pio2;out.octant=u32(nFloat)&3u;out.r=residual;return out;}fn reduce_pio2(ax:f32)->Pio2{let ix=bitcast<u32>(ax);let e=(ix>>23u)&0xffu;let mant=(ix&0x7fffffu)|0x800000u;let p=i32(e)-150;var a0=0u;var a1=0u;var a2=0u;for(var k=0u;k<14u;k=k+1u){let shift=p-24*(i32(k)+1)+94;if(shift<=-48||shift>=96){continue;}let t=TWO_OVER_PI[k];let mlo=mant&0xffffu;let mhi=mant>>16u;let tlo=t&0xffffu;let thi=t>>16u;let ll=mlo*tlo;let mid=mlo*thi+mhi*tlo;let hh=mhi*thi;let plo=ll+((mid&0xffffu)<<16u);let carry=select(0u,1u,plo<ll);let phi32=hh+(mid>>16u)+carry;var v0=0u;var v1=0u;var off=0;if(shift<0){let s=u32(-shift);if(s<32u){v0=(plo>>s)|(phi32<<(32u-s));v1=phi32>>s;}else{v0=phi32>>(s-32u);v1=0u;}off=0;}else{v0=plo;v1=phi32;off=shift;}let li=off/32;let b=u32(off%32);var w0=0u;var w1=0u;var w2=0u;if(b==0u){w0=v0;w1=v1;w2=0u;}else{w0=v0<<b;w1=(v1<<b)|(v0>>(32u-b));w2=v1>>(32u-b);}var t0=0u;var t1=0u;var t2=0u;if(li==0){t0=w0;t1=w1;t2=w2;}else if(li==1){t1=w0;t2=w1;}else{t2=w0;}let n0=a0+t0;let c0=select(0u,1u,n0<a0);let n1a=a1+t1;let c1a=select(0u,1u,n1a<a1);let n1=n1a+c0;let c1b=select(0u,1u,n1<n1a);let n2=a2+t2+c1a+c1b;a0=n0;a1=n1;a2=n2;}var n=a2>>30u;var phi=f32(a2&0x3fffffffu)*(1.0/1073741824.0)+f32(a1)*(1.0/4611686018427387904.0);if(phi>=0.5){phi=phi-1.0;n=n+1u;}var out:Pio2;out.octant=n&3u;out.r=phi*PIO2_F;return out;}
{% endmacro %}{% macro emit_cos_accurate() %}fn cos_accurate(x:f32)->f32{let ax=abs(x);let ax_bits=bitcast<u32>(ax);if(((ax_bits>>23u)&0xffu)==0xffu){return bitcast<f32>(ax_bits|0x00400000u);}var red:Pio2;if(ax<REDUCE_THRESHOLD){red=reduce_pio2_fast(ax);}else{red=reduce_pio2(ax);}let reduced=precise_sincos_half_pi(red.r);switch(red.octant){case 0u:{return reduced.x;}case 1u:{return-reduced.y;}case 2u:{return-reduced.x;}default:{return reduced.y;} } }
{% endmacro %}{{ emit_trig_reduction_support() }} {{ emit_cos_accurate() }}`],["ops/_shared/fft-residual-protected-arithmetic.547d7e2074.wgsl.jinja","fn fft_round_bits(x:f32)->f32{let bits=bitcast<u32>(x);return bitcast<f32>(select(0u,bits,(bits&0x7fffffffu)!=0u));}fn fft_pair_add(a:vec2<f32>,b:vec2<f32>)->vec2<f32>{let swap=abs(a.x)<abs(b.x);let large=select(a.x,b.x,swap);let small=select(b.x,a.x,swap);let sum=fft_round_bits(large+small);let delta=fft_round_bits(sum-large);let error=(small-delta)+(a.y+b.y);let high=fft_round_bits(sum+error);return vec2<f32>(high,error-fft_round_bits(high-sum));}fn fft_product_error(a:f32,b:f32,p:f32)->f32{let ah=bitcast<f32>(bitcast<u32>(a)&0xfffff000u);let al=fma(-1.0,ah,a);let bh=bitcast<f32>(bitcast<u32>(b)&0xfffff000u);let bl=fma(-1.0,bh,b);let error=fma(fma(fma(ah,bh,-p),1.0,ah*bl),1.0,al*bh);return fma(error,1.0,al*bl);}fn fft_pair_mul(a:vec2<f32>,b:vec2<f32>)->vec2<f32>{let p=fma(a.x,b.x,0.0);let e=fma(a.y,b.y,fma(a.x,b.y,fma(a.y,b.x,fft_product_error(a.x,b.x,p))));let hi=fft_round_bits(p+e);return vec2<f32>(hi,e-fft_round_bits(hi-p));}"],["ops/_shared/fft-quarter-twiddle-pair.efbc1c05d1.wgsl.jinja","fn fft_quarter_twiddle_pair(phase:u32,period:u32)->vec4<f32>{let p=phase%period;let quadrant=(4u*p+period/2u)/period;let remainder=f32(i32(4u*p)-i32(quadrant*period));let t=remainder/f32(period);let tl=fma(-t,f32(period),remainder)/f32(period);let x=fft_pair_mul(vec2<f32>(t,tl),vec2<f32>(1.5707963705062866,-4.3711390001862426e-8));let x2=fft_pair_mul(x,x);var cp=vec2<f32>(4.7794772561329454e-14,7.6254440444864298e-22);cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(-1.147074536050896e-11,-2.3722076892312381e-19));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(2.0876755879584152e-09,1.1082839147459852e-16));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(-2.755731998149713e-07,7.5751122090511949e-15));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(2.4801587642286904e-05,-3.4069960936668198e-13));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(-0.0013888889225199819,3.3631094437103215e-11));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(0.041666667908430099,-1.2417634698280722e-09));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(-0.5,0));cp=fft_pair_add(fft_pair_mul(cp,x2),vec2<f32>(1,0));var sp=vec2<f32>(2.8114573589663704e-15,-1.0462084739763658e-22);sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(-7.6471636098127127e-13,-1.2200710471178288e-20));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(1.6059044372074283e-10,-5.3525265115627256e-18));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(-2.5052107943679403e-08,-4.4176230446483665e-16));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(2.7557318844628753e-06,3.7935712242972291e-14));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(-0.00019841270113829523,2.7255968749334558e-12));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(0.0083333337679505348,-4.3461720333759502e-10));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(-0.1666666716337204,4.9670538793122887e-09));sp=fft_pair_add(fft_pair_mul(sp,x2),vec2<f32>(1,0));let c=cp;let sn=fft_pair_mul(sp,x);switch(quadrant%4u){case 0u:{return vec4<f32>(c,-sn);}case 1u:{return vec4<f32>(-sn,-c);}case 2u:{return vec4<f32>(-c,sn);}default:{return vec4<f32>(sn,c);} } }"],["ops/_shared/precise-trig-f32.c64e28bb9f.wgsl.jinja",`const PRECISE_TRIG_PI:f32=3.141592653589793;const PRECISE_TRIG_TWO_PI:f32=6.283185307179586;const PRECISE_TRIG_HALF_PI:f32=1.5707963267948966;
{% include "ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja" %}{{ emit_precise_sincos_half_pi() }}
fn precise_sincos_centered(x:f32)->vec2<f32>{var folded=x;var cosineSign=1.0;if(folded>PRECISE_TRIG_HALF_PI){folded=PRECISE_TRIG_PI-folded;cosineSign=-1.0;}else if(folded<-PRECISE_TRIG_HALF_PI){folded=-PRECISE_TRIG_PI-folded;cosineSign=-1.0;}let value=precise_sincos_half_pi(folded);return vec2<f32>(cosineSign*value.x,value.y);}`],["ops/_shared/dft-precise-twiddle.16fc24b25e.wgsl.jinja",'{% include "ops/_shared/precise-trig-f32.c64e28bb9f.wgsl.jinja" %}fn dft_precise_twiddle(phase:u32,period:u32,sign:f32)->vec2<f32>{let reducedPhase=phase%period;if(reducedPhase==0u){return vec2<f32>(1.0,sign*0.0);}if(period%4u==0u){let quarter=period/4u;if(reducedPhase==quarter){return vec2<f32>(0.0,sign);}if(reducedPhase==2u*quarter){return vec2<f32>(-1.0,sign*0.0);}if(reducedPhase==3u*quarter){return vec2<f32>(0.0,-sign);} }let turns=f32(reducedPhase)/f32(period);let centeredTurns=turns-floor(turns+0.5);return precise_sincos_centered(sign*PRECISE_TRIG_TWO_PI*centeredTurns);}'],["ops/_shared/float-io-cast.9e460ce304.wgsl.jinja","{% macro io_f32(expr) %}{{ expr }}{% endmacro %}{% macro io_element(expr) %}{{ expr }}{% endmacro %}"],["ops/_shared/normalize-index.fbcdd05a27.wgsl.jinja","fn normalize_index(raw:{{ indexScalar }},dim:u32)->u32{let dim_i=i32(dim);var index=raw;if(index<0){index=index+dim_i;}if(index<0||index>=dim_i){return INVALID_INDEX;}return u32(index);}"],["ops/_shared/gelu-erf.a53d9fe8bd.wgsl.jinja","fn erf_approx(x:f32)->f32{let ax=abs(x);if(ax<9.5367431640625e-7){return 1.1283791670955126*x;}let sign=select(-1.0,1.0,x>=0.0);let t=1.0/(1.0+0.3275911*ax);let y=1.0-(((((1.061405429*t-1.453152027)*t)+1.421413741)*t-0.284496736)*t+0.254829592)*t*exp(-(ax*ax));return sign*y;}"],["ops/_shared/unary-numeric-helpers.8bfc3a16e6.wgsl.jinja",`{% macro emit_gelu_value(approximation) %}fn gelu_value(v:f32)->f32{
{% if approximation == "tanh" %}return 0.5*v*(1.0+tanh_safe(0.7978845608028654*(v+0.044715*v*v*v)));
{% else %}return 0.5*v*(1.0+erf_approx(v*0.7071067811865476));
{% endif %}}
{% endmacro %}{% include "ops/_shared/gelu-erf.a53d9fe8bd.wgsl.jinja" %}{{ emit_gelu_value(approximateSpec) }}`],["ops/_shared/unary-numeric-helpers.ba735b4ed2.wgsl.jinja",`{% macro emit_gelu_value(approximation) %}fn gelu_value(v:f32)->f32{
{% if approximation == "tanh" %}return 0.5*v*(1.0+tanh_safe(0.7978845608028654*(v+0.044715*v*v*v)));
{% else %}return 0.5*v*(1.0+erf_approx(v*0.7071067811865476));
{% endif %}}
{% endmacro %}{% include "ops/_shared/gelu-erf.a53d9fe8bd.wgsl.jinja" %}{{ emit_gelu_value(approximate) }}`],["ops/_shared/broadcast-offset.87f1ae294f.wgsl.jinja",`{% macro offset_fn(fn_name, opShape, opRank, op_same, op_numel, outShape, outRank, out_numel) %}fn {{ fn_name }}({% if out_numel != 0 and op_numel != 1 %}out_index: u32{% endif %}) -> u32 {
{% if out_numel == 0 %}return 0u;
{% elif op_numel == 1 %}return 0u;
{% elif op_same %}return out_index;
{% else %}var offset=0u;
{% for axis in range(outRank) %}{% set op_axis = axis - (outRank - opRank) %}{% if op_axis >= 0 and opShape[op_axis] != 1 %}{% set c_stride = namespace(value=1) %}{% for j in range(axis + 1, outRank) %}{% set c_stride.value = c_stride.value * outShape[j] %}{% endfor %}{% set op_stride = namespace(value=1) %}{% for j in range(op_axis + 1, opRank) %}{% set op_stride.value = op_stride.value * opShape[j] %}{% endfor %}{% if c_stride.value == 1 %}let coord{{ axis }}=out_index% {{ outShape[axis] }}u;
{% else %}let coord{{ axis }}=(out_index/{{ c_stride.value }}u)% {{ outShape[axis] }}u;
{% endif %}{% if op_stride.value == 1 %}offset=offset+coord{{ axis }};
{% else %}offset=offset+coord{{ axis }}*{{ op_stride.value }}u;
{% endif %}{% endif %}{% endfor %}return offset;
{% endif %}}{% endmacro %}{% macro broadcast_offset_call(fn_name, opShape, outShape, out_index) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{{ fn_name }}({% if out_numel.value != 0 and op_numel.value != 1 %}{{ out_index }}{% endif %}){% endmacro %}{% macro broadcast_offset_fn(fn_name, opShape, opRank, outShape, outRank) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{% set op_same = namespace(value=(opRank == outRank)) %}{% if op_same.value %}{% for axis in range(outRank) %}{% if opShape[axis] != outShape[axis] %}{% set op_same.value = false %}{% endif %}{% endfor %}{% endif %}{{ offset_fn(fn_name, opShape, opRank, op_same.value, op_numel.value, outShape, outRank, out_numel.value) }}{% endmacro %}`],["ops/_shared/tree-fold-inplace.58fdcb3797.wgsl.jinja",`{% macro wgsl_tree_fold_stmt(a, op, idx, svar) %}{% if op == "max" or op == "min" %}{{ a }}[{{ idx }}] = {{ op }}({{ a }}[{{ idx }}], {{ a }}[{{ idx }} + {{ svar }}]);{% else %}{{ a }}[{{ idx }}] = {{ a }}[{{ idx }}] {{ "*" if op == "prod" else "+" }} {{ a }}[{{ idx }} + {{ svar }}];{% endif %}{% endmacro %}{% macro wgsl_tree_fold(arrays, op="add", idx="lid", wg="WORKGROUP_SIZE", svar="stride", typed=false, form="tail", barrierFirst=false, reuse=false) %}var {{ svar }}{{ ": u32 " if typed else " " }}={{ wg }}/2u;loop{if({{ svar }}==0u){break;}if({{ idx }}<{{ svar }}){
{% for a in arrays %}{{ wgsl_tree_fold_stmt(a, op, idx, svar) }}
{% endfor %}} {{ svar }}={{ svar }}/2u;workgroupBarrier();
  }{% endmacro %}`],["ops/_shared/reduce-sum.fdec0f92c8.wgsl.jinja",'{% include "ops/_shared/tree-fold-inplace.58fdcb3797.wgsl.jinja" %}{% set reduceSumName = "reduce_sum" %}{% set reduceSumBank = "partial" %}fn {{ reduceSumName }}(value:f32,tid:u32)->f32{ {{ reduceSumBank }}[tid]=value;workgroupBarrier();{{ wgsl_tree_fold([reduceSumBank], idx="tid", wg="WG", form="head") }} return {{ reduceSumBank }}[0];}'],["ops/_shared/reduce-log-of-sum.0bd3391873.wgsl.jinja","fn log_of_sum(total:f32)->f32{let bits=bitcast<u32>(total);let magnitude=bits&0x7fffffffu;let defined=select(log(total),total,magnitude==0x7f800000u);let zero_or_defined=select(defined,bitcast<f32>(bits|0xff800000u),magnitude==0u);return select(zero_or_defined,bitcast<f32>(bits|0x7fc00000u),magnitude>0x7f800000u||bits>0x80000000u);}"],["ops/_shared/reduce-minmax-identity.a93b328a6b.wgsl.jinja",`{% macro wgsl_minmax_identity(name, op, scalar="f32") %}{% set logicalBoolIdentity = false %}fn {{ name }}()->{{ scalar }} {
{% if scalar == "i32" %}return {{ "-2147483647i - 1i" if op == "max" else "2147483647i" }};
{% elif scalar == "u32" %}return {{ "1u" if logicalBoolIdentity and op == "min" else "0u" if op == "max" else "4294967295u" }};
{% else %}var bits={{ "0xff800000u" if op == "max" else "0x7f800000u" }};return bitcast<f32>(bits);
{% endif %}}{% endmacro %}`],["ops/_shared/float-neg-inf.29b2a70cbe.wgsl.jinja","fn negative_infinity()->f32{var bits=0xff800000u;return bitcast<f32>(bits);}"],["ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja","fn is_nan_f32(value:f32)->bool{let bits=bitcast<u32>(value);return(bits&0x7f800000u)==0x7f800000u&&(bits&0x007fffffu)!=0u;}"],["ops/_shared/scatter-output-offset.620f3e6760.wgsl.jinja",`fn output_offset(update_index:u32,gathered:u32)->u32{var rem=update_index;var offset=0u;
{% for axis in range(rank) %}{% set update_stride = namespace(value=1) %}{% for j in range(axis + 1, rank) %}{% set update_stride.value = update_stride.value * updatesShape[j] %}{% endfor %}{% set update_stride_divisor = update_stride.value if update_stride.value != 0 else 1 %}{% set data_stride = namespace(value=1) %}{% for j in range(axis + 1, rank) %}{% set data_stride.value = data_stride.value * dataShape[j] %}{% endfor %}{% if axis != axisSpec %}let coord{{ axis }}=rem/{{ update_stride_divisor }}u;
{% endif %}rem=rem% {{ update_stride_divisor }}u;
{% if axis == axisSpec %}offset=offset+gathered*{{ data_stride.value }}u;
{% else %}offset=offset+coord{{ axis }}*{{ data_stride.value }}u;
{% endif %}{% endfor %}return offset;}`],["ops/_shared/sigmoid-safe.0c98964ef7.wgsl.jinja","fn sigmoid_safe(x:f32)->f32{if(x>=0.0){let z=exp(-x);return 1.0/(1.0+z);}let z=exp(x);return z/(1.0+z);}"],["ops/_shared/unary-numeric-helpers.1773d907a8.wgsl.jinja",'{% include "ops/_shared/sigmoid-safe.0c98964ef7.wgsl.jinja" %}'],["ops/_shared/unary-numeric-helpers.142e982e4e.wgsl.jinja",`{% include "ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja" %}{% macro emit_inf_classifiers() %}fn is_inf_f32(value:f32)->bool{return(bitcast<u32>(value)&0x7fffffffu)==0x7f800000u;}fn nan_from_inf_f32(value:f32)->f32{return bitcast<f32>(bitcast<u32>(value)|0x00400000u);}
{% endmacro %}{% macro emit_trig_reduction_support() %}{% include "ops/_shared/precise-trig-f32.5a0bc5b040.wgsl.jinja" %}const TWO_OVER_PI:array<u32,14>=array<u32,14>(0xa2f983u,0x6e4e44u,0x1529fcu,0x2757d1u,0xf534ddu,0xc0db62u,0x95993cu,0x439041u,0xfe5163u,0xabdebbu,0xc561b7u,0x246e3au,0x424dd2u,0xe00649u);const PIO2_F:f32=1.5707963267948966;const INV_PIO2_F:f32=0.6366197723675814;const PIO2_HI_F:f32=1.570796251296997;const PIO2_LO_F:f32=7.549789415861596e-8;const REDUCE_THRESHOLD:f32={{ "65536.0" if robustTanReduction | default(false) else "1.0e4" }};struct Pio2{octant:u32,r:f32};fn reduce_pio2_fast(ax:f32)->Pio2{let nFloat=floor(fma(ax,INV_PIO2_F,0.5));var residual=fma(-nFloat,PIO2_HI_F,ax);residual=fma(-nFloat,PIO2_LO_F,residual);var out:Pio2;out.octant=u32(nFloat)&3u;out.r=residual;return out;}fn reduce_pio2(ax:f32)->Pio2{let ix=bitcast<u32>(ax);let e=(ix>>23u)&0xffu;let mant=(ix&0x7fffffu)|0x800000u;let p=i32(e)-150;var a0=0u;var a1=0u;var a2=0u;for(var k=0u;k<14u;k=k+1u){let shift=p-24*(i32(k)+1)+94;if(shift<=-48||shift>=96){continue;}let t=TWO_OVER_PI[k];let mlo=mant&0xffffu;let mhi=mant>>16u;let tlo=t&0xffffu;let thi=t>>16u;let ll=mlo*tlo;let mid=mlo*thi+mhi*tlo;let hh=mhi*thi;let plo=ll+((mid&0xffffu)<<16u);let carry=select(0u,1u,plo<ll);let phi32=hh+(mid>>16u)+carry;var v0=0u;var v1=0u;var off=0;if(shift<0){let s=u32(-shift);if(s<32u){v0=(plo>>s)|(phi32<<(32u-s));v1=phi32>>s;}else{v0=phi32>>(s-32u);v1=0u;}off=0;}else{v0=plo;v1=phi32;off=shift;}let li=off/32;let b=u32(off%32);var w0=0u;var w1=0u;var w2=0u;if(b==0u){w0=v0;w1=v1;w2=0u;}else{w0=v0<<b;w1=(v1<<b)|(v0>>(32u-b));w2=v1>>(32u-b);}var t0=0u;var t1=0u;var t2=0u;if(li==0){t0=w0;t1=w1;t2=w2;}else if(li==1){t1=w0;t2=w1;}else{t2=w0;}let n0=a0+t0;let c0=select(0u,1u,n0<a0);let n1a=a1+t1;let c1a=select(0u,1u,n1a<a1);let n1=n1a+c0;let c1b=select(0u,1u,n1<n1a);let n2=a2+t2+c1a+c1b;a0=n0;a1=n1;a2=n2;}var n=a2>>30u;var phi=f32(a2&0x3fffffffu)*(1.0/1073741824.0)+f32(a1)*(1.0/4611686018427387904.0);if(phi>=0.5){phi=phi-1.0;n=n+1u;}var out:Pio2;out.octant=n&3u;out.r=phi*PIO2_F;return out;}
{% endmacro %}{% macro emit_sin_accurate() %}fn sin_accurate(x:f32)->f32{if(is_nan_f32(x)){return x;}if(is_inf_f32(x)){return nan_from_inf_f32(x);}let ax=abs(x);var red:Pio2;if(ax<REDUCE_THRESHOLD){red=reduce_pio2_fast(ax);}else{red=reduce_pio2(ax);}let reduced=precise_sincos_half_pi(red.r);var s:f32;switch(red.octant){case 0u:{s=reduced.y;}case 1u:{s=reduced.x;}case 2u:{s=-reduced.y;}default:{s=-reduced.x;} }return bitcast<f32>(bitcast<u32>(s)^(bitcast<u32>(x)&0x80000000u));}
{% endmacro %}{% include "ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja" %}{{ emit_trig_reduction_support() }} {{ emit_inf_classifiers() }} {{ emit_sin_accurate() }}`],["ops/_shared/unary-numeric-helpers.c06e05473e.wgsl.jinja",`{% include "ops/_shared/precise-trig-polynomial.c7df3fd5a1.wgsl.jinja" %}{% macro emit_inf_classifiers() %}fn is_inf_f32(value:f32)->bool{return(bitcast<u32>(value)&0x7fffffffu)==0x7f800000u;}fn nan_from_inf_f32(value:f32)->f32{return bitcast<f32>(bitcast<u32>(value)|0x00400000u);}
{% endmacro %}{% macro emit_trig_reduction_support() %}{% include "ops/_shared/precise-trig-f32.5a0bc5b040.wgsl.jinja" %}const TWO_OVER_PI:array<u32,14>=array<u32,14>(0xa2f983u,0x6e4e44u,0x1529fcu,0x2757d1u,0xf534ddu,0xc0db62u,0x95993cu,0x439041u,0xfe5163u,0xabdebbu,0xc561b7u,0x246e3au,0x424dd2u,0xe00649u);const PIO2_F:f32=1.5707963267948966;const INV_PIO2_F:f32=0.6366197723675814;const PIO2_HI_F:f32=1.570796251296997;const PIO2_LO_F:f32=7.549789415861596e-8;const REDUCE_THRESHOLD:f32={{ "65536.0" if robustTanReduction | default(false) else "1.0e4" }};struct Pio2{octant:u32,r:f32};fn reduce_pio2_fast(ax:f32)->Pio2{let nFloat=floor(fma(ax,INV_PIO2_F,0.5));var residual=fma(-nFloat,PIO2_HI_F,ax);residual=fma(-nFloat,PIO2_LO_F,residual);var out:Pio2;out.octant=u32(nFloat)&3u;out.r=residual;return out;}fn reduce_pio2(ax:f32)->Pio2{let ix=bitcast<u32>(ax);let e=(ix>>23u)&0xffu;let mant=(ix&0x7fffffu)|0x800000u;let p=i32(e)-150;var a0=0u;var a1=0u;var a2=0u;for(var k=0u;k<14u;k=k+1u){let shift=p-24*(i32(k)+1)+94;if(shift<=-48||shift>=96){continue;}let t=TWO_OVER_PI[k];let mlo=mant&0xffffu;let mhi=mant>>16u;let tlo=t&0xffffu;let thi=t>>16u;let ll=mlo*tlo;let mid=mlo*thi+mhi*tlo;let hh=mhi*thi;let plo=ll+((mid&0xffffu)<<16u);let carry=select(0u,1u,plo<ll);let phi32=hh+(mid>>16u)+carry;var v0=0u;var v1=0u;var off=0;if(shift<0){let s=u32(-shift);if(s<32u){v0=(plo>>s)|(phi32<<(32u-s));v1=phi32>>s;}else{v0=phi32>>(s-32u);v1=0u;}off=0;}else{v0=plo;v1=phi32;off=shift;}let li=off/32;let b=u32(off%32);var w0=0u;var w1=0u;var w2=0u;if(b==0u){w0=v0;w1=v1;w2=0u;}else{w0=v0<<b;w1=(v1<<b)|(v0>>(32u-b));w2=v1>>(32u-b);}var t0=0u;var t1=0u;var t2=0u;if(li==0){t0=w0;t1=w1;t2=w2;}else if(li==1){t1=w0;t2=w1;}else{t2=w0;}let n0=a0+t0;let c0=select(0u,1u,n0<a0);let n1a=a1+t1;let c1a=select(0u,1u,n1a<a1);let n1=n1a+c0;let c1b=select(0u,1u,n1<n1a);let n2=a2+t2+c1a+c1b;a0=n0;a1=n1;a2=n2;}var n=a2>>30u;var phi=f32(a2&0x3fffffffu)*(1.0/1073741824.0)+f32(a1)*(1.0/4611686018427387904.0);if(phi>=0.5){phi=phi-1.0;n=n+1u;}var out:Pio2;out.octant=n&3u;out.r=phi*PIO2_F;return out;}
{% endmacro %}{% macro emit_sin_accurate() %}fn sin_accurate(x:f32)->f32{if(is_nan_f32(x)){return x;}if(is_inf_f32(x)){return nan_from_inf_f32(x);}let ax=abs(x);var red:Pio2;if(ax<REDUCE_THRESHOLD){red=reduce_pio2_fast(ax);}else{red=reduce_pio2(ax);}let reduced=precise_sincos_half_pi(red.r);var s:f32;switch(red.octant){case 0u:{s=reduced.y;}case 1u:{s=reduced.x;}case 2u:{s=-reduced.y;}default:{s=-reduced.x;} }return bitcast<f32>(bitcast<u32>(s)^(bitcast<u32>(x)&0x80000000u));}
{% endmacro %}{% macro emit_tan_accurate() %}fn tan_accurate(x:f32)->f32{if(is_nan_f32(x)){return x;}if(is_inf_f32(x)){return nan_from_inf_f32(x);}let ax=abs(x);var red:Pio2;if(ax<REDUCE_THRESHOLD){red=reduce_pio2_fast(ax);}else{red=reduce_pio2(ax);}let reduced=precise_sincos_half_pi(red.r);let sr=reduced.y;let cr=reduced.x;var sn:f32;var cs:f32;switch(red.octant){case 0u:{sn=sr;cs=cr;}case 1u:{sn=cr;cs=-sr;}case 2u:{sn=-sr;cs=-cr;}default:{sn=-cr;cs=sr;} }let tn=sn/cs;return bitcast<f32>(bitcast<u32>(tn)^(bitcast<u32>(x)&0x80000000u));}
{% endmacro %}{{ emit_trig_reduction_support() }}
{% include "ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja" %}{{ emit_inf_classifiers() }} {{ emit_sin_accurate() if op == "sin" else emit_tan_accurate() }}`],["ops/_shared/swiglu-value.ebb7c81802.wgsl.jinja",'{% macro sigmoid_denominator(value, one) %}{{ one }} + exp(-{{ value }}){% endmacro %}{% macro swiglu_gate(gate, alpha, one) %}{{ gate }} / ({{ sigmoid_denominator(gate ~ " * " ~ alpha, one) }}){% endmacro %}'],["ops/_shared/tree-fold-inplace.629ddaa87a.wgsl.jinja",`{% macro wgsl_tree_fold_stmt(a, op, idx, svar) %}{% if op == "max" or op == "min" %}{{ a }}[{{ idx }}] = {{ op }}({{ a }}[{{ idx }}], {{ a }}[{{ idx }} + {{ svar }}]);{% else %}{{ a }}[{{ idx }}] = {{ a }}[{{ idx }}] {{ "*" if op == "prod" else "+" }} {{ a }}[{{ idx }} + {{ svar }}];{% endif %}{% endmacro %}{% macro wgsl_tree_fold(arrays, op="add", idx="lid", wg="WORKGROUP_SIZE", svar="stride", typed=false, form="tail", barrierFirst=false, reuse=false) %}var {{ svar }}{{ ": u32 " if typed else " " }}={{ wg }}/2u;loop{if({{ idx }}<{{ svar }}){
{% for a in arrays %}{{ wgsl_tree_fold_stmt(a, op, idx, svar) }}
{% endfor %}}workgroupBarrier();if({{ svar }}==1u){break;} {{ svar }}={{ svar }}/2u;
  }{% endmacro %}`],["ops/_shared/bf16.721558b02a.wgsl.jinja","fn widen_bf16(bits:u32)->f32{return bitcast<f32>((bits&0xffffu)<<16u);}"],["ops/_shared/bf16.3b861ea8df.wgsl.jinja","fn load_bf16_word(word:u32,column:u32)->f32{let bits=select(word,word>>16u,(column&1u)==1u);return widen_bf16(bits);}"],["ops/_shared/compensated-dot.bd8b60df37.wgsl.jinja",'{% set dotType = "f32" %}struct DotAccumulator{hi:{{ dotType }},lo:{{ dotType }},}fn dot_accumulate(acc:DotAccumulator,a:{{ dotType }},b:{{ dotType }})->DotAccumulator{let product=fma(a,b,{{ dotType }}(0.0));let productError=fma(a,b,-product);let sum=fma(acc.hi,{{ dotType }}(1.0),product);let bv=fma({{ dotType }}(-1.0),acc.hi,sum);let av=fma({{ dotType }}(-1.0),bv,sum);let ae=fma({{ dotType }}(-1.0),av,acc.hi);let be=fma({{ dotType }}(-1.0),bv,product);let error=fma(fma(fma(ae,{{ dotType }}(1.0),be),{{ dotType }}(1.0),acc.lo),{{ dotType }}(1.0),productError);let hi=fma(sum,{{ dotType }}(1.0),error);return DotAccumulator(hi,fma({{ dotType }}(-1.0),fma({{ dotType }}(-1.0),sum,hi),error));}fn dot_value(acc:DotAccumulator)->{{ dotType }} {return fma(acc.hi,{{ dotType }}(1.0),acc.lo);}'],["ops/_shared/reduce-tree.2706db3b42.wgsl.jinja",`{% macro wgsl_tree_reduce_f32(name, mode, buffer="partial", wg="WG", trailingBarrier=true, valueType="f32") %}fn {{ name }}(value:{{ valueType }},tid:u32)->{{ valueType }} { {{ buffer }}[tid]=value;workgroupBarrier();var n:u32={{ wg }};loop{let half=(n+1u)/2u;if(tid<n-half){ {{ buffer }}[tid]=dot_accumulate(dot_accumulate({{ buffer }}[tid],{{ buffer }}[tid+half].hi,1.0),{{ buffer }}[tid+half].lo,1.0);}workgroupBarrier();n=half;if(n==1u){break;} }let reduced={{ buffer }}[0];workgroupBarrier();return reduced;}
{% endmacro %}`],["ops/_shared/softmax-stable-helpers.49d3e37058.wgsl.jinja","const FLT_MAX:f32=3.4028234663852886e38;fn is_finite_f32(value:f32)->bool{return select(false,value<=FLT_MAX,value>=-FLT_MAX);}fn shifted_value(value:f32,maxValue:f32)->f32{let equalFiniteMax=select(false,value==maxValue,is_finite_f32(maxValue));return select(value-maxValue,0.0,equalFiniteMax);}fn exp_shift(value:f32,maxValue:f32)->f32{return exp(shifted_value(value,maxValue));}"],["ops/_shared/softmax-md-combine.e13c0ec67f.wgsl.jinja",`{% set combineSubgroups = combineSubgroups is defined and combineSubgroups %}{% if combineSubgroups %}var<workgroup>partialM:array<f32,WG>;var<workgroup>partialD:array<f32,WG>;var<workgroup>leaderMask:array<atomic<u32>,(WG+31u)/32u>;var<workgroup>combinedMD:vec2<f32>;fn combine_partials(m:f32,d:f32,lidx:u32)->vec2<f32>{let sgM=subgroupMax(m);let sgD=subgroupAdd(d*exp_shift(m,sgM));if(subgroupElect()){partialM[lidx]=sgM;partialD[lidx]=sgD;atomicOr(&leaderMask[lidx/32u],1u<<(lidx%32u));}workgroupBarrier();if(lidx==0u){var accM=-FLT_MAX;var accD=0.0;for(var w=0u;w<(WG+31u)/32u;w=w+1u){var bits=atomicExchange(&leaderMask[w],0u);while(bits!=0u){let slot=w*32u+firstTrailingBit(bits);bits=bits&(bits-1u);let mNew=max(accM,partialM[slot]);accD=accD*exp_shift(accM,mNew)+partialD[slot]*exp_shift(partialM[slot],mNew);accM=mNew;} }combinedMD=vec2<f32>(accM,accD);}workgroupBarrier();return combinedMD;}
{% else %}{% set mdExtent = "WG" %}var<workgroup>partialM:array<f32,{{ mdExtent }}>;var<workgroup>partialD:array<f32,{{ mdExtent }}>;fn combine_partials(m:f32,d:f32,lidx:u32)->vec2<f32>{partialM[lidx]=m;partialD[lidx]=d;workgroupBarrier();var stride=WG/2u;loop{if(stride==0u){break;}if(lidx<stride){let m1=partialM[lidx];let d1=partialD[lidx];let m2=partialM[lidx+stride];let d2=partialD[lidx+stride];let mNew=max(m1,m2);partialD[lidx]=d1*exp_shift(m1,mNew)+d2*exp_shift(m2,mNew);partialM[lidx]=mNew;}workgroupBarrier();stride=stride/2u;}let merged=vec2<f32>(partialM[0],partialD[0]);workgroupBarrier();return merged;}
{% endif %}`],["ops/_shared/attn-scale-value.ae17ac7a82.wgsl.jinja","fn scale_value()->f32{return {{ ATTN_SCALE_OVERRIDE }};}"],["ops/_shared/attn-online-merge-tile.d088626ed8.wgsl.jinja","let newMax=max(runningMax,tile.x);let correction=exp_shift(runningMax,newMax);runningDenom=runningDenom*correction+tile.y*exp_shift(tile.x,newMax);runningMax=newMax;var prob=0.0;if(keyAllowed){prob=exp_shift(score,newMax);}probs[tid]=prob;workgroupBarrier();"],["ops/_shared/kv-cache-index.26c94f7350.wgsl.jinja",`{% if ringRows is defined and ringRows %}fn cache_index(pos:u32,head:u32,dim:u32)->u32{return(((LAYER*CACHE_LEN+(pos&{{ ringRows - 1 }}u))*NUM_KV_HEADS+head)*HEAD_DIM+dim);}
{% else %}fn cache_index(pos:u32,head:u32,dim:u32)->u32{return(((LAYER*CACHE_LEN+pos)*NUM_KV_HEADS+head)*HEAD_DIM+dim);}
{% endif %}`],["ops/_shared/attn-scale-value.88c0e1c03f.wgsl.jinja",`{% set ATTN_SCALE_DIM = "HEAD_DIM" %}fn scale_value() -> f32 {
{% if ATTN_SCALE_OVERRIDE is defined %}return {{ ATTN_SCALE_OVERRIDE }};
{% else %}if(params.scale!=0.0){return params.scale;}return inverseSqrt(f32({{ ATTN_SCALE_DIM }}));
{% endif %}}`],["ops/_shared/subgroup-matrix-software-8x8.0a6b6c163d.wgsl.jinja",`var<private>matrixSoftwareLane:u32;fn matrixSoftwareMultiplyAccumulate(lhs:array<f32,8>,rhs:array<vec2<f32>,8>,acc:vec2<f32>)->vec2<f32>{var value=acc;for(var k=0u;k<8u;k+=1u){value=fma(vec2<f32>(lhs[k]),rhs[k],value);}return value;}
{% macro matrixType(software, kind, scalar) %}{% if software %}{% if kind == "left" %}array<f32, 8>{% elif kind == "right" %}array<vec2<f32>, 8>{% else %}vec2<{{ scalar }}>{% endif %}{% else %}subgroup_matrix_{{ kind }}<{{ scalar }}, 8, 8>{% endif %}{% endmacro %}{% macro matrixLoad(software, kind, scalar, major, pointer, offset, stride) %}{% if not software %}subgroupMatrixLoad<{{ matrixType(false, kind, scalar) }}, {{ major }}>({{ pointer }}, {{ offset }}, {{ stride }}){% else %}{% if kind == "left" %}array<f32, 8>(
{% for k in range(8) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}(matrixSoftwareLane / 4u) * ({{ stride }}) + {{ k }}u{% else %}{{ k }}u * ({{ stride }}) + matrixSoftwareLane / 4u{% endif %}]){{ "," if k < 7 else "" }}
{% endfor %})
{% elif kind == "right" %}array<vec2<f32>, 8>(
{% for k in range(8) %}vec2<f32>(
{% for h in range(2) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}{{ k }}u * ({{ stride }}) + (matrixSoftwareLane % 4u) * 2u + {{ h }}u{% else %}((matrixSoftwareLane % 4u) * 2u + {{ h }}u) * ({{ stride }}) + {{ k }}u{% endif %}]){{ "," if h == 0 else "" }}
{% endfor %}){{ "," if k < 7 else "" }}
{% endfor %})
{% else %}vec2<{{ scalar }}>(
{% for h in range(2) %}(*({{ pointer }}))[({{ offset }}) + {{ {"row_major": "(matrixSoftwareLane / 4u) * (" ~ stride ~ ") + (matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u", "col_major": "((matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u) * (" ~ stride ~ ") + matrixSoftwareLane / 4u"}[major] }}]{{ "," if h == 0 else "" }}
{% endfor %})
{% endif %}{% endif %}{% endmacro %}{% macro matrixMma(software, scalar, lhs, rhs, acc) %}{% if not software %}subgroupMatrixMultiplyAccumulate({{ lhs }}, {{ rhs }}, {{ acc }}){% else %}vec2<{{ scalar }}>(matrixSoftwareMultiplyAccumulate({{ lhs }}, {{ rhs }}, vec2<f32>({{ acc }}))){% endif %}{% endmacro %}{% macro matrixStore(software, major, pointer, offset, value, stride) %}{% if not software %}subgroupMatrixStore<{{ major }}>({{ pointer }}, {{ offset }}, {{ value }}, {{ stride }});{% else %}{
let matrixPointer={{ pointer }};let matrixValue={{ value }};
{% for h in range(2) %}  (*matrixPointer)[({{ offset }}) + {{ {"row_major": "(matrixSoftwareLane / 4u) * (" ~ stride ~ ") + (matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u", "col_major": "((matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u) * (" ~ stride ~ ") + matrixSoftwareLane / 4u"}[major] }}] = matrixValue.{{ "x" if h == 0 else "y" }};
{% endfor %}}{% endif %}{% endmacro %}`],["ops/_shared/packed-4x8-compat.cf3c88fe57.wgsl.jinja",`{% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}@must_use fn unpack4xU8(e:u32)->vec4<u32>{return(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);}
{% endif %}`],["ops/_shared/subgroup-runtime-fold.f33ebc1f56.wgsl.jinja",`{% macro sg_fold_workgroup(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,SG_SAFE_WG,sgNative);
{% endfor %}{% endmacro %}{% macro sg_fold_cluster32(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,32u,sgNative);
{% endfor %}{% endmacro %}`],["ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja",'const SG_SAFE_WG:u32={{ sgSafeStorageSize }}u;var<workgroup>sgSafeRejected:atomic<u32>;var<workgroup>sgSafeAccepted:u32;var<workgroup>sgSafeScratch:array<f32,{{ sgSafeStorageSize }}>;fn sg_native_layout(tid:u32,lane:u32,width:u32)->bool{let sgActiveMask=subgroupBallot(true);let population=countOneBits(sgActiveMask.x)+countOneBits(sgActiveMask.y)+countOneBits(sgActiveMask.z)+countOneBits(sgActiveMask.w);let first=subgroupBroadcastFirst(tid);if(width!=32u||population!=32u||lane!=(tid&31u)||first!=(tid&~31u)){atomicOr(&sgSafeRejected,1u);}workgroupBarrier();if(tid==0u){sgSafeAccepted=select(0u,1u,atomicLoad(&sgSafeRejected)==0u);}return workgroupUniformLoad(&sgSafeAccepted)!=0u;}fn sg_safe_sum(value:f32,tid:u32,span:u32,native:bool)->f32{if(native){let total=subgroupAdd(value);if(span==32u){return total;}if(subgroupElect()){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid>>5u]{{ sgSafeScratchComponent | default("") }}=total;}workgroupBarrier();var sum=0.0;for(var i=0u;i<(SG_SAFE_WG+31u)/32u;i=i+1u){sum=sum+{{ sgSafeScratchName | default("sgSafeScratch") }}[i]{{ sgSafeScratchComponent | default("") }};}workgroupBarrier();return sum;} {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}=value;workgroupBarrier();let first=(tid/span)*span;let lane=tid-first;var live=span;loop{if(live<=1u){break;}let half=(live+1u)>>1u;if(lane<live-half){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}={{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}+{{ sgSafeScratchName | default("sgSafeScratch") }}[tid+half]{{ sgSafeScratchComponent | default("") }};}workgroupBarrier();live=half;}let total={{ sgSafeScratchName | default("sgSafeScratch") }}[first]{{ sgSafeScratchComponent | default("") }};workgroupBarrier();return total;}'],["ops/_shared/q4-dequant-helpers.245c302430.wgsl.jinja","fn q4_lo(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}fn q4_hi(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}fn q4wp(word:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q4_lo(word))+dot(hi,q4_hi(word));}"],["ops/_shared/q4-block-dot-flags.5f810c6c81.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sMeta = packMeta %}{% set sLutSfx = "" %}'],["ops/_shared/q4-block-dot.e2026cefd6.wgsl.jinja","let {{ sPre }}blk={{ sBlk }};let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;"],["ops/_shared/q4-decode-row-block-accumulate.97715fdec3.wgsl.jinja",`{% set packMeta = "q4_scales" %}{% set accPrefix = "acc" %}{% set rowBase = "base0" %}{% include "ops/_shared/q4-block-dot-flags.5f810c6c81.wgsl.jinja" %}    let sa = dot(x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7, vec4<f32>(1.0));
{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.e2026cefd6.wgsl.jinja" %}{% endfor %}`],["ops/_shared/q4-normed-projection-multirow.11b6390ad1.wgsl.jinja",`{% set packMeta = "q4_scales" %}{% for r in range(regRows) %}var acc{{ r }}=0.0;
{% endfor %}for(var b=lane;b<BPR;b=b+32u){let hbase=b*8u;
{% for i in range(8) %}let x{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% include "ops/_shared/q4-decode-row-block-accumulate.97715fdec3.wgsl.jinja" %}  }`],["ops/_shared/projection-multirow-reduce.367c218e5c.wgsl.jinja",`{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if useSubgroups %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=sg_safe_sum({{ accPrefix }}{{ r }},lid,32u,sgNative);
{% endfor %}{% elif sgRuntime | default(false) %}{% set sgFold = namespace(pairs=[]) %}{% for r in range(regRows) %}{% set sgFold.pairs = sgFold.pairs + [[accPrefix ~ r, totalPrefix ~ r]] %}{% endfor %}{{ sg_fold_cluster32(sgFold.pairs) }}
{% elif partialsRowVector | default(false) %}  partials4[lid] = vec{{ regRows }}<f32>({% for r in range(regRows) %}{{ ", " if r else "" }}{{ accPrefix }}{{ r }}{% endfor %});
workgroupBarrier();var {{ totalPrefix }}Stride=16u;loop{if({{ totalPrefix }}Stride==0u){break;}if(lane<{{ totalPrefix }}Stride){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride];}workgroupBarrier();{{ totalPrefix }}Stride={{ totalPrefix }}Stride/2u;}let {{ totalPrefix }}Rows=partials4[cluster*32u];
{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}={{ totalPrefix }}Rows.{{ "xyzw"[r] }};
{% endfor %}workgroupBarrier();
{% else %}{% for r in range(regRows) %}partials4[lid]={{ accPrefix }}{{ r }};workgroupBarrier();var {{ totalPrefix }}Stride{{ r }}=16u;loop{if({{ totalPrefix }}Stride{{ r }}==0u){break;}if(lane<{{ totalPrefix }}Stride{{ r }}){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride{{ r }}];}workgroupBarrier();{{ totalPrefix }}Stride{{ r }}={{ totalPrefix }}Stride{{ r }}/2u;}let {{ totalPrefix }}{{ r }}=partials4[cluster*32u];workgroupBarrier();
{% endfor %}{% endif %}`],["ops/_shared/q4-decode-row-block-accumulate.0cf5917f99.wgsl.jinja",`{% set packMeta = packMeta | default("q4_metadata") %}{% set accPrefix = accPrefix | default("acc") %}{% set rowBase = "row_block_base" %}{% include "ops/_shared/q4-block-dot-flags.5f810c6c81.wgsl.jinja" %}    let sa = dot(x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7, vec4<f32>(1.0));
{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.e2026cefd6.wgsl.jinja" %}{% endfor %}`],["ops/_shared/q4-normed-projection-single-row.ce563d4459.wgsl.jinja",`{% set packMeta = packMeta | default("q4_scales") %}var acc0=0.0;for(var b=lid;b<BPR;b=b+WORKGROUP_SIZE){let hbase=b*8u;
{% for i in range(8) %}let {{ "x" if preNormed else "h" }}{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% set regRows = 1 %}{% set rowBase = "row_block_base" %}{% include "ops/_shared/q4-decode-row-block-accumulate.0cf5917f99.wgsl.jinja" %}  }
{% if useSubgroups %}let acc_total=sg_safe_sum(acc0,lid,SG_SAFE_WG,sgNative);
{% elif sgRuntime | default(false) %}{{ sg_fold_workgroup([["acc0", "acc_total"]] if preNormed else [["acc0", "acc_total"], ["ss", "ss_total"]]) }}
{% else %}partials[lid]=acc0;workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){partials[lid]=partials[lid]+partials[lid+stride];}workgroupBarrier();stride=stride/2u;}let acc_total=partials[0];
{% endif %}`],["ops/_shared/subgroup-runtime-fold.58b998281f.wgsl.jinja",`{% macro sg_fold_workgroup(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,SG_SAFE_WG,sgNative);
{% endfor %}{% endmacro %}`],["ops/_shared/subgroup-active-lanes.97fae981c2.wgsl.jinja",`{% macro active_lanes(lane, prefix="active") %}let {{ prefix }}Mask=subgroupBallot(true);let {{ prefix }}Word={{ lane }}/32u;let {{ prefix }}Lower=(1u<<({{ lane }} %32u))-1u;let {{ prefix }}Count=max(countOneBits({{ prefix }}Mask.x)+countOneBits({{ prefix }}Mask.y)+countOneBits({{ prefix }}Mask.z)+countOneBits({{ prefix }}Mask.w),1u);var {{ prefix }}Rank=countOneBits({{ prefix }}Mask[{{ prefix }}Word]&{{ prefix }}Lower);for(var word=0u;word<{{ prefix }}Word;word++){ {{ prefix }}Rank+=countOneBits({{ prefix }}Mask[word]);}
{% endmacro %}`],["ops/_shared/silu-fn.d10dc95662.wgsl.jinja","fn silu(x:f32)->f32{return x/(1.0+exp(-x));}"],["ops/_shared/q4-block-dot-flags.af1c2cef54.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sScales = "q4_scales" %}{% set sMeta = packMeta %}{% set sHasMin = false %}{% set sQ5 = 0 %}{% set sQ6k = false %}{% set sQ5k = false %}{% set sQ3k = 0 %}{% set sQ2k = 0 %}{% set sQ4knc = true %}{% set sLut16 = false %}{% set sLutc = 0 %}{% set sLutc16 = 0 %}{% set sLut2 = 0 %}{% set sQ8Pair = false %}{% set sIq4xs = false %}{% set sIq1m = false %}{% set sIq1s = false %}{% set sIq1md = false %}{% set sIq2xxs = false %}{% set sIq2xs = false %}{% set sIq2s = false %}{% set sIq3xxs = false %}{% set sIq3s = false %}{% set sLutId = 0 %}{% set sLutSfx = "" %}{% set sIqSfx = "" %}{% set sPairView = false %}'],["ops/_shared/q4-block-dot-shared-flags.c1c2072bbb.wgsl.jinja",'{% set sBits = "shared_bits" %}{% set sScales = "shared_scales" %}{% set sMeta = "shared_scales" %}{% set sHasMin = false %}{% set sQ5 = false %}{% set sQ6k = false %}{% set sQ5k = false %}{% set sQ3k = false %}{% set sQ2k = false %}{% set sQ4knc = false %}{% set sLut16 = false %}{% set sLutc = false %}{% set sLutc16 = false %}{% set sLut2 = false %}{% set sQ8Pair = false %}{% set sIq4xs = false %}{% set sIq1m = false %}{% set sIq1s = false %}{% set sIq1md = false %}{% set sIq2xxs = false %}{% set sIq2xs = false %}{% set sIq2s = false %}{% set sIq3xxs = false %}{% set sIq3s = false %}{% set sLutId = 0 %}{% set sLutSfx = "" %}{% set sIqSfx = "" %}{% set sPairView = false %}'],["ops/_shared/lut2-block-scale.f72dd0c68a.wgsl.jinja",`{% macro lut2_native_scale(block, metadata, codebook) %}{% if codebook == 9 %}unpack2x16float({{ metadata }}[({{ block }}) >> 3u])[(({{ block }}) >> 2u) & 1u]{% else %}unpack2x16float({{ metadata }}[(({{ block }}) >> 3u) * 3u + ((({{ block }}) & 7u) >> 2u)])[(({{ block }}) >> 1u) & 1u]{% endif %}{% endmacro %}{% macro lut2_block_scale(block, prefix, name, lo, hi, delta, per16, metadata, suffix, codebook) %}{% set scale = "lutScale" ~ suffix %}{% if codebook in [8, 9] %}let {{ name }}={{ lut2_native_scale(block, metadata, codebook) }};let {{ delta }}=vec4<u32>(0u);
{% elif per16 %}let {{ prefix }}mb=(({{ block }})>>3u)*5u;let {{ prefix }}dl=bitcast<f32>({{ metadata }}[{{ prefix }}mb+4u]);let {{ prefix }}cw={{ metadata }}[{{ prefix }}mb+((({{ block }})&7u)>>1u)];let {{ prefix }}cs=(({{ block }})&1u)*16u;let {{ prefix }}bLo=({{ prefix }}cw>>{{ prefix }}cs)&255u;let {{ prefix }}bHi=({{ prefix }}cw>>({{ prefix }}cs+8u))&255u;let {{ lo }}={{ prefix }}dl*{{ scale }}({{ prefix }}bLo&15u);let {{ hi }}={{ prefix }}dl*{{ scale }}({{ prefix }}bHi&15u);let {{ delta }}=((vec4<u32>({{ prefix }}bLo,{{ prefix }}bLo,{{ prefix }}bHi,{{ prefix }}bHi)>>vec4<u32>(4u,5u,4u,5u))&vec4<u32>(1u))*8u;
{% else %}let {{ prefix }}mb=(({{ block }})>>3u)*3u;let {{ prefix }}b=({{ metadata }}[{{ prefix }}mb+((({{ block }})&7u)>>2u)]>>((({{ block }})&3u)*8u))&255u;let {{ name }}=bitcast<f32>({{ metadata }}[{{ prefix }}mb+2u])*{{ scale }}({{ prefix }}b&15u);let {{ delta }}=vec4<u32>((({{ prefix }}b>>4u)&1u)*8u);
{% endif %}{% endmacro %}`],["ops/_shared/iq4xs-block-scale.e8819371b2.wgsl.jinja",`{% macro iq4xs_block_scale(block, prefix, name, metadata) %}let {{ prefix }}mb=(({{ block }})>>3u)*2u;let {{ prefix }}k=({{ block }})&7u;let {{ prefix }}m0={{ metadata }}[{{ prefix }}mb];let {{ prefix }}ls=(({{ metadata }}[{{ prefix }}mb+1u]>>({{ prefix }}k*4u))&15u)|((({{ prefix }}m0>>(16u+{{ prefix }}k*2u))&3u)<<4u);let {{ name }}=unpack2x16float({{ prefix }}m0).x*(f32({{ prefix }}ls)-32.0);
{%- endmacro %}`],["ops/_shared/q4-block-dot.e9fc34b355.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.f72dd0c68a.wgsl.jinja" %}{% include "ops/_shared/iq4xs-block-scale.e8819371b2.wgsl.jinja" %}{% set rawPair = sBits ~ "[" ~ sPre ~ "blk]" if sPairView else "vec2<u32>(" ~ sBits ~ "[" ~ sPre ~ "w2], " ~ sBits ~ "[" ~ sPre ~ "w2 + 1u])" %}let {{ sPre }}blk={{ sBlk }};
{% if sIq1m %}{% if sIq1s %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*12u;let {{ sPre }}mt=({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);{{ sAcc }}={{ sAcc }}+iq1sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt,{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq1md %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*14u;let {{ sPre }}mt=(({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu)|((({{ sBits }}[{{ sPre }}sbw+dqCo]>>dqCs)&0x3Fu)<<16u);let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}(vec2<u32>({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt),{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% else %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq2xxs %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq2dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq2xs %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*18u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq2xsdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq2s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*20u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq2sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+18u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq3xxs %}let {{ sPre }}b3={{ sPre }}blk*3u;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq3dot{{ sIqSfx }}({{ sBits }}[{{ sPre }}b3],{{ sBits }}[{{ sPre }}b3+1u],{{ sBits }}[{{ sPre }}b3+2u],{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq3s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*27u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq3sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+1u],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+2u],({{ sBits }}[{{ sPre }}sbw+24u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+26u]>>({{ sPre }}k*4u))&0xFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sQ2k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*5u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}sw={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}j>>1u];let {{ sPre }}bLo=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u))&0xFFu;let {{ sPre }}bHi=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u+8u))&0xFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb*2u]);let {{ sPre }}dm=f32({{ sScales }}[{{ sPre }}sb*2u+1u]);{{ sAcc }}={{ sAcc }}+({{ sPre }}d*f32({{ sPre }}bLo&15u))*(dot(x0,q2w({{ sPre }}cLo,0u))+dot(x1,q2w({{ sPre }}cLo,1u))+dot(x2,q2w({{ sPre }}cLo,2u))+dot(x3,q2w({{ sPre }}cLo,3u)))-({{ sPre }}dm*f32({{ sPre }}bLo>>4u))*xsLo+({{ sPre }}d*f32({{ sPre }}bHi&15u))*(dot(x4,q2w({{ sPre }}cHi,0u))+dot(x5,q2w({{ sPre }}cHi,1u))+dot(x6,q2w({{ sPre }}cHi,2u))+dot(x7,q2w({{ sPre }}cHi,3u)))-({{ sPre }}dm*f32({{ sPre }}bHi>>4u))*xsHi;
{% elif sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ5 %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}g=({{ sPre }}blk>>2u)*5u;let {{ sPre }}ln={{ sPre }}blk&3u;let {{ sPre }}w={{ sBits }}[{{ sPre }}g+{{ sPre }}ln];let {{ sPre }}qh={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}ln];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q5wp({{ sPre }}w.x,{{ sPre }}qh,0u,x0,x4)+q5wp({{ sPre }}w.y,{{ sPre }}qh,1u,x1,x5)+q5wp({{ sPre }}w.z,{{ sPre }}qh,2u,x2,x6)+q5wp({{ sPre }}w.w,{{ sPre }}qh,3u,x3,x7));
{% elif sQ3k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*7u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}hw={{ sBits }}[{{ sPre }}g+4u+({{ sPre }}j>>2u)][{{ sPre }}j&3u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+6u][{{ sPre }}j>>1u]);let {{ sPre }}d3=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,0u))+dot(x1,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,1u))+dot(x2,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,2u))+dot(x3,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,3u)))+{{ sPre }}scHi*(dot(x4,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,0u))+dot(x5,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,1u))+dot(x6,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,2u))+dot(x7,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,3u)));
{% elif sLut2 %}{% filter indent(4, true) %}{{ lut2_block_scale(sPre ~ "blk", sPre ~ "2_", sPre ~ "s2", sPre ~ "s2lo", sPre ~ "s2hi", sPre ~ "db2", sLutc16, sMeta, sLutSfx, sLutId) }}{% endfilter %}let {{ sPre }}wa={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}wb={{ sBits }}[{{ sPre }}blk*2u+1u];
{% if sLutc16 %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2lo*(dot(x0,q2q{{ sLutSfx }}({{ sPre }}wa,0u,{{ sPre }}db2[0]))+dot(x1,q2q{{ sLutSfx }}({{ sPre }}wa,8u,{{ sPre }}db2[0]))+dot(x2,q2q{{ sLutSfx }}({{ sPre }}wa,16u,{{ sPre }}db2[1]))+dot(x3,q2q{{ sLutSfx }}({{ sPre }}wa,24u,{{ sPre }}db2[1])))+{{ sPre }}s2hi*(dot(x4,q2q{{ sLutSfx }}({{ sPre }}wb,0u,{{ sPre }}db2[2]))+dot(x5,q2q{{ sLutSfx }}({{ sPre }}wb,8u,{{ sPre }}db2[2]))+dot(x6,q2q{{ sLutSfx }}({{ sPre }}wb,16u,{{ sPre }}db2[3]))+dot(x7,q2q{{ sLutSfx }}({{ sPre }}wb,24u,{{ sPre }}db2[3])));
{% else %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2*(q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,0u,x0,x4,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,8u,x1,x5,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,16u,x2,x6,{{ sPre }}db2[1],{{ sPre }}db2[3])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,24u,x3,x7,{{ sPre }}db2[1],{{ sPre }}db2[3]));
{% endif %}{% elif sLut16 or (sLutc and sLutc16) %}{% if sLutc and sLutc16 %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*5u;let {{ sPre }}dl=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+4u]);let {{ sPre }}cw={{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>1u)];let {{ sPre }}cs=({{ sPre }}blk&1u)*16u;let {{ sPre }}s_lo={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>{{ sPre }}cs)&255u);let {{ sPre }}s_hi={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>({{ sPre }}cs+8u))&255u);
{% else %}let {{ sPre }}s_lo=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}s_hi=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s_lo*(dot(x0,q4_lo{{ sLutSfx }}({{ sPre }}w.x))+dot(x1,q4_lo{{ sLutSfx }}({{ sPre }}w.y))+dot(x2,q4_lo{{ sLutSfx }}({{ sPre }}w.z))+dot(x3,q4_lo{{ sLutSfx }}({{ sPre }}w.w)))+{{ sPre }}s_hi*(dot(x4,q4_hi{{ sLutSfx }}({{ sPre }}w.x))+dot(x5,q4_hi{{ sLutSfx }}({{ sPre }}w.y))+dot(x6,q4_hi{{ sLutSfx }}({{ sPre }}w.z))+dot(x7,q4_hi{{ sLutSfx }}({{ sPre }}w.w)));
{% elif sQ8Pair %}let {{ sPre }}s=q8pair_scale({{ sPre }}blk);let {{ sPre }}cb=q8pair_code_base({{ sPre }}blk);let {{ sPre }}w0={{ sBits }}[{{ sPre }}cb];let {{ sPre }}w1={{ sBits }}[{{ sPre }}cb+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3)+q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% else %}{% if sLutc %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*3u;let {{ sPre }}s=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+2u])*lutScale{{ sLutSfx }}(({{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>2u)]>>(({{ sPre }}blk&3u)*8u))&255u);
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-block-dot.0926c1462b.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.f72dd0c68a.wgsl.jinja" %}{% include "ops/_shared/iq4xs-block-scale.e8819371b2.wgsl.jinja" %}{% set rawPair = sBits ~ "[" ~ sPre ~ "blk]" if sPairView else "vec2<u32>(" ~ sBits ~ "[" ~ sPre ~ "w2], " ~ sBits ~ "[" ~ sPre ~ "w2 + 1u])" %}let {{ sPre }}blk={{ sBlk }};
{% if sIq1m %}{% if sIq1s %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*12u;let {{ sPre }}mt=({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);{{ sAcc }}={{ sAcc }}+iq1sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt,{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq1md %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*14u;let {{ sPre }}mt=(({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu)|((({{ sBits }}[{{ sPre }}sbw+dqCo]>>dqCs)&0x3Fu)<<16u);let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}(vec2<u32>({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt),{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% else %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq2xxs %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq2dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq2xs %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*18u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq2xsdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq2s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*20u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq2sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+18u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq3xxs %}let {{ sPre }}b3={{ sPre }}blk*3u;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);{{ sAcc }}={{ sAcc }}+iq3dot{{ sIqSfx }}({{ sBits }}[{{ sPre }}b3],{{ sBits }}[{{ sPre }}b3+1u],{{ sBits }}[{{ sPre }}b3+2u],{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sIq3s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*27u;let {{ sPre }}k={{ sPre }}blk&7u;{{ sAcc }}={{ sAcc }}+iq3sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+1u],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+2u],({{ sBits }}[{{ sPre }}sbw+24u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+26u]>>({{ sPre }}k*4u))&0xFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% elif sQ2k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*5u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}sw={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}j>>1u];let {{ sPre }}bLo=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u))&0xFFu;let {{ sPre }}bHi=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u+8u))&0xFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb*2u]);let {{ sPre }}dm=f32({{ sScales }}[{{ sPre }}sb*2u+1u]);{{ sAcc }}={{ sAcc }}+({{ sPre }}d*f32({{ sPre }}bLo&15u))*(dot(x0,q2w({{ sPre }}cLo,0u))+dot(x1,q2w({{ sPre }}cLo,1u))+dot(x2,q2w({{ sPre }}cLo,2u))+dot(x3,q2w({{ sPre }}cLo,3u)))-({{ sPre }}dm*f32({{ sPre }}bLo>>4u))*xsLo+({{ sPre }}d*f32({{ sPre }}bHi&15u))*(dot(x4,q2w({{ sPre }}cHi,0u))+dot(x5,q2w({{ sPre }}cHi,1u))+dot(x6,q2w({{ sPre }}cHi,2u))+dot(x7,q2w({{ sPre }}cHi,3u)))-({{ sPre }}dm*f32({{ sPre }}bHi>>4u))*xsHi;
{% elif sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sHasMin %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}bias=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ5 %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}g=({{ sPre }}blk>>2u)*5u;let {{ sPre }}ln={{ sPre }}blk&3u;let {{ sPre }}w={{ sBits }}[{{ sPre }}g+{{ sPre }}ln];let {{ sPre }}qh={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}ln];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q5wp({{ sPre }}w.x,{{ sPre }}qh,0u,x0,x4)+q5wp({{ sPre }}w.y,{{ sPre }}qh,1u,x1,x5)+q5wp({{ sPre }}w.z,{{ sPre }}qh,2u,x2,x6)+q5wp({{ sPre }}w.w,{{ sPre }}qh,3u,x3,x7));
{% elif sQ6k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));
{% elif sQ3k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*7u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}hw={{ sBits }}[{{ sPre }}g+4u+({{ sPre }}j>>2u)][{{ sPre }}j&3u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+6u][{{ sPre }}j>>1u]);let {{ sPre }}d3=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,0u))+dot(x1,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,1u))+dot(x2,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,2u))+dot(x3,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,3u)))+{{ sPre }}scHi*(dot(x4,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,0u))+dot(x5,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,1u))+dot(x6,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,2u))+dot(x7,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,3u)));
{% elif sQ5k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*11u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}qh={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>2u)][{{ sPre }}j&3u];let {{ sPre }}smv={{ sBits }}[{{ sPre }}g+10u];let {{ sPre }}sc=f32(({{ sPre }}smv[{{ sPre }}j>>2u]>>(({{ sPre }}j&3u)*8u))&0xFFu);let {{ sPre }}mn=f32(({{ sPre }}smv[2u+({{ sPre }}j>>2u)]>>(({{ sPre }}j&3u)*8u))&0xFFu);let {{ sPre }}scEff=f32({{ sScales }}[{{ sPre }}sb*2u])*{{ sPre }}sc;let {{ sPre }}mEff=f32({{ sScales }}[{{ sPre }}sb*2u+1u])*{{ sPre }}mn;{{ sAcc }}={{ sAcc }}-{{ sPre }}mEff*sa+{{ sPre }}scEff*(dot(x0,q5k_lo({{ sPre }}words.x,{{ sPre }}qh&15u))+dot(x1,q5k_lo({{ sPre }}words.y,({{ sPre }}qh>>4u)&15u))+dot(x2,q5k_lo({{ sPre }}words.z,({{ sPre }}qh>>8u)&15u))+dot(x3,q5k_lo({{ sPre }}words.w,({{ sPre }}qh>>12u)&15u))+dot(x4,q5k_hi({{ sPre }}words.x,({{ sPre }}qh>>16u)&15u))+dot(x5,q5k_hi({{ sPre }}words.y,({{ sPre }}qh>>20u)&15u))+dot(x6,q5k_hi({{ sPre }}words.z,({{ sPre }}qh>>24u)&15u))+dot(x7,q5k_hi({{ sPre }}words.w,({{ sPre }}qh>>28u)&15u)));
{% elif sLut2 %}{% filter indent(4, true) %}{{ lut2_block_scale(sPre ~ "blk", sPre ~ "2_", sPre ~ "s2", sPre ~ "s2lo", sPre ~ "s2hi", sPre ~ "db2", sLutc16, sMeta, sLutSfx, sLutId) }}{% endfilter %}let {{ sPre }}wa={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}wb={{ sBits }}[{{ sPre }}blk*2u+1u];
{% if sLutc16 %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2lo*(dot(x0,q2q{{ sLutSfx }}({{ sPre }}wa,0u,{{ sPre }}db2[0]))+dot(x1,q2q{{ sLutSfx }}({{ sPre }}wa,8u,{{ sPre }}db2[0]))+dot(x2,q2q{{ sLutSfx }}({{ sPre }}wa,16u,{{ sPre }}db2[1]))+dot(x3,q2q{{ sLutSfx }}({{ sPre }}wa,24u,{{ sPre }}db2[1])))+{{ sPre }}s2hi*(dot(x4,q2q{{ sLutSfx }}({{ sPre }}wb,0u,{{ sPre }}db2[2]))+dot(x5,q2q{{ sLutSfx }}({{ sPre }}wb,8u,{{ sPre }}db2[2]))+dot(x6,q2q{{ sLutSfx }}({{ sPre }}wb,16u,{{ sPre }}db2[3]))+dot(x7,q2q{{ sLutSfx }}({{ sPre }}wb,24u,{{ sPre }}db2[3])));
{% else %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2*(q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,0u,x0,x4,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,8u,x1,x5,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,16u,x2,x6,{{ sPre }}db2[1],{{ sPre }}db2[3])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,24u,x3,x7,{{ sPre }}db2[1],{{ sPre }}db2[3]));
{% endif %}{% elif sLut16 or (sLutc and sLutc16) %}{% if sLutc and sLutc16 %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*5u;let {{ sPre }}dl=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+4u]);let {{ sPre }}cw={{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>1u)];let {{ sPre }}cs=({{ sPre }}blk&1u)*16u;let {{ sPre }}s_lo={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>{{ sPre }}cs)&255u);let {{ sPre }}s_hi={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>({{ sPre }}cs+8u))&255u);
{% else %}let {{ sPre }}s_lo=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}s_hi=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s_lo*(dot(x0,q4_lo{{ sLutSfx }}({{ sPre }}w.x))+dot(x1,q4_lo{{ sLutSfx }}({{ sPre }}w.y))+dot(x2,q4_lo{{ sLutSfx }}({{ sPre }}w.z))+dot(x3,q4_lo{{ sLutSfx }}({{ sPre }}w.w)))+{{ sPre }}s_hi*(dot(x4,q4_hi{{ sLutSfx }}({{ sPre }}w.x))+dot(x5,q4_hi{{ sLutSfx }}({{ sPre }}w.y))+dot(x6,q4_hi{{ sLutSfx }}({{ sPre }}w.z))+dot(x7,q4_hi{{ sLutSfx }}({{ sPre }}w.w)));
{% elif sQ8Pair %}let {{ sPre }}s=q8pair_scale({{ sPre }}blk);let {{ sPre }}cb=q8pair_code_base({{ sPre }}blk);let {{ sPre }}w0={{ sBits }}[{{ sPre }}cb];let {{ sPre }}w1={{ sBits }}[{{ sPre }}cb+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3)+q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% else %}{% if sLutc %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*3u;let {{ sPre }}s=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+2u])*lutScale{{ sLutSfx }}(({{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>2u)]>>(({{ sPre }}blk&3u)*8u))&255u);
{% elif sIq4xs %}{{ iq4xs_block_scale(sPre ~ "blk", sPre ~ "n", sPre ~ "s", sMeta) | indent(4, true) }}
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-gate-up-accumulate-reduce.fe1bc3f98d.wgsl.jinja",`{% set gateUpIq1m = gateUpIq1m | default(iq1m | default(false)) %}{% set gateUpIq1s = gateUpIq1s | default(iq1s | default(false)) %}{% set gateUpIq1md = gateUpIq1md | default(iq1md | default(false)) %}{% set gateUpIq2xxs = gateUpIq2xxs | default(iq2xxs | default(false)) %}{% set gateUpIq3xxs = gateUpIq3xxs | default(iq3xxs | default(false)) %}{% set gateUpIq3s = gateUpIq3s | default(iq3s | default(false)) %}{% set gateUpIq2xs = gateUpIq2xs | default(iq2xs | default(false)) %}{% set gateUpIq2s = gateUpIq2s | default(iq2s | default(false)) %}{% set gateUpSuffixes = gateUpSuffixes | default([""]) %}{% set gateUpTiled = gateUpSuffixes | length > 1 %}{% set packMeta = packMeta | default("q4_scales") %}{% set q2Split = q2Split | default(false) %}{% set xOpen = xOpen | default("") %}{% set xClose = xClose | default("") %}{% set gateUpSplit = gateUpSplit | default(false) %}{% set gateUpShared = gateUpShared | default(false) %}{% set upIq4xs = upIq4xs | default(iq4xs | default(false)) %}{% set gateUpNeedsSa = (hasMin or (q4knCompact is defined and q4knCompact) or (q5k is defined and q5k)
      or (gateUpSplit and (upHasMin or upQ4knc or upQ5k))) if not gateUpShared else ((sharedQ4knc | default(false)) or (sharedQ5k | default(false))) %}{% set pairGateBits = "gate_bits" if q2Split else "q4_bits" %}{% set pairGateScales = "gate_scales" if q2Split else "q4_scales" %}{% set pairUpBits = "up_bits" if (q2Split or gateUpSplit) else "q4_bits" %}{% set pairUpScales = "up_scales" if (q2Split or gateUpSplit) else "q4_scales" %}{% set pairView = not gateUpSplit %}{% if gateUpOwned | default(false) %}{% set gateUpSumOpen = "subgroupAdd(" %}{% set gateUpSumClose = ")" %}{% else %}{% set gateUpSumOpen = "sg_safe_sum(" %}{% set gateUpSumClose = ", lid, SG_SAFE_WG, sgNative)" %}{% endif %}{% if (gateUpIq1s or (upIq1s | default(false)) or gateUpIq1md or (upIq1md | default(false))) and not gateUpShared %}let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);
{% if gateUpIq1md or (upIq1md | default(false)) %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% endif %}{% for m in range(mRows) %}{% set accG = "acc_g" if m == 0 else "accM" ~ m ~ "_g" %}{% set accU = "acc_u" if m == 0 else "accM" ~ m ~ "_u" %}{% if m > 0 %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endif %}{% if gateUpNeedsSa %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in gateUpSuffixes %}{% if gateUpTiled %}{
{% endif %}{% set sPre = "g" %}{% set sBlk = "gate_block_base" ~ r ~ " + b" %}{% set sAcc = accG ~ r %}{% include "ops/_shared/q4-block-dot-flags.af1c2cef54.wgsl.jinja" %}{% set sBits = pairGateBits %}{% set sScales = pairGateScales %}{% set sIq1m = gateUpIq1m %}{% set sIq1s = gateUpIq1s %}{% set sIq1md = gateUpIq1md %}{% set sIq2xxs = gateUpIq2xxs %}{% set sIq3xxs = gateUpIq3xxs %}{% set sIq3s = gateUpIq3s %}{% set sIq2xs = gateUpIq2xs %}{% set sIq2s = gateUpIq2s %}{% set sLutSfx = "" %}{% set sPairView = pairView %}{% set sQ8Pair = 0 %}{% if gateUpShared %}{% include "ops/_shared/q4-block-dot-shared-flags.c1c2072bbb.wgsl.jinja" %}{% endif %}{% include "ops/_shared/q4-block-dot.e9fc34b355.wgsl.jinja" %}{% set sPre = "u" %}{% set sBlk = "up_block_base" ~ r ~ " + b" %}{% set sAcc = accU ~ r %}{% set sBits = pairUpBits %}{% set sScales = pairUpScales %}{% if gateUpSplit %}{% set sMeta = "up_scales" %}{% set sHasMin = 0 %}{% set sQ6k = 0 %}{% set sQ5k = 0 %}{% set sQ3k = 0 %}{% set sQ2k = 0 %}{% set sQ4knc = 1 %}{% set sLut16 = 0 %}{% set sLutc = 0 %}{% set sLutc16 = 0 %}{% set sLut2 = 0 %}{% set sIq4xs = upIq4xs %}{% set sIq1m = false %}{% set sIq1s = false %}{% set sIq1md = false %}{% set sIq2xxs = false %}{% set sIq3xxs = false %}{% set sIq3s = false %}{% set sIq2xs = false %}{% set sIq2s = false %}{% set sLutId = 0 %}{% set sLutSfx = "" %}{% set sIqSfx = "" %}{% endif %}{% if gateUpShared %}{% include "ops/_shared/q4-block-dot-shared-flags.c1c2072bbb.wgsl.jinja" %}{% endif %}{% include "ops/_shared/q4-block-dot.0926c1462b.wgsl.jinja" %}{% if gateUpTiled %}}
{% endif %}{% endfor %}{% if m > 0 %}}
{% endif %}{% endfor %}}
{% set gateUpRuntime = sgRuntime | default(false) %}{% include "ops/_shared/subgroup-runtime-fold.58b998281f.wgsl.jinja" %}{% if gateUpRuntime %}{% set gateUpFold = namespace(pairs=[]) %}{% for r in gateUpSuffixes %}{% set gateUpFold.pairs = gateUpFold.pairs + [["acc_g" ~ r, "g" ~ r], ["acc_u" ~ r, "u" ~ r]] %}{% endfor %}{% for m in range(1, mRows) %}{% for r in gateUpSuffixes %}{% set gateUpFold.pairs = gateUpFold.pairs + [["accM" ~ m ~ "_g" ~ r, "gM" ~ m ~ "_" ~ r], ["accM" ~ m ~ "_u" ~ r, "uM" ~ m ~ "_" ~ r]] %}{% endfor %}{% endfor %}{{ sg_fold_workgroup(gateUpFold.pairs) }}
{% elif gateUpTiled %}{% if useSubgroups %}{% for r in gateUpSuffixes %}let g{{ r }}={{ gateUpSumOpen }}acc_g{{ r }}{{ gateUpSumClose }};let u{{ r }}={{ gateUpSumOpen }}acc_u{{ r }}{{ gateUpSumClose }};
{% endfor %}{% else %}{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=acc_g{{ r }};partials_u{{ r }}[lid]=acc_u{{ r }};
{% endfor %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=partials_g{{ r }}[lid]+partials_g{{ r }}[lid+stride];partials_u{{ r }}[lid]=partials_u{{ r }}[lid]+partials_u{{ r }}[lid+stride];
{% endfor %}}workgroupBarrier();stride=stride/2u;}
{% for r in gateUpSuffixes %}let g{{ r }}=partials_g{{ r }}[0];let u{{ r }}=partials_u{{ r }}[0];
{% endfor %}{% endif %}{% else %}{% if useSubgroups %}let g={{ gateUpSumOpen }}acc_g{{ gateUpSumClose }};let u={{ gateUpSumOpen }}acc_u{{ gateUpSumClose }};
{% else %}partials_g[lid]=acc_g;partials_u[lid]=acc_u;workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){partials_g[lid]=partials_g[lid]+partials_g[lid+stride];partials_u[lid]=partials_u[lid]+partials_u[lid+stride];}workgroupBarrier();stride=stride/2u;}let g=partials_g[0];let u=partials_u[0];
{% endif %}{% endif %}{% for m in range(1, 1 if gateUpRuntime else mRows) %}{% if not useSubgroups %}workgroupBarrier();
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=accM{{ m }}_g{{ r }};partials_u{{ r }}[lid]=accM{{ m }}_u{{ r }};
{% endfor %}workgroupBarrier();var strideM{{ m }}=WORKGROUP_SIZE/2u;loop{if(strideM{{ m }}==0u){break;}if(lid<strideM{{ m }}){
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=partials_g{{ r }}[lid]+partials_g{{ r }}[lid+strideM{{ m }}];partials_u{{ r }}[lid]=partials_u{{ r }}[lid]+partials_u{{ r }}[lid+strideM{{ m }}];
{% endfor %}}workgroupBarrier();strideM{{ m }}=strideM{{ m }}/2u;}
{% for r in gateUpSuffixes %}let gM{{ m }}_{{ r }}=partials_g{{ r }}[0];let uM{{ m }}_{{ r }}=partials_u{{ r }}[0];
{% endfor %}{% else %}{% for r in gateUpSuffixes %}let gM{{ m }}_{{ r }}={{ gateUpSumOpen }}accM{{ m }}_g{{ r }}{{ gateUpSumClose }};let uM{{ m }}_{{ r }}={{ gateUpSumOpen }}accM{{ m }}_u{{ r }}{{ gateUpSumClose }};
{% endfor %}{% endif %}{% endfor %}`],["ops/_shared/packed-4x8-compat.ac66f822e2.wgsl.jinja",`{% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}@must_use fn unpack4xI8(e:u32)->vec4<i32>{let bytes=(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);return vec4<i32>(bytes^vec4<u32>(0x80u))-vec4<i32>(128i);}@must_use fn unpack4xU8(e:u32)->vec4<u32>{return(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);}
{% endif %}`],["ops/_shared/q4-dequant-helpers.1f30cb9761.wgsl.jinja","fn q6_lo(word:u32,hLo:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|(((hLo>>(2u*w))&0x03030303u)<<4u)))-32.0;}fn q6_hi(word:u32,hHi:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|(((hHi>>(2u*w))&0x03030303u)<<4u)))-32.0;}"],["ops/_shared/matmul-argmax-hidden4.c040ca2b55.wgsl.jinja","fn hidden4(dim4:u32)->vec4<f32>{let base=dim4*4u;return vec4<f32>(hidden[base],hidden[base+1u],hidden[base+2u],hidden[base+3u]);}"],["ops/_shared/q4-block-dot-flags.02560bb475.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sScales = "q4_scales" %}'],["ops/_shared/q4-block-dot.a5cefe4290.wgsl.jinja","let {{ sPre }}blk={{ sBlk }};let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));"],["ops/_shared/q4-decode-row-block-accumulate.6b9bc7b528.wgsl.jinja",'{% set accPrefix = "acc" %}{% set rowBase = "base0" %}{% include "ops/_shared/q4-block-dot-flags.02560bb475.wgsl.jinja" %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.a5cefe4290.wgsl.jinja" %}{% endfor %}'],["ops/_shared/argmax-wg-tree.13d09b6502.wgsl.jinja",`best_values[lid]=best_value;best_indices[lid]=best_token;workgroupBarrier();
{% if workgroupSize > 1 %}var am_stride=WORKGROUP_SIZE/2u;loop{if(lid<am_stride){let other_value=best_values[lid+am_stride];let other_index=best_indices[lid+am_stride];if(other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid])){best_values[lid]=other_value;best_indices[lid]=other_index;} }workgroupBarrier();if(am_stride==1u){break;}am_stride=am_stride/2u;}
{% endif %}if(lid==0u){best_values_out[block]=best_values[0];best_ids_out[block]=best_indices[0];}`],["ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja",`{% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}{% if (packed4x8CompatUnpackI8 | default(false)) or (packed4x8CompatUnpackU8 | default(false)) or (packed4x8CompatPackI8 | default(false)) or (packed4x8CompatDotI8 | default(false)) %}{% endif %}{% if packed4x8CompatUnpackI8 | default(false) %}@must_use fn unpack4xI8(e:u32)->vec4<i32>{let bytes=(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);return vec4<i32>(bytes^vec4<u32>(0x80u))-vec4<i32>(128i);}
{% endif %}{% if packed4x8CompatUnpackU8 | default(false) %}@must_use fn unpack4xU8(e:u32)->vec4<u32>{return(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);}
{% endif %}{% endif %}`],["ops/_shared/q4-dequant-helpers.33f455d750.wgsl.jinja",`{% if nibbleHelpers | default(true) %}fn q4_lo(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}fn q4_hi(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}fn q4wp(word:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q4_lo(word))+dot(hi,q4_hi(word));}
{% endif %}{% if q6k %}fn q6_lo(word:u32,hLo:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|(((hLo>>(2u*w))&0x03030303u)<<4u)))-32.0;}fn q6_hi(word:u32,hHi:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|(((hHi>>(2u*w))&0x03030303u)<<4u)))-32.0;}
{% endif %}`],["ops/_shared/q4-block-dot-flags.491790ccdf.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sScales = "q4_scales" %}{% set sMeta = packMeta %}{% set sHasMin = 0 %}{% set sQ6k = q6k | default(false) %}{% set sQ5k = 0 %}{% set sQ4knc = q4knCompact | default(false) %}{% set sLutSfx = "" %}'],["ops/_shared/q4-block-dot.fa5ab044ee.wgsl.jinja",`let {{ sPre }}blk={{ sBlk }};
{% if sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ6k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-decode-row-block-accumulate.b09a43b589.wgsl.jinja",`{% set packMeta = "q4_scales" %}{% set accPrefix = "acc" %}{% set rowBase = "base0" %}{% include "ops/_shared/q4-block-dot-flags.491790ccdf.wgsl.jinja" %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.fa5ab044ee.wgsl.jinja" %}{% endfor %}`],["ops/_shared/q4-normed-projection-multirow.ed1980421f.wgsl.jinja",`{% set packMeta = "q4_scales" %}{% for r in range(regRows) %}var acc{{ r }}=0.0;
{% endfor %}for(var b=lane;b<BPR;b=b+32u){let hbase=b*8u;
{% for i in range(8) %}let x{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% include "ops/_shared/q4-decode-row-block-accumulate.b09a43b589.wgsl.jinja" %}  }`],["ops/_shared/q4-decode-row-block-accumulate.9f465ebc3a.wgsl.jinja",`{% set packMeta = packMeta | default("q4_metadata") %}{% set accPrefix = accPrefix | default("acc") %}{% set rowBase = "row_block_base" %}{% include "ops/_shared/q4-block-dot-flags.491790ccdf.wgsl.jinja" %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.fa5ab044ee.wgsl.jinja" %}{% endfor %}`],["ops/_shared/q4-normed-projection-single-row.7d02c37204.wgsl.jinja",`{% set packMeta = packMeta | default("q4_scales") %}var acc0=0.0;for(var b=lid;b<BPR;b=b+WORKGROUP_SIZE){let hbase=b*8u;
{% for i in range(8) %}let {{ "x" if preNormed else "h" }}{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% set regRows = 1 %}{% set rowBase = "row_block_base" %}{% include "ops/_shared/q4-decode-row-block-accumulate.9f465ebc3a.wgsl.jinja" %}  }
{% if useSubgroups %}let acc_total=sg_safe_sum(acc0,lid,SG_SAFE_WG,sgNative);
{% elif sgRuntime | default(false) %}{{ sg_fold_workgroup([["acc0", "acc_total"]] if preNormed else [["acc0", "acc_total"], ["ss", "ss_total"]]) }}
{% else %}partials[lid]=acc0;workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){partials[lid]=partials[lid]+partials[lid+stride];}workgroupBarrier();if(stride==1u){break;}stride=stride/2u;}let acc_total=partials[0];
{% endif %}`],["ops/_shared/subgroup-owned-clusters.87076f64db.wgsl.jinja",`{% include "ops/_shared/subgroup-active-lanes.97fae981c2.wgsl.jinja" %}{% set sgClusterPaths = ["general", "full"] %}{% set sgClusterBuiltins = ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32, @builtin(subgroup_id) sg_id: u32, @builtin(num_subgroups) sg_count: u32" %}{% macro sg_cluster_open(path, clusters, totals) %}{% if path == "general" %}if(sg_size!=32u||sg_count!={{ clusters }}){if(sg_id==0u){ {{ active_lanes("sg_lid") }} let lane=activeRank;var sgTotals:array<f32,{{ totals }}>;for(var cluster=0u;cluster<{{ clusters }};cluster=cluster+1u){
{% else %}}else{let cluster=sg_id;let lane=sg_lid&31u;
{% endif %}{% endmacro %}{% macro sg_cluster_store(clusters) %}}for(var cluster=0u;cluster<{{ clusters }};cluster=cluster+1u){
{% endmacro %}{% macro sg_cluster_close(path) %}{% if path == "general" %}} }
{% else %}}
{% endif %}{% endmacro %}`],["ops/_shared/iq-inline-dot.a397430180.wgsl.jinja",`{% set iqAtGrid1 = iq1grid_at if iq1grid_at is defined else none %}{% set iqAtIq2sLo = iq2sLo_at if iq2sLo_at is defined else none %}{% set iqAtIq2sHi = iq2sHi_at if iq2sHi_at is defined else none %}{% set iqAtMag4 = iq3mag_at if iq3mag_at is defined else none %}{% macro iq_dot_acc(P, acc, kind) %}{% if kind == "two" %}{{ acc }}={{ acc }}+(dlA{{ P }}*((dot(g{{ P }}_0,x0)+dot(g{{ P }}_1,x1))+(dot(g{{ P }}_2,x2)+dot(g{{ P }}_3,x3)))+dlB{{ P }}*((dot(g{{ P }}_4,x4)+dot(g{{ P }}_5,x5))+(dot(g{{ P }}_6,x6)+dot(g{{ P }}_7,x7))));
{% else %}{{ acc }}={{ acc }}+db{{ P }}*((dot(g{{ P }}_0,x0)+dot(g{{ P }}_1,x1))+(dot(g{{ P }}_2,x2)+dot(g{{ P }}_3,x3))+(dot(g{{ P }}_4,x4)+dot(g{{ P }}_5,x5))+(dot(g{{ P }}_6,x6)+dot(g{{ P }}_7,x7)));
{% endif %}{% endmacro %}{% macro iq1_regs(P, q, m, d, sfx="") %}let iq1q{{ P }}={{ q }};let iq1m{{ P }}={{ m }};let iq1d{{ P }}={{ d }};let dlA{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>16u)&7u)+1u);let dlB{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>19u)&7u)+1u);let qh0_{{ P }}=iq1m{{ P }}&0xFFu;let qh1_{{ P }}=(iq1m{{ P }}>>8u)&0xFFu;
{% for c in range(4) %}{% set lo = ["iq1q" ~ P ~ " & 0xFFu", "(iq1q" ~ P ~ " >> 8u) & 0xFFu", "(iq1q" ~ P ~ " >> 16u) & 0xFFu", "iq1q" ~ P ~ " >> 24u"][c] %}{% set qh = ("qh0_" ~ P) if c < 2 else ("qh1_" ~ P) %}{% set hi = "(" ~ qh ~ (" << 8u)" if c % 2 == 0 else " << 4u)") ~ " & 0x700u" %}{% set neg = "(" ~ qh ~ (" & 0x08u)" if c % 2 == 0 else " & 0x80u)") ~ " != 0u" %}let gi{{ P }}_{{ c }}=({{ lo }})|({{ hi }});let gg{{ P }}_{{ c }}={{ iqAtGrid1("gi" ~ P ~ "_" ~ c, sfx) }};let dd{{ P }}_{{ c }}=select(0.125,-0.125,{{ neg }});let g{{ P }}_{{ 2 * c }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.x))+dd{{ P }}_{{ c }};let g{{ P }}_{{ 2 * c + 1 }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.y))+dd{{ P }}_{{ c }};
{% endfor %}{% endmacro %}{% macro iq2s_regs(P, qw, sw, qh, sc, d, sfx="") %}let iq2qw{{ P }}={{ qw }};let iq2sw{{ P }}={{ sw }};let iq2qh{{ P }}={{ qh }};let iq2sc{{ P }}={{ sc }};let iq2d{{ P }}={{ d }};let dlA{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}&0xFu))*0.25;let dlB{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}>>4u))*0.25;
{% for c in range(4) %}{% set qwn = "iq2qw" ~ P %}{% set qhn = "iq2qh" ~ P %}{% set gi = [
"("~qwn~"&0xFFu)|(("~qhn~"<<8u)&0x300u)","(("~qwn~">>8u)&0xFFu)|(("~qhn~"<<6u)&0x300u)","(("~qwn~">>16u)&0xFFu)|(("~qhn~"<<4u)&0x300u)",
  "(" ~ qwn ~ " >> 24u) | ((" ~ qhn ~ " << 2u) & 0x300u)"][c] %}{% set sv = ("iq2sw" ~ P) if c == 0 else ("iq2sw" ~ P ~ " >> " ~ (8 * c) ~ "u") %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}={{ sv }};let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtIq2sLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtIq2sHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}{% macro iq3xxs_regs(P, w0, w1, sw, d, sfx="") %}let iq3w0_{{ P }}={{ w0 }};let iq3w1_{{ P }}={{ w1 }};let iq3sw{{ P }}={{ sw }};let db{{ P }}={{ d }}*(0.5+f32(iq3sw{{ P }}>>28u))*0.5;
{% for c in range(4) %}{% set word = "iq3w0_" ~ P if c < 2 else "iq3w1_" ~ P %}{% set lo = 16 * (c % 2) %}let s{{ P }}_{{ c }}=ksg((iq3sw{{ P }}>>{{ 7 * c }}u)&127u);let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtMag4("(" ~ word ~ " >> " ~ lo ~ "u) & 0xFFu", sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtMag4(("(" ~ word ~ " >> " ~ (lo + 8) ~ "u) & 0xFFu") if lo == 0 else (word ~ " >> " ~ (lo + 8) ~ "u"), sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}`],["ops/_shared/q4-block-dot.e711a4e15c.wgsl.jinja",`{% include "ops/_shared/iq-inline-dot.a397430180.wgsl.jinja" %}    let {{ sPre }}blk = {{ sBlk }};
{% if sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ6k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-decode-row-block-accumulate.2710523b30.wgsl.jinja",`{% set accPrefix = accPrefix | default("acc") %}{% set rowBase = rowBase | default("base0") %}{% include "ops/_shared/q4-block-dot-flags.491790ccdf.wgsl.jinja" %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.e711a4e15c.wgsl.jinja" %}{% endfor %}`],["ops/_shared/projection-multirow-reduce.ccd7057c03.wgsl.jinja",`{% set accPrefix = accPrefix | default("acc") %}{% set totalPrefix = totalPrefix | default("total") %}{% if (sgPath | default("")) == "general" %}{% for r in range(regRows) %}sgTotals[{{ sgTotalsSlot | default("") }}{{ r }}u]=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif sgOwned | default(false) %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif useSubgroups %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=sg_safe_sum({{ accPrefix }}{{ r }},lid,32u,sgNative);
{% endfor %}{% elif sgRuntime | default(false) %}{% set sgFold = namespace(pairs=[]) %}{% for r in range(regRows) %}{% set sgFold.pairs = sgFold.pairs + [[accPrefix ~ r, totalPrefix ~ r]] %}{% endfor %}{{ sg_fold_cluster32(sgFold.pairs) }}
{% elif partialsRowVector | default(false) %}  partials4[lid] = vec{{ regRows }}<f32>({% for r in range(regRows) %}{{ ", " if r else "" }}{{ accPrefix }}{{ r }}{% endfor %});
workgroupBarrier();var {{ totalPrefix }}Stride=16u;loop{if({{ totalPrefix }}Stride==0u){break;}if(lane<{{ totalPrefix }}Stride){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride];}workgroupBarrier();{{ totalPrefix }}Stride={{ totalPrefix }}Stride/2u;}let {{ totalPrefix }}Rows=partials4[cluster*32u];
{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}={{ totalPrefix }}Rows.{{ "xyzw"[r] }};
{% endfor %}workgroupBarrier();
{% else %}{% for r in range(regRows) %}partials4[lid]={{ accPrefix }}{{ r }};workgroupBarrier();var {{ totalPrefix }}Stride{{ r }}=16u;loop{if({{ totalPrefix }}Stride{{ r }}==0u){break;}if(lane<{{ totalPrefix }}Stride{{ r }}){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride{{ r }}];}workgroupBarrier();{{ totalPrefix }}Stride{{ r }}={{ totalPrefix }}Stride{{ r }}/2u;}let {{ totalPrefix }}{{ r }}=partials4[cluster*32u];workgroupBarrier();
{% endfor %}{% endif %}`],["ops/_shared/packed-4x8-compat.0c5ce8f6e4.wgsl.jinja",`{% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}@must_use fn unpack4xI8(e:u32)->vec4<i32>{let bytes=(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);return vec4<i32>(bytes^vec4<u32>(0x80u))-vec4<i32>(128i);}
{% endif %}`],["ops/_shared/subgroup-runtime-fold.3a89b81fa3.wgsl.jinja",`{% macro sg_fold_cluster32(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,32u,sgNative);
{% endfor %}{% endmacro %}`],["ops/_shared/projection-multirow-reduce.e8bf64d477.wgsl.jinja",`{% set accPrefix = accPrefix | default("acc") %}{% set totalPrefix = totalPrefix | default("total") %}{% if (sgPath | default("")) == "general" %}{% for r in range(regRows) %}sgTotals[{{ sgTotalsSlot | default("") }}{{ r }}u]=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif sgOwned | default(false) %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif useSubgroups %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=sg_safe_sum({{ accPrefix }}{{ r }},lid,32u,sgNative);
{% endfor %}{% elif sgRuntime | default(false) %}{% set sgFold = namespace(pairs=[]) %}{% for r in range(regRows) %}{% set sgFold.pairs = sgFold.pairs + [[accPrefix ~ r, totalPrefix ~ r]] %}{% endfor %}{{ sg_fold_cluster32(sgFold.pairs) }}
{% else %}{% for r in range(regRows) %}partials4[lid]={{ accPrefix }}{{ r }};workgroupBarrier();var {{ totalPrefix }}Stride{{ r }}=16u;loop{if({{ totalPrefix }}Stride{{ r }}==0u){break;}if(lane<{{ totalPrefix }}Stride{{ r }}){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride{{ r }}];}workgroupBarrier();{{ totalPrefix }}Stride{{ r }}={{ totalPrefix }}Stride{{ r }}/2u;}let {{ totalPrefix }}{{ r }}=partials4[cluster*32u];workgroupBarrier();
{% endfor %}{% endif %}`],["ops/_shared/f16-bits-to-f32.09629f6275.wgsl.jinja","fn f16_bits_to_f32(h:u32)->f32{let sign=(h&0x8000u)<<16u;let exponent=(h>>10u)&0x1fu;let mantissa=h&0x3ffu;if(exponent==0u){return bitcast<f32>(sign|bitcast<u32>(f32(mantissa)*5.9604644775390625e-8));}if(exponent==31u){return bitcast<f32>(sign|0x7f800000u|(mantissa<<13u));}return bitcast<f32>(sign|((exponent+112u)<<23u)|(mantissa<<13u));}"],["ops/_shared/matmul-reg-microtile-fma.1983670e1d.wgsl.jinja",'{% set regAccumulator = "acc" %}{% set regColumnStride = 0 %}{% macro regARead(row) %}tileA[{{ row }}][kv]{% endmacro %}{% macro regBRead(column) %}{% set regBWord = ("tileB[kv][" ~ column ~ "]") if regColumnStride else ("tileB[" ~ column ~ "][kv]") %}{{ regBWord }}{% endmacro %}let aRow=lid.y*TM;let bCol=lid.x*TN;for(var kv:u32=0u;kv<BK/4u;kv=kv+1u){var av:array<vec4<{{ regT }}>,TM>;var bv:array<vec4<{{ regT }}>,TN>;for(var i:u32=0u;i<TM;i=i+1u){av[i]={{ regARead("aRow + i") }};}for(var j:u32=0u;j<TN;j=j+1u){bv[j]={{ regBRead(("bCol + j * " ~ regColumnStride ~ "u") if regColumnStride else "bCol + j") }};}for(var i:u32=0u;i<TM;i=i+1u){for(var j:u32=0u;j<TN;j=j+1u){ {{ regAccumulator }}[i*TN+j]={{ regAccumulator }}[i*TN+j]+dot(av[i],bv[j]);} } }'],["ops/_shared/iq-grid-iq1s.4a6f38db3c.wgsl.jinja","vec2<u32>(4294967295u,4294967295u),vec2<u32>(4294967041u,4294967295u),vec2<u32>(4294901760u,4294967295u),vec2<u32>(4294902271u,4294967295u),vec2<u32>(4294902017u,4294967295u),vec2<u32>(4278255360u,4294967295u),vec2<u32>(4278190080u,4294967295u),vec2<u32>(4278321151u,4294967295u),vec2<u32>(4278320897u,4294967295u),vec2<u32>(4278256127u,4294967295u),vec2<u32>(4278255873u,4294967295u),vec2<u32>(16711680u,4294967295u),vec2<u32>(65280u,4294967295u),vec2<u32>(255u,4294967295u),vec2<u32>(1u,4294967295u),vec2<u32>(65536u,4294967295u),vec2<u32>(33554431u,4294967295u),vec2<u32>(33554177u,4294967295u),vec2<u32>(33489407u,4294967295u),vec2<u32>(33489153u,4294967295u),vec2<u32>(16777216u,4294967295u),vec2<u32>(16908287u,4294967295u),vec2<u32>(16908033u,4294967295u),vec2<u32>(16843263u,4294967295u),vec2<u32>(16843009u,4294967295u),vec2<u32>(4294902015u,4294967040u),vec2<u32>(4294901760u,4294967040u),vec2<u32>(4278255360u,4294967040u),vec2<u32>(4278190335u,4294967040u),vec2<u32>(4278190081u,4294967040u),vec2<u32>(4278190336u,4294967040u),vec2<u32>(4278190337u,4294967040u),vec2<u32>(4278255616u,4294967040u),vec2<u32>(16776960u,4294967040u),vec2<u32>(16711681u,4294967040u),vec2<u32>(16711936u,4294967040u),vec2<u32>(65281u,4294967040u),vec2<u32>(0u,4294967040u),vec2<u32>(257u,4294967040u),vec2<u32>(130816u,4294967040u),vec2<u32>(65791u,4294967040u),vec2<u32>(65537u,4294967040u),vec2<u32>(66047u,4294967040u),vec2<u32>(33488896u,4294967040u),vec2<u32>(16842496u,4294967040u),vec2<u32>(16777471u,4294967040u),vec2<u32>(16777217u,4294967040u),vec2<u32>(16842752u,4294967040u),vec2<u32>(4294967295u,4294967041u),vec2<u32>(4294967041u,4294967041u),vec2<u32>(4294902271u,4294967041u),vec2<u32>(4294902017u,4294967041u),vec2<u32>(4278190080u,4294967041u),vec2<u32>(4278321151u,4294967041u),vec2<u32>(4278320897u,4294967041u),vec2<u32>(4278256127u,4294967041u),vec2<u32>(4278255873u,4294967041u),vec2<u32>(16711680u,4294967041u),vec2<u32>(65280u,4294967041u),vec2<u32>(256u,4294967041u),vec2<u32>(65791u,4294967041u),vec2<u32>(65792u,4294967041u),vec2<u32>(33554431u,4294967041u),vec2<u32>(33554177u,4294967041u),vec2<u32>(33489407u,4294967041u),vec2<u32>(33489153u,4294967041u),vec2<u32>(16842496u,4294967041u),vec2<u32>(16777216u,4294967041u),vec2<u32>(16777472u,4294967041u),vec2<u32>(16908287u,4294967041u),vec2<u32>(16908033u,4294967041u),vec2<u32>(16843263u,4294967041u),vec2<u32>(16843009u,4294967041u),vec2<u32>(4278255360u,4294902015u),vec2<u32>(4278190335u,4294902015u),vec2<u32>(4278190081u,4294902015u),vec2<u32>(4278255616u,4294902015u),vec2<u32>(16776960u,4294902015u),vec2<u32>(16711936u,4294902015u),vec2<u32>(0u,4294902015u),vec2<u32>(257u,4294902015u),vec2<u32>(65791u,4294902015u),vec2<u32>(65536u,4294902015u),vec2<u32>(16842496u,4294902015u),vec2<u32>(16777472u,4294902015u),vec2<u32>(16842752u,4294902015u),vec2<u32>(4294967040u,4294901760u),vec2<u32>(4294902015u,4294901760u),vec2<u32>(4294901760u,4294901760u),vec2<u32>(4294901761u,4294901760u),vec2<u32>(4278190080u,4294901760u),vec2<u32>(4278190591u,4294901760u),vec2<u32>(4278190337u,4294901760u),vec2<u32>(4278255872u,4294901760u),vec2<u32>(16777215u,4294901760u),vec2<u32>(16711680u,4294901760u),vec2<u32>(16711937u,4294901760u),vec2<u32>(65535u,4294901760u),vec2<u32>(65280u,4294901760u),vec2<u32>(255u,4294901760u),vec2<u32>(0u,4294901760u),vec2<u32>(1u,4294901760u),vec2<u32>(256u,4294901760u),vec2<u32>(131071u,4294901760u),vec2<u32>(130817u,4294901760u),vec2<u32>(65536u,4294901760u),vec2<u32>(66047u,4294901760u),vec2<u32>(65793u,4294901760u),vec2<u32>(33554176u,4294901760u),vec2<u32>(16842496u,4294901760u),vec2<u32>(16777216u,4294901760u),vec2<u32>(16777727u,4294901760u),vec2<u32>(16777473u,4294901760u),vec2<u32>(16908032u,4294901760u),vec2<u32>(16843007u,4294901760u),vec2<u32>(16842752u,4294901760u),vec2<u32>(16842753u,4294901760u),vec2<u32>(16843008u,4294901760u),vec2<u32>(4278190335u,4294901761u),vec2<u32>(4278190336u,4294901761u),vec2<u32>(16776960u,4294901761u),vec2<u32>(16711935u,4294901761u),vec2<u32>(65535u,4294901761u),vec2<u32>(65281u,4294901761u),vec2<u32>(0u,4294901761u),vec2<u32>(511u,4294901761u),vec2<u32>(131071u,4294901761u),vec2<u32>(130816u,4294901761u),vec2<u32>(65537u,4294901761u),vec2<u32>(65792u,4294901761u),vec2<u32>(33488896u,4294901761u),vec2<u32>(16842496u,4294901761u),vec2<u32>(16777471u,4294901761u),vec2<u32>(16777472u,4294901761u),vec2<u32>(4294967295u,4294902271u),vec2<u32>(4294967041u,4294902271u),vec2<u32>(4294902271u,4294902271u),vec2<u32>(4294902017u,4294902271u),vec2<u32>(4278190080u,4294902271u),vec2<u32>(4278321151u,4294902271u),vec2<u32>(4278320897u,4294902271u),vec2<u32>(4278256127u,4294902271u),vec2<u32>(4278255873u,4294902271u),vec2<u32>(16711680u,4294902271u),vec2<u32>(65280u,4294902271u),vec2<u32>(1u,4294902271u),vec2<u32>(65536u,4294902271u),vec2<u32>(33554431u,4294902271u),vec2<u32>(33554177u,4294902271u),vec2<u32>(33489407u,4294902271u),vec2<u32>(33489153u,4294902271u),vec2<u32>(16777216u,4294902271u),vec2<u32>(16908287u,4294902271u),vec2<u32>(16908033u,4294902271u),vec2<u32>(16843263u,4294902271u),vec2<u32>(16843009u,4294902271u),vec2<u32>(4294901760u,4294902016u),vec2<u32>(4278255360u,4294902016u),vec2<u32>(4278190335u,4294902016u),vec2<u32>(4278190336u,4294902016u),vec2<u32>(4278255871u,4294902016u),vec2<u32>(4278255616u,4294902016u),vec2<u32>(16776960u,4294902016u),vec2<u32>(65535u,4294902016u),vec2<u32>(65280u,4294902016u),vec2<u32>(0u,4294902016u),vec2<u32>(130816u,4294902016u),vec2<u32>(65791u,4294902016u),vec2<u32>(65792u,4294902016u),vec2<u32>(16842496u,4294902016u),vec2<u32>(16777471u,4294902016u),vec2<u32>(16777217u,4294902016u),vec2<u32>(16777472u,4294902016u),vec2<u32>(16842752u,4294902016u),vec2<u32>(4294967295u,4294902017u),vec2<u32>(4294967041u,4294902017u),vec2<u32>(4294902271u,4294902017u),vec2<u32>(4294902017u,4294902017u),vec2<u32>(4278190080u,4294902017u),vec2<u32>(4278321151u,4294902017u),vec2<u32>(4278320897u,4294902017u),vec2<u32>(4278256127u,4294902017u),vec2<u32>(4278255873u,4294902017u),vec2<u32>(16711680u,4294902017u),vec2<u32>(65280u,4294902017u),vec2<u32>(256u,4294902017u),vec2<u32>(130816u,4294902017u),vec2<u32>(65536u,4294902017u),vec2<u32>(33554431u,4294902017u),vec2<u32>(33554177u,4294902017u),vec2<u32>(33488896u,4294902017u),vec2<u32>(33489407u,4294902017u),vec2<u32>(33489153u,4294902017u),vec2<u32>(16777216u,4294902017u),vec2<u32>(16908287u,4294902017u),vec2<u32>(16908033u,4294902017u),vec2<u32>(16843263u,4294902017u),vec2<u32>(16843009u,4294902017u),vec2<u32>(4278255615u,4278255615u),vec2<u32>(4278255360u,4278255615u),vec2<u32>(4278190335u,4278255615u),vec2<u32>(4278190336u,4278255615u),vec2<u32>(4278255871u,4278255615u),vec2<u32>(4278255616u,4278255615u),vec2<u32>(16776960u,4278255615u),vec2<u32>(16711935u,4278255615u),vec2<u32>(65535u,4278255615u),vec2<u32>(0u,4278255615u),vec2<u32>(511u,4278255615u),vec2<u32>(130816u,4278255615u),vec2<u32>(65791u,4278255615u),vec2<u32>(65536u,4278255615u),vec2<u32>(65792u,4278255615u),vec2<u32>(16842496u,4278255615u),vec2<u32>(16777471u,4278255615u),vec2<u32>(16777217u,4278255615u),vec2<u32>(16908032u,4278255615u),vec2<u32>(16842752u,4278255615u),vec2<u32>(4294967040u,4278255360u),vec2<u32>(4294902015u,4278255360u),vec2<u32>(4294901761u,4278255360u),vec2<u32>(4294902016u,4278255360u),vec2<u32>(4278255615u,4278255360u),vec2<u32>(4278255361u,4278255360u),vec2<u32>(4278190080u,4278255360u),vec2<u32>(4278190591u,4278255360u),vec2<u32>(4278320896u,4278255360u),vec2<u32>(4278255871u,4278255360u),vec2<u32>(4278255872u,4278255360u),vec2<u32>(16711680u,4278255360u),vec2<u32>(16711937u,4278255360u),vec2<u32>(65535u,4278255360u),vec2<u32>(65280u,4278255360u),vec2<u32>(65281u,4278255360u),vec2<u32>(255u,4278255360u),vec2<u32>(0u,4278255360u),vec2<u32>(1u,4278255360u),vec2<u32>(256u,4278255360u),vec2<u32>(131071u,4278255360u),vec2<u32>(65536u,4278255360u),vec2<u32>(33489151u,4278255360u),vec2<u32>(16842497u,4278255360u),vec2<u32>(16777216u,4278255360u),vec2<u32>(16908032u,4278255360u),vec2<u32>(16843007u,4278255360u),vec2<u32>(4278255360u,4278255361u),vec2<u32>(4278190335u,4278255361u),vec2<u32>(4278190081u,4278255361u),vec2<u32>(4278255616u,4278255361u),vec2<u32>(16777215u,4278255361u),vec2<u32>(16711681u,4278255361u),vec2<u32>(16711936u,4278255361u),vec2<u32>(65281u,4278255361u),vec2<u32>(0u,4278255361u),vec2<u32>(511u,4278255361u),vec2<u32>(257u,4278255361u),vec2<u32>(65791u,4278255361u),vec2<u32>(65537u,4278255361u),vec2<u32>(33488896u,4278255361u),vec2<u32>(16842496u,4278255361u),vec2<u32>(16777471u,4278255361u),vec2<u32>(16777217u,4278255361u),vec2<u32>(16842752u,4278255361u),vec2<u32>(4294967040u,4278190335u),vec2<u32>(4294901761u,4278190335u),vec2<u32>(4294902016u,4278190335u),vec2<u32>(4278190335u,4278190335u),vec2<u32>(4278190080u,4278190335u),vec2<u32>(4278190591u,4278190335u),vec2<u32>(4278190336u,4278190335u),vec2<u32>(4278320896u,4278190335u),vec2<u32>(4278255617u,4278190335u),vec2<u32>(16776960u,4278190335u),vec2<u32>(16711680u,4278190335u),vec2<u32>(16711681u,4278190335u),vec2<u32>(16712191u,4278190335u),vec2<u32>(16711937u,4278190335u),vec2<u32>(65280u,4278190335u),vec2<u32>(255u,4278190335u),vec2<u32>(0u,4278190335u),vec2<u32>(1u,4278190335u),vec2<u32>(256u,4278190335u),vec2<u32>(130817u,4278190335u),vec2<u32>(65536u,4278190335u),vec2<u32>(66047u,4278190335u),vec2<u32>(33489151u,4278190335u),vec2<u32>(33489152u,4278190335u),vec2<u32>(16842751u,4278190335u),vec2<u32>(16777471u,4278190335u),vec2<u32>(16777216u,4278190335u),vec2<u32>(16777727u,4278190335u),vec2<u32>(16777472u,4278190335u),vec2<u32>(16777473u,4278190335u),vec2<u32>(16908032u,4278190335u),vec2<u32>(16843007u,4278190335u),vec2<u32>(16842752u,4278190335u),vec2<u32>(16843008u,4278190335u),vec2<u32>(4294967041u,4278190080u),vec2<u32>(4294901760u,4278190080u),vec2<u32>(4294902017u,4278190080u),vec2<u32>(4278255360u,4278190080u),vec2<u32>(4278190335u,4278190080u),vec2<u32>(4278190080u,4278190080u),vec2<u32>(4278190081u,4278190080u),vec2<u32>(4278190336u,4278190080u),vec2<u32>(4278321151u,4278190080u),vec2<u32>(4278320897u,4278190080u),vec2<u32>(4278255616u,4278190080u),vec2<u32>(4278256127u,4278190080u),vec2<u32>(4278255873u,4278190080u),vec2<u32>(16776960u,4278190080u),vec2<u32>(16711935u,4278190080u),vec2<u32>(16711680u,4278190080u),vec2<u32>(16711681u,4278190080u),vec2<u32>(65280u,4278190080u),vec2<u32>(65281u,4278190080u),vec2<u32>(255u,4278190080u),vec2<u32>(0u,4278190080u),vec2<u32>(1u,4278190080u),vec2<u32>(256u,4278190080u),vec2<u32>(257u,4278190080u),vec2<u32>(130816u,4278190080u),vec2<u32>(65791u,4278190080u),vec2<u32>(65536u,4278190080u),vec2<u32>(65537u,4278190080u),vec2<u32>(65792u,4278190080u),vec2<u32>(33554431u,4278190080u),vec2<u32>(33554177u,4278190080u),vec2<u32>(33489151u,4278190080u),vec2<u32>(33488896u,4278190080u),vec2<u32>(33489407u,4278190080u),vec2<u32>(33489153u,4278190080u),vec2<u32>(16842751u,4278190080u),vec2<u32>(16842496u,4278190080u),vec2<u32>(16777471u,4278190080u),vec2<u32>(16777216u,4278190080u),vec2<u32>(16777217u,4278190080u),vec2<u32>(16777472u,4278190080u),vec2<u32>(16777473u,4278190080u),vec2<u32>(16908287u,4278190080u),vec2<u32>(16908033u,4278190080u),vec2<u32>(16842752u,4278190080u),vec2<u32>(4294967040u,4278190081u),vec2<u32>(4294902015u,4278190081u),vec2<u32>(4294901760u,4278190081u),vec2<u32>(4294901761u,4278190081u),vec2<u32>(4278190080u,4278190081u),vec2<u32>(4278190081u,4278190081u),vec2<u32>(4278190591u,4278190081u),vec2<u32>(4278190337u,4278190081u),vec2<u32>(4278320896u,4278190081u),vec2<u32>(4278255617u,4278190081u),vec2<u32>(16777215u,4278190081u),vec2<u32>(16776961u,4278190081u),vec2<u32>(16711935u,4278190081u),vec2<u32>(16711680u,4278190081u),vec2<u32>(16712191u,4278190081u),vec2<u32>(16711937u,4278190081u),vec2<u32>(65280u,4278190081u),vec2<u32>(0u,4278190081u),vec2<u32>(1u,4278190081u),vec2<u32>(511u,4278190081u),vec2<u32>(256u,4278190081u),vec2<u32>(130816u,4278190081u),vec2<u32>(65791u,4278190081u),vec2<u32>(65536u,4278190081u),vec2<u32>(66047u,4278190081u),vec2<u32>(65792u,4278190081u),vec2<u32>(65793u,4278190081u),vec2<u32>(33488897u,4278190081u),vec2<u32>(33489153u,4278190081u),vec2<u32>(16842497u,4278190081u),vec2<u32>(16777216u,4278190081u),vec2<u32>(16843007u,4278190081u),vec2<u32>(16843008u,4278190081u),vec2<u32>(4278255360u,4278190591u),vec2<u32>(4278190081u,4278190591u),vec2<u32>(4278255616u,4278190591u),vec2<u32>(16776960u,4278190591u),vec2<u32>(16711935u,4278190591u),vec2<u32>(16711681u,4278190591u),vec2<u32>(16711936u,4278190591u),vec2<u32>(65535u,4278190591u),vec2<u32>(0u,4278190591u),vec2<u32>(511u,4278190591u),vec2<u32>(257u,4278190591u),vec2<u32>(131071u,4278190591u),vec2<u32>(130816u,4278190591u),vec2<u32>(65791u,4278190591u),vec2<u32>(65537u,4278190591u),vec2<u32>(65792u,4278190591u),vec2<u32>(33488896u,4278190591u),vec2<u32>(16842496u,4278190591u),vec2<u32>(16777471u,4278190591u),vec2<u32>(16842752u,4278190591u),vec2<u32>(4278255615u,4278190336u),vec2<u32>(4278255361u,4278190336u),vec2<u32>(4278190080u,4278190336u),vec2<u32>(4278190337u,4278190336u),vec2<u32>(4278320896u,4278190336u),vec2<u32>(4278255616u,4278190336u),vec2<u32>(16776961u,4278190336u),vec2<u32>(16711935u,4278190336u),vec2<u32>(16711680u,4278190336u),vec2<u32>(16712191u,4278190336u),vec2<u32>(65280u,4278190336u),vec2<u32>(255u,4278190336u),vec2<u32>(0u,4278190336u),vec2<u32>(1u,4278190336u),vec2<u32>(256u,4278190336u),vec2<u32>(257u,4278190336u),vec2<u32>(131071u,4278190336u),vec2<u32>(65536u,4278190336u),vec2<u32>(65793u,4278190336u),vec2<u32>(33489152u,4278190336u),vec2<u32>(16842496u,4278190336u),vec2<u32>(16842497u,4278190336u),vec2<u32>(16777216u,4278190336u),vec2<u32>(16777727u,4278190336u),vec2<u32>(16908032u,4278190336u),vec2<u32>(16842753u,4278190336u),vec2<u32>(16843008u,4278190336u),vec2<u32>(4294902016u,4278190337u),vec2<u32>(4278190081u,4278190337u),vec2<u32>(4278255871u,4278190337u),vec2<u32>(4278255617u,4278190337u),vec2<u32>(16711935u,4278190337u),vec2<u32>(16711681u,4278190337u),vec2<u32>(16711936u,4278190337u),vec2<u32>(65535u,4278190337u),vec2<u32>(65281u,4278190337u),vec2<u32>(0u,4278190337u),vec2<u32>(511u,4278190337u),vec2<u32>(130816u,4278190337u),vec2<u32>(65537u,4278190337u),vec2<u32>(65792u,4278190337u),vec2<u32>(33488896u,4278190337u),vec2<u32>(16842496u,4278190337u),vec2<u32>(16777217u,4278190337u),vec2<u32>(16777473u,4278190337u),vec2<u32>(4294967295u,4278321151u),vec2<u32>(4294967041u,4278321151u),vec2<u32>(4294902271u,4278321151u),vec2<u32>(4294902017u,4278321151u),vec2<u32>(4278190080u,4278321151u),vec2<u32>(4278321151u,4278321151u),vec2<u32>(4278320897u,4278321151u),vec2<u32>(4278255616u,4278321151u),vec2<u32>(4278256127u,4278321151u),vec2<u32>(4278255873u,4278321151u),vec2<u32>(16711680u,4278321151u),vec2<u32>(65280u,4278321151u),vec2<u32>(256u,4278321151u),vec2<u32>(130816u,4278321151u),vec2<u32>(65536u,4278321151u),vec2<u32>(33554431u,4278321151u),vec2<u32>(33554177u,4278321151u),vec2<u32>(33489407u,4278321151u),vec2<u32>(33489153u,4278321151u),vec2<u32>(16777216u,4278321151u),vec2<u32>(16908287u,4278321151u),vec2<u32>(16908033u,4278321151u),vec2<u32>(16842752u,4278321151u),vec2<u32>(16843263u,4278321151u),vec2<u32>(16843009u,4278321151u),vec2<u32>(4294901760u,4278320896u),vec2<u32>(4278255360u,4278320896u),vec2<u32>(4278190335u,4278320896u),vec2<u32>(4278190336u,4278320896u),vec2<u32>(4278255616u,4278320896u),vec2<u32>(16776961u,4278320896u),vec2<u32>(16711935u,4278320896u),vec2<u32>(16711936u,4278320896u),vec2<u32>(0u,4278320896u),vec2<u32>(511u,4278320896u),vec2<u32>(257u,4278320896u),vec2<u32>(130816u,4278320896u),vec2<u32>(65791u,4278320896u),vec2<u32>(65536u,4278320896u),vec2<u32>(65537u,4278320896u),vec2<u32>(33488896u,4278320896u),vec2<u32>(16842751u,4278320896u),vec2<u32>(16777217u,4278320896u),vec2<u32>(16777472u,4278320896u),vec2<u32>(16842752u,4278320896u),vec2<u32>(4294967040u,4278320897u),vec2<u32>(4294902271u,4278320897u),vec2<u32>(4294902017u,4278320897u),vec2<u32>(4278255360u,4278320897u),vec2<u32>(4278190080u,4278320897u),vec2<u32>(4278321151u,4278320897u),vec2<u32>(4278320897u,4278320897u),vec2<u32>(4278256127u,4278320897u),vec2<u32>(4278255873u,4278320897u),vec2<u32>(16711680u,4278320897u),vec2<u32>(65280u,4278320897u),vec2<u32>(1u,4278320897u),vec2<u32>(256u,4278320897u),vec2<u32>(65536u,4278320897u),vec2<u32>(33554176u,4278320897u),vec2<u32>(33489407u,4278320897u),vec2<u32>(33489153u,4278320897u),vec2<u32>(16842496u,4278320897u),vec2<u32>(16777216u,4278320897u),vec2<u32>(16908287u,4278320897u),vec2<u32>(16908033u,4278320897u),vec2<u32>(16843263u,4278320897u),vec2<u32>(16843009u,4278320897u),vec2<u32>(4294901760u,4278255871u),vec2<u32>(4278190335u,4278255871u),vec2<u32>(4278190081u,4278255871u),vec2<u32>(4278190336u,4278255871u),vec2<u32>(4278255616u,4278255871u),vec2<u32>(16711935u,4278255871u),vec2<u32>(16711680u,4278255871u),vec2<u32>(16711681u,4278255871u),vec2<u32>(16711936u,4278255871u),vec2<u32>(65281u,4278255871u),vec2<u32>(0u,4278255871u),vec2<u32>(511u,4278255871u),vec2<u32>(257u,4278255871u),vec2<u32>(65537u,4278255871u),vec2<u32>(33488896u,4278255871u),vec2<u32>(16842496u,4278255871u),vec2<u32>(16777471u,4278255871u),vec2<u32>(16777472u,4278255871u),vec2<u32>(16908032u,4278255871u),vec2<u32>(16842752u,4278255871u),vec2<u32>(4294902016u,4278255616u),vec2<u32>(4278190080u,4278255616u),vec2<u32>(4278320896u,4278255616u),vec2<u32>(4278255872u,4278255616u),vec2<u32>(16777215u,4278255616u),vec2<u32>(16711680u,4278255616u),vec2<u32>(16712191u,4278255616u),vec2<u32>(65280u,4278255616u),vec2<u32>(255u,4278255616u),vec2<u32>(0u,4278255616u),vec2<u32>(256u,4278255616u),vec2<u32>(130817u,4278255616u),vec2<u32>(65536u,4278255616u),vec2<u32>(66047u,4278255616u),vec2<u32>(33489152u,4278255616u),vec2<u32>(16842751u,4278255616u),vec2<u32>(16777471u,4278255616u),vec2<u32>(16777216u,4278255616u),vec2<u32>(16777727u,4278255616u),vec2<u32>(16777473u,4278255616u),vec2<u32>(16908032u,4278255616u),vec2<u32>(16843007u,4278255616u),vec2<u32>(16842753u,4278255616u),vec2<u32>(16843008u,4278255616u),vec2<u32>(4294901760u,4278255617u),vec2<u32>(4278255615u,4278255617u),vec2<u32>(4278255361u,4278255617u),vec2<u32>(4278190336u,4278255617u),vec2<u32>(4278255616u,4278255617u),vec2<u32>(16776960u,4278255617u),vec2<u32>(16711936u,4278255617u),vec2<u32>(0u,4278255617u),vec2<u32>(131071u,4278255617u),vec2<u32>(130816u,4278255617u),vec2<u32>(65792u,4278255617u),vec2<u32>(33489151u,4278255617u),vec2<u32>(33488897u,4278255617u),vec2<u32>(16842751u,4278255617u),vec2<u32>(16777473u,4278255617u),vec2<u32>(4294967295u,4278256127u),vec2<u32>(4294967041u,4278256127u),vec2<u32>(4294902271u,4278256127u),vec2<u32>(4294902017u,4278256127u),vec2<u32>(4278190080u,4278256127u),vec2<u32>(4278321151u,4278256127u),vec2<u32>(4278320897u,4278256127u),vec2<u32>(4278256127u,4278256127u),vec2<u32>(4278255873u,4278256127u),vec2<u32>(16711680u,4278256127u),vec2<u32>(65280u,4278256127u),vec2<u32>(255u,4278256127u),vec2<u32>(65536u,4278256127u),vec2<u32>(33554431u,4278256127u),vec2<u32>(33554177u,4278256127u),vec2<u32>(33489407u,4278256127u),vec2<u32>(33489153u,4278256127u),vec2<u32>(16908287u,4278256127u),vec2<u32>(16908033u,4278256127u),vec2<u32>(16843263u,4278256127u),vec2<u32>(16843009u,4278256127u),vec2<u32>(4294902016u,4278255872u),vec2<u32>(4278255360u,4278255872u),vec2<u32>(4278190335u,4278255872u),vec2<u32>(4278190336u,4278255872u),vec2<u32>(4278255616u,4278255872u),vec2<u32>(16711681u,4278255872u),vec2<u32>(16711936u,4278255872u),vec2<u32>(65281u,4278255872u),vec2<u32>(0u,4278255872u),vec2<u32>(130816u,4278255872u),vec2<u32>(65791u,4278255872u),vec2<u32>(65537u,4278255872u),vec2<u32>(65792u,4278255872u),vec2<u32>(33488896u,4278255872u),vec2<u32>(16842751u,4278255872u),vec2<u32>(16777217u,4278255872u),vec2<u32>(16777472u,4278255872u),vec2<u32>(16843007u,4278255872u),vec2<u32>(16842752u,4278255872u),vec2<u32>(4294967295u,4278255873u),vec2<u32>(4294967041u,4278255873u),vec2<u32>(4294902271u,4278255873u),vec2<u32>(4294902017u,4278255873u),vec2<u32>(4278321151u,4278255873u),vec2<u32>(4278320897u,4278255873u),vec2<u32>(4278256127u,4278255873u),vec2<u32>(4278255873u,4278255873u),vec2<u32>(16711680u,4278255873u),vec2<u32>(65280u,4278255873u),vec2<u32>(1u,4278255873u),vec2<u32>(256u,4278255873u),vec2<u32>(65536u,4278255873u),vec2<u32>(33554431u,4278255873u),vec2<u32>(33554177u,4278255873u),vec2<u32>(33489407u,4278255873u),vec2<u32>(33489153u,4278255873u),vec2<u32>(16777216u,4278255873u),vec2<u32>(16908287u,4278255873u),vec2<u32>(16908033u,4278255873u),vec2<u32>(16843263u,4278255873u),vec2<u32>(16843009u,4278255873u),vec2<u32>(4294901760u,16777215u),vec2<u32>(4278255360u,16777215u),vec2<u32>(4278190081u,16777215u),vec2<u32>(4278255616u,16777215u),vec2<u32>(16711936u,16777215u),vec2<u32>(65281u,16777215u),vec2<u32>(0u,16777215u),vec2<u32>(511u,16777215u),vec2<u32>(257u,16777215u),vec2<u32>(130816u,16777215u),vec2<u32>(65791u,16777215u),vec2<u32>(65537u,16777215u),vec2<u32>(16777471u,16777215u),vec2<u32>(16777472u,16777215u),vec2<u32>(16908032u,16777215u),vec2<u32>(16842753u,16777215u),vec2<u32>(4294967295u,16776960u),vec2<u32>(4294967040u,16776960u),vec2<u32>(4294902015u,16776960u),vec2<u32>(4294901761u,16776960u),vec2<u32>(4294902016u,16776960u),vec2<u32>(4278255361u,16776960u),vec2<u32>(4278190080u,16776960u),vec2<u32>(4278190081u,16776960u),vec2<u32>(4278190591u,16776960u),vec2<u32>(4278190337u,16776960u),vec2<u32>(4278320896u,16776960u),vec2<u32>(4278255617u,16776960u),vec2<u32>(4278255872u,16776960u),vec2<u32>(16711680u,16776960u),vec2<u32>(16712191u,16776960u),vec2<u32>(16711937u,16776960u),vec2<u32>(65280u,16776960u),vec2<u32>(255u,16776960u),vec2<u32>(0u,16776960u),vec2<u32>(1u,16776960u),vec2<u32>(256u,16776960u),vec2<u32>(257u,16776960u),vec2<u32>(65536u,16776960u),vec2<u32>(66047u,16776960u),vec2<u32>(65793u,16776960u),vec2<u32>(33554176u,16776960u),vec2<u32>(33489151u,16776960u),vec2<u32>(33488897u,16776960u),vec2<u32>(16842751u,16776960u),vec2<u32>(16842497u,16776960u),vec2<u32>(16777216u,16776960u),vec2<u32>(16908287u,16776960u),vec2<u32>(16908032u,16776960u),vec2<u32>(16908033u,16776960u),vec2<u32>(4294901760u,16776961u),vec2<u32>(4278255360u,16776961u),vec2<u32>(4278190335u,16776961u),vec2<u32>(4278190081u,16776961u),vec2<u32>(4278255616u,16776961u),vec2<u32>(16776960u,16776961u),vec2<u32>(65281u,16776961u),vec2<u32>(0u,16776961u),vec2<u32>(257u,16776961u),vec2<u32>(65791u,16776961u),vec2<u32>(65792u,16776961u),vec2<u32>(33489152u,16776961u),vec2<u32>(16777471u,16776961u),vec2<u32>(16842752u,16776961u),vec2<u32>(4294967040u,16711935u),vec2<u32>(4278190080u,16711935u),vec2<u32>(4278190336u,16711935u),vec2<u32>(4278255872u,16711935u),vec2<u32>(16711680u,16711935u),vec2<u32>(16712191u,16711935u),vec2<u32>(16711937u,16711935u),vec2<u32>(65280u,16711935u),vec2<u32>(255u,16711935u),vec2<u32>(0u,16711935u),vec2<u32>(1u,16711935u),vec2<u32>(130816u,16711935u),vec2<u32>(130817u,16711935u),vec2<u32>(65536u,16711935u),vec2<u32>(66047u,16711935u),vec2<u32>(65793u,16711935u),vec2<u32>(33554176u,16711935u),vec2<u32>(33488897u,16711935u),vec2<u32>(33489152u,16711935u),vec2<u32>(16842751u,16711935u),vec2<u32>(16842497u,16711935u),vec2<u32>(16777216u,16711935u),vec2<u32>(16908287u,16711935u),vec2<u32>(16908032u,16711935u),vec2<u32>(16843008u,16711935u),vec2<u32>(4294967040u,16711680u),vec2<u32>(4294967041u,16711680u),vec2<u32>(4294901760u,16711680u),vec2<u32>(4294902017u,16711680u),vec2<u32>(4278255360u,16711680u),vec2<u32>(4278190335u,16711680u),vec2<u32>(4278190080u,16711680u),vec2<u32>(4278190081u,16711680u),vec2<u32>(4278190336u,16711680u),vec2<u32>(4278321151u,16711680u),vec2<u32>(4278255616u,16711680u),vec2<u32>(4278255873u,16711680u),vec2<u32>(16776960u,16711680u),vec2<u32>(16711935u,16711680u),vec2<u32>(16711680u,16711680u),vec2<u32>(16711681u,16711680u),vec2<u32>(16711936u,16711680u),vec2<u32>(65535u,16711680u),vec2<u32>(65280u,16711680u),vec2<u32>(255u,16711680u),vec2<u32>(0u,16711680u),vec2<u32>(1u,16711680u),vec2<u32>(511u,16711680u),vec2<u32>(256u,16711680u),vec2<u32>(130816u,16711680u),vec2<u32>(65791u,16711680u),vec2<u32>(65536u,16711680u),vec2<u32>(65537u,16711680u),vec2<u32>(65792u,16711680u),vec2<u32>(33554177u,16711680u),vec2<u32>(33489151u,16711680u),vec2<u32>(33488896u,16711680u),vec2<u32>(33489407u,16711680u),vec2<u32>(16842496u,16711680u),vec2<u32>(16777471u,16711680u),vec2<u32>(16777216u,16711680u),vec2<u32>(16777217u,16711680u),vec2<u32>(16777472u,16711680u),vec2<u32>(16777473u,16711680u),vec2<u32>(16842752u,16711680u),vec2<u32>(16843263u,16711680u),vec2<u32>(16843009u,16711680u),vec2<u32>(4294967040u,16711681u),vec2<u32>(4294901760u,16711681u),vec2<u32>(4294902016u,16711681u),vec2<u32>(4278190335u,16711681u),vec2<u32>(4278190080u,16711681u),vec2<u32>(4278190591u,16711681u),vec2<u32>(4278190337u,16711681u),vec2<u32>(4278320896u,16711681u),vec2<u32>(4278255871u,16711681u),vec2<u32>(4278255872u,16711681u),vec2<u32>(16777215u,16711681u),vec2<u32>(16776961u,16711681u),vec2<u32>(16711680u,16711681u),vec2<u32>(16712191u,16711681u),vec2<u32>(65535u,16711681u),vec2<u32>(65280u,16711681u),vec2<u32>(65281u,16711681u),vec2<u32>(0u,16711681u),vec2<u32>(1u,16711681u),vec2<u32>(256u,16711681u),vec2<u32>(130817u,16711681u),vec2<u32>(65536u,16711681u),vec2<u32>(66047u,16711681u),vec2<u32>(33554176u,16711681u),vec2<u32>(33488896u,16711681u),vec2<u32>(33489153u,16711681u),vec2<u32>(16777471u,16711681u),vec2<u32>(16777216u,16711681u),vec2<u32>(16908032u,16711681u),vec2<u32>(16843007u,16711681u),vec2<u32>(16842753u,16711681u),vec2<u32>(4294901760u,16712191u),vec2<u32>(4278255360u,16712191u),vec2<u32>(4278190080u,16712191u),vec2<u32>(4278190337u,16712191u),vec2<u32>(4278255616u,16712191u),vec2<u32>(16776961u,16712191u),vec2<u32>(16711936u,16712191u),vec2<u32>(65535u,16712191u),vec2<u32>(0u,16712191u),vec2<u32>(511u,16712191u),vec2<u32>(130816u,16712191u),vec2<u32>(65791u,16712191u),vec2<u32>(65537u,16712191u),vec2<u32>(65792u,16712191u),vec2<u32>(33488896u,16712191u),vec2<u32>(16842496u,16712191u),vec2<u32>(16777471u,16712191u),vec2<u32>(16777217u,16712191u),vec2<u32>(16777472u,16712191u),vec2<u32>(16842752u,16712191u),vec2<u32>(4294967040u,16711936u),vec2<u32>(4294901760u,16711936u),vec2<u32>(4294901761u,16711936u),vec2<u32>(4294902017u,16711936u),vec2<u32>(4278255615u,16711936u),vec2<u32>(4278190335u,16711936u),vec2<u32>(4278190080u,16711936u),vec2<u32>(4278190591u,16711936u),vec2<u32>(4278320896u,16711936u),vec2<u32>(4278255871u,16711936u),vec2<u32>(4278255617u,16711936u),vec2<u32>(16777215u,16711936u),vec2<u32>(16711680u,16711936u),vec2<u32>(16711937u,16711936u),vec2<u32>(65280u,16711936u),vec2<u32>(65281u,16711936u),vec2<u32>(255u,16711936u),vec2<u32>(0u,16711936u),vec2<u32>(1u,16711936u),vec2<u32>(256u,16711936u),vec2<u32>(131071u,16711936u),vec2<u32>(130817u,16711936u),vec2<u32>(65536u,16711936u),vec2<u32>(65537u,16711936u),vec2<u32>(65793u,16711936u),vec2<u32>(33488897u,16711936u),vec2<u32>(33489152u,16711936u),vec2<u32>(16842497u,16711936u),vec2<u32>(16777216u,16711936u),vec2<u32>(16777217u,16711936u),vec2<u32>(16777727u,16711936u),vec2<u32>(16908032u,16711936u),vec2<u32>(16843007u,16711936u),vec2<u32>(16842753u,16711936u),vec2<u32>(16843008u,16711936u),vec2<u32>(4278190081u,16711937u),vec2<u32>(16711935u,16711937u),vec2<u32>(16711681u,16711937u),vec2<u32>(16711936u,16711937u),vec2<u32>(0u,16711937u),vec2<u32>(511u,16711937u),vec2<u32>(257u,16711937u),vec2<u32>(65791u,16711937u),vec2<u32>(65792u,16711937u),vec2<u32>(16777471u,16711937u),vec2<u32>(16842752u,16711937u),vec2<u32>(4294967040u,65535u),vec2<u32>(4294902015u,65535u),vec2<u32>(4294901760u,65535u),vec2<u32>(4294901761u,65535u),vec2<u32>(4294902016u,65535u),vec2<u32>(4278255361u,65535u),vec2<u32>(4278190080u,65535u),vec2<u32>(4278190337u,65535u),vec2<u32>(4278320896u,65535u),vec2<u32>(4278255871u,65535u),vec2<u32>(4278255872u,65535u),vec2<u32>(16777215u,65535u),vec2<u32>(16711680u,65535u),vec2<u32>(16712191u,65535u),vec2<u32>(65280u,65535u),vec2<u32>(255u,65535u),vec2<u32>(0u,65535u),vec2<u32>(1u,65535u),vec2<u32>(256u,65535u),vec2<u32>(65536u,65535u),vec2<u32>(66047u,65535u),vec2<u32>(33488897u,65535u),vec2<u32>(33489152u,65535u),vec2<u32>(16777216u,65535u),vec2<u32>(16777727u,65535u),vec2<u32>(16908287u,65535u),vec2<u32>(16908032u,65535u),vec2<u32>(16842753u,65535u),vec2<u32>(16843008u,65535u),vec2<u32>(4294901760u,65280u),vec2<u32>(4294902271u,65280u),vec2<u32>(4294902016u,65280u),vec2<u32>(4294902017u,65280u),vec2<u32>(4278255360u,65280u),vec2<u32>(4278190335u,65280u),vec2<u32>(4278190080u,65280u),vec2<u32>(4278190081u,65280u),vec2<u32>(4278190591u,65280u),vec2<u32>(4278190336u,65280u),vec2<u32>(4278321151u,65280u),vec2<u32>(4278255616u,65280u),vec2<u32>(4278255617u,65280u),vec2<u32>(4278256127u,65280u),vec2<u32>(4278255873u,65280u),vec2<u32>(16776960u,65280u),vec2<u32>(16711935u,65280u),vec2<u32>(16711680u,65280u),vec2<u32>(16711681u,65280u),vec2<u32>(16711936u,65280u),vec2<u32>(65535u,65280u),vec2<u32>(65280u,65280u),vec2<u32>(65281u,65280u),vec2<u32>(255u,65280u),vec2<u32>(0u,65280u),vec2<u32>(1u,65280u),vec2<u32>(511u,65280u),vec2<u32>(256u,65280u),vec2<u32>(257u,65280u),vec2<u32>(130816u,65280u),vec2<u32>(65791u,65280u),vec2<u32>(65536u,65280u),vec2<u32>(65537u,65280u),vec2<u32>(65792u,65280u),vec2<u32>(33554177u,65280u),vec2<u32>(33488896u,65280u),vec2<u32>(16842496u,65280u),vec2<u32>(16777471u,65280u),vec2<u32>(16777216u,65280u),vec2<u32>(16777217u,65280u),vec2<u32>(16777472u,65280u),vec2<u32>(16908287u,65280u),vec2<u32>(16842752u,65280u),vec2<u32>(16843009u,65280u),vec2<u32>(4294967040u,65281u),vec2<u32>(4294901761u,65281u),vec2<u32>(4278255361u,65281u),vec2<u32>(4278190080u,65281u),vec2<u32>(4278190337u,65281u),vec2<u32>(4278320896u,65281u),vec2<u32>(4278255871u,65281u),vec2<u32>(16776961u,65281u),vec2<u32>(16711680u,65281u),vec2<u32>(16711937u,65281u),vec2<u32>(65280u,65281u),vec2<u32>(255u,65281u),vec2<u32>(0u,65281u),vec2<u32>(1u,65281u),vec2<u32>(256u,65281u),vec2<u32>(130817u,65281u),vec2<u32>(65536u,65281u),vec2<u32>(33488896u,65281u),vec2<u32>(16842751u,65281u),vec2<u32>(16842497u,65281u),vec2<u32>(16777216u,65281u),vec2<u32>(16777472u,65281u),vec2<u32>(16777473u,65281u),vec2<u32>(16843007u,65281u),vec2<u32>(4294902015u,255u),vec2<u32>(4294901760u,255u),vec2<u32>(4278255360u,255u),vec2<u32>(4278190335u,255u),vec2<u32>(4278190080u,255u),vec2<u32>(4278190081u,255u),vec2<u32>(4278190591u,255u),vec2<u32>(4278190336u,255u),vec2<u32>(4278320896u,255u),vec2<u32>(4278255616u,255u),vec2<u32>(4278256127u,255u),vec2<u32>(4278255873u,255u),vec2<u32>(16776960u,255u),vec2<u32>(16711935u,255u),vec2<u32>(16711680u,255u),vec2<u32>(16711681u,255u),vec2<u32>(16711936u,255u),vec2<u32>(16711937u,255u),vec2<u32>(65535u,255u),vec2<u32>(65280u,255u),vec2<u32>(255u,255u),vec2<u32>(0u,255u),vec2<u32>(1u,255u),vec2<u32>(511u,255u),vec2<u32>(256u,255u),vec2<u32>(257u,255u),vec2<u32>(130816u,255u),vec2<u32>(130817u,255u),vec2<u32>(65791u,255u),vec2<u32>(65536u,255u),vec2<u32>(65537u,255u),vec2<u32>(65792u,255u),vec2<u32>(33554431u,255u),vec2<u32>(33489407u,255u),vec2<u32>(33489153u,255u),vec2<u32>(16842496u,255u),vec2<u32>(16777471u,255u),vec2<u32>(16777216u,255u),vec2<u32>(16777217u,255u),vec2<u32>(16777472u,255u),vec2<u32>(16908032u,255u),vec2<u32>(16843007u,255u),vec2<u32>(16842752u,255u),vec2<u32>(16843009u,255u),vec2<u32>(4294967040u,0u),vec2<u32>(4294967041u,0u),vec2<u32>(4294902015u,0u),vec2<u32>(4294901760u,0u),vec2<u32>(4294901761u,0u),vec2<u32>(4294902016u,0u),vec2<u32>(4278255615u,0u),vec2<u32>(4278255360u,0u),vec2<u32>(4278255361u,0u),vec2<u32>(4278190335u,0u),vec2<u32>(4278190080u,0u),vec2<u32>(4278190081u,0u),vec2<u32>(4278190336u,0u),vec2<u32>(4278190337u,0u),vec2<u32>(4278320896u,0u),vec2<u32>(4278255871u,0u),vec2<u32>(4278255616u,0u),vec2<u32>(4278255617u,0u),vec2<u32>(4278255872u,0u),vec2<u32>(16777215u,0u),vec2<u32>(16776960u,0u),vec2<u32>(16776961u,0u),vec2<u32>(16711935u,0u),vec2<u32>(16711680u,0u),vec2<u32>(16711681u,0u),vec2<u32>(16712191u,0u),vec2<u32>(16711936u,0u),vec2<u32>(65535u,0u),vec2<u32>(65280u,0u),vec2<u32>(65281u,0u),vec2<u32>(255u,0u),vec2<u32>(0u,0u),vec2<u32>(1u,0u),vec2<u32>(511u,0u),vec2<u32>(256u,0u),vec2<u32>(257u,0u),vec2<u32>(131071u,0u),vec2<u32>(130816u,0u),vec2<u32>(65791u,0u),vec2<u32>(65536u,0u),vec2<u32>(65537u,0u),vec2<u32>(66047u,0u),vec2<u32>(65792u,0u),vec2<u32>(65793u,0u),vec2<u32>(33554176u,0u),vec2<u32>(33489151u,0u),vec2<u32>(33488896u,0u),vec2<u32>(33489152u,0u),vec2<u32>(33489153u,0u),vec2<u32>(16842751u,0u),vec2<u32>(16842496u,0u),vec2<u32>(16777471u,0u),vec2<u32>(16777216u,0u),vec2<u32>(16777217u,0u),vec2<u32>(16777727u,0u),vec2<u32>(16777472u,0u),vec2<u32>(16908032u,0u),vec2<u32>(16843007u,0u),vec2<u32>(16842752u,0u),vec2<u32>(16842753u,0u),vec2<u32>(16843008u,0u),vec2<u32>(4294967295u,1u),vec2<u32>(4294967040u,1u),vec2<u32>(4294967041u,1u),vec2<u32>(4294902015u,1u),vec2<u32>(4294901761u,1u),vec2<u32>(4294902271u,1u),vec2<u32>(4294902016u,1u),vec2<u32>(4278255360u,1u),vec2<u32>(4278190335u,1u),vec2<u32>(4278190080u,1u),vec2<u32>(4278190591u,1u),vec2<u32>(4278190336u,1u),vec2<u32>(4278321151u,1u),vec2<u32>(4278320896u,1u),vec2<u32>(4278320897u,1u),vec2<u32>(4278255871u,1u),vec2<u32>(4278255616u,1u),vec2<u32>(4278255617u,1u),vec2<u32>(4278256127u,1u),vec2<u32>(4278255872u,1u),vec2<u32>(16776960u,1u),vec2<u32>(16711680u,1u),vec2<u32>(16711681u,1u),vec2<u32>(16712191u,1u),vec2<u32>(16711936u,1u),vec2<u32>(16711937u,1u),vec2<u32>(65535u,1u),vec2<u32>(65280u,1u),vec2<u32>(65281u,1u),vec2<u32>(255u,1u),vec2<u32>(0u,1u),vec2<u32>(1u,1u),vec2<u32>(511u,1u),vec2<u32>(256u,1u),vec2<u32>(257u,1u),vec2<u32>(130816u,1u),vec2<u32>(65791u,1u),vec2<u32>(65536u,1u),vec2<u32>(65792u,1u),vec2<u32>(33554177u,1u),vec2<u32>(33488896u,1u),vec2<u32>(33488897u,1u),vec2<u32>(33489407u,1u),vec2<u32>(33489152u,1u),vec2<u32>(33489153u,1u),vec2<u32>(16842496u,1u),vec2<u32>(16777216u,1u),vec2<u32>(16777473u,1u),vec2<u32>(16908033u,1u),vec2<u32>(16842752u,1u),vec2<u32>(16842753u,1u),vec2<u32>(16843263u,1u),vec2<u32>(16843008u,1u),vec2<u32>(4294902015u,511u),vec2<u32>(4294901760u,511u),vec2<u32>(4294901761u,511u),vec2<u32>(4294902016u,511u),vec2<u32>(4278255615u,511u),vec2<u32>(4278190080u,511u),vec2<u32>(4278190591u,511u),vec2<u32>(4278320896u,511u),vec2<u32>(4278255873u,511u),vec2<u32>(16711680u,511u),vec2<u32>(16712191u,511u),vec2<u32>(16711937u,511u),vec2<u32>(65280u,511u),vec2<u32>(255u,511u),vec2<u32>(0u,511u),vec2<u32>(1u,511u),vec2<u32>(511u,511u),vec2<u32>(256u,511u),vec2<u32>(131071u,511u),vec2<u32>(130817u,511u),vec2<u32>(65791u,511u),vec2<u32>(65536u,511u),vec2<u32>(33554177u,511u),vec2<u32>(33489152u,511u),vec2<u32>(16842751u,511u),vec2<u32>(16842497u,511u),vec2<u32>(16777216u,511u),vec2<u32>(16777727u,511u),vec2<u32>(16908032u,511u),vec2<u32>(16843008u,511u),vec2<u32>(4294967040u,256u),vec2<u32>(4294967041u,256u),vec2<u32>(4294901760u,256u),vec2<u32>(4294902017u,256u),vec2<u32>(4278255360u,256u),vec2<u32>(4278190335u,256u),vec2<u32>(4278190080u,256u),vec2<u32>(4278190081u,256u),vec2<u32>(4278190336u,256u),vec2<u32>(4278255616u,256u),vec2<u32>(16776960u,256u),vec2<u32>(16711935u,256u),vec2<u32>(16711680u,256u),vec2<u32>(16711681u,256u),vec2<u32>(16711936u,256u),vec2<u32>(65535u,256u),vec2<u32>(65280u,256u),vec2<u32>(65281u,256u),vec2<u32>(255u,256u),vec2<u32>(0u,256u),vec2<u32>(1u,256u),vec2<u32>(511u,256u),vec2<u32>(256u,256u),vec2<u32>(257u,256u),vec2<u32>(130816u,256u),vec2<u32>(65791u,256u),vec2<u32>(65536u,256u),vec2<u32>(65537u,256u),vec2<u32>(65792u,256u),vec2<u32>(33554176u,256u),vec2<u32>(33488896u,256u),vec2<u32>(33489152u,256u),vec2<u32>(16842496u,256u),vec2<u32>(16777471u,256u),vec2<u32>(16777216u,256u),vec2<u32>(16777217u,256u),vec2<u32>(16777727u,256u),vec2<u32>(16777472u,256u),vec2<u32>(16842752u,256u),vec2<u32>(4294902015u,257u),vec2<u32>(4294902271u,257u),vec2<u32>(4278190080u,257u),vec2<u32>(4278190337u,257u),vec2<u32>(4278321151u,257u),vec2<u32>(4278255616u,257u),vec2<u32>(4278255617u,257u),vec2<u32>(4278255872u,257u),vec2<u32>(16711680u,257u),vec2<u32>(16712191u,257u),vec2<u32>(16711936u,257u),vec2<u32>(65280u,257u),vec2<u32>(0u,257u),vec2<u32>(1u,257u),vec2<u32>(511u,257u),vec2<u32>(256u,257u),vec2<u32>(130817u,257u),vec2<u32>(65536u,257u),vec2<u32>(66047u,257u),vec2<u32>(65793u,257u),vec2<u32>(33554176u,257u),vec2<u32>(33489153u,257u),vec2<u32>(16842497u,257u),vec2<u32>(16777216u,257u),vec2<u32>(16777217u,257u),vec2<u32>(16777727u,257u),vec2<u32>(16777473u,257u),vec2<u32>(16908032u,257u),vec2<u32>(4294901760u,131071u),vec2<u32>(4278190335u,131071u),vec2<u32>(4278190081u,131071u),vec2<u32>(4278190336u,131071u),vec2<u32>(4278255616u,131071u),vec2<u32>(16711935u,131071u),vec2<u32>(65535u,131071u),vec2<u32>(0u,131071u),vec2<u32>(1u,131071u),vec2<u32>(511u,131071u),vec2<u32>(257u,131071u),vec2<u32>(130816u,131071u),vec2<u32>(65791u,131071u),vec2<u32>(65537u,131071u),vec2<u32>(65792u,131071u),vec2<u32>(33554176u,131071u),vec2<u32>(16777217u,131071u),vec2<u32>(16842752u,131071u),vec2<u32>(4294967040u,130816u),vec2<u32>(4294902015u,130816u),vec2<u32>(4294901761u,130816u),vec2<u32>(4294902016u,130816u),vec2<u32>(4278255361u,130816u),vec2<u32>(4278190080u,130816u),vec2<u32>(4278320896u,130816u),vec2<u32>(4278320897u,130816u),vec2<u32>(4278255617u,130816u),vec2<u32>(4278255872u,130816u),vec2<u32>(16711680u,130816u),vec2<u32>(16711936u,130816u),vec2<u32>(65280u,130816u),vec2<u32>(0u,130816u),vec2<u32>(1u,130816u),vec2<u32>(256u,130816u),vec2<u32>(65536u,130816u),vec2<u32>(65537u,130816u),vec2<u32>(65793u,130816u),vec2<u32>(33489151u,130816u),vec2<u32>(33489153u,130816u),vec2<u32>(16842497u,130816u),vec2<u32>(16777216u,130816u),vec2<u32>(16908032u,130816u),vec2<u32>(16842753u,130816u),vec2<u32>(16843008u,130816u),vec2<u32>(4278255360u,130817u),vec2<u32>(4278190081u,130817u),vec2<u32>(4278190336u,130817u),vec2<u32>(16777215u,130817u),vec2<u32>(16776960u,130817u),vec2<u32>(16711681u,130817u),vec2<u32>(0u,130817u),vec2<u32>(1u,130817u),vec2<u32>(511u,130817u),vec2<u32>(131071u,130817u),vec2<u32>(33488896u,130817u),vec2<u32>(16842496u,130817u),vec2<u32>(16777217u,130817u),vec2<u32>(16842752u,130817u),vec2<u32>(4278255360u,65791u),vec2<u32>(4278255361u,65791u),vec2<u32>(4278190080u,65791u),vec2<u32>(4278190081u,65791u),vec2<u32>(4278190337u,65791u),vec2<u32>(4278320896u,65791u),vec2<u32>(4278255617u,65791u),vec2<u32>(4278255872u,65791u),vec2<u32>(16777215u,65791u),vec2<u32>(16776961u,65791u),vec2<u32>(16711680u,65791u),vec2<u32>(16712191u,65791u),vec2<u32>(16711937u,65791u),vec2<u32>(65280u,65791u),vec2<u32>(255u,65791u),vec2<u32>(0u,65791u),vec2<u32>(1u,65791u),vec2<u32>(256u,65791u),vec2<u32>(257u,65791u),vec2<u32>(131071u,65791u),vec2<u32>(130817u,65791u),vec2<u32>(65536u,65791u),vec2<u32>(33489151u,65791u),vec2<u32>(33488896u,65791u),vec2<u32>(33489152u,65791u),vec2<u32>(16842751u,65791u),vec2<u32>(16842497u,65791u),vec2<u32>(16777471u,65791u),vec2<u32>(16777216u,65791u),vec2<u32>(16777217u,65791u),vec2<u32>(16777727u,65791u),vec2<u32>(16777473u,65791u),vec2<u32>(16908032u,65791u),vec2<u32>(16843007u,65791u),vec2<u32>(16843008u,65791u),vec2<u32>(4294901760u,65536u),vec2<u32>(4294902271u,65536u),vec2<u32>(4294902017u,65536u),vec2<u32>(4278255360u,65536u),vec2<u32>(4278190080u,65536u),vec2<u32>(4278190081u,65536u),vec2<u32>(4278190336u,65536u),vec2<u32>(16711935u,65536u),vec2<u32>(16711680u,65536u),vec2<u32>(16711681u,65536u),vec2<u32>(16711936u,65536u),vec2<u32>(65535u,65536u),vec2<u32>(65280u,65536u),vec2<u32>(255u,65536u),vec2<u32>(0u,65536u),vec2<u32>(1u,65536u),vec2<u32>(256u,65536u),vec2<u32>(130816u,65536u),vec2<u32>(65791u,65536u),vec2<u32>(65536u,65536u),vec2<u32>(65537u,65536u),vec2<u32>(65792u,65536u),vec2<u32>(33488897u,65536u),vec2<u32>(33489152u,65536u),vec2<u32>(33489153u,65536u),vec2<u32>(16842496u,65536u),vec2<u32>(16777216u,65536u),vec2<u32>(16777217u,65536u),vec2<u32>(16777472u,65536u),vec2<u32>(16777473u,65536u),vec2<u32>(16908033u,65536u),vec2<u32>(16842752u,65536u),vec2<u32>(16842753u,65536u),vec2<u32>(16843263u,65536u),vec2<u32>(4294967041u,65537u),vec2<u32>(4294902016u,65537u),vec2<u32>(4278190080u,65537u),vec2<u32>(4278321151u,65537u),vec2<u32>(4278255617u,65537u),vec2<u32>(4278256127u,65537u),vec2<u32>(4278255872u,65537u),vec2<u32>(16777215u,65537u),vec2<u32>(16711680u,65537u),vec2<u32>(16712191u,65537u),vec2<u32>(16711937u,65537u),vec2<u32>(65280u,65537u),vec2<u32>(255u,65537u),vec2<u32>(0u,65537u),vec2<u32>(1u,65537u),vec2<u32>(511u,65537u),vec2<u32>(257u,65537u),vec2<u32>(131071u,65537u),vec2<u32>(65536u,65537u),vec2<u32>(66047u,65537u),vec2<u32>(33554431u,65537u),vec2<u32>(33554177u,65537u),vec2<u32>(33488896u,65537u),vec2<u32>(33489153u,65537u),vec2<u32>(16777471u,65537u),vec2<u32>(16777217u,65537u),vec2<u32>(16777727u,65537u),vec2<u32>(16777472u,65537u),vec2<u32>(16908287u,65537u),vec2<u32>(16843007u,65537u),vec2<u32>(16842753u,65537u),vec2<u32>(16843009u,65537u),vec2<u32>(4278190081u,66047u),vec2<u32>(4278190336u,66047u),vec2<u32>(4278255616u,66047u),vec2<u32>(16776960u,66047u),vec2<u32>(65281u,66047u),vec2<u32>(0u,66047u),vec2<u32>(257u,66047u),vec2<u32>(130816u,66047u),vec2<u32>(65792u,66047u),vec2<u32>(33488896u,66047u),vec2<u32>(16842496u,66047u),vec2<u32>(16777727u,66047u),vec2<u32>(16842753u,66047u),vec2<u32>(4294967040u,65792u),vec2<u32>(4294902015u,65792u),vec2<u32>(4278255615u,65792u),vec2<u32>(4278190080u,65792u),vec2<u32>(4278320896u,65792u),vec2<u32>(4278255871u,65792u),vec2<u32>(4278255617u,65792u),vec2<u32>(4278255872u,65792u),vec2<u32>(16777215u,65792u),vec2<u32>(16776960u,65792u),vec2<u32>(16711680u,65792u),vec2<u32>(16711681u,65792u),vec2<u32>(16712191u,65792u),vec2<u32>(65280u,65792u),vec2<u32>(255u,65792u),vec2<u32>(0u,65792u),vec2<u32>(1u,65792u),vec2<u32>(256u,65792u),vec2<u32>(131071u,65792u),vec2<u32>(65536u,65792u),vec2<u32>(65793u,65792u),vec2<u32>(33554177u,65792u),vec2<u32>(33489151u,65792u),vec2<u32>(33489153u,65792u),vec2<u32>(16777216u,65792u),vec2<u32>(16908032u,65792u),vec2<u32>(16843007u,65792u),vec2<u32>(16842752u,65792u),vec2<u32>(16843008u,65792u),vec2<u32>(4278255360u,65793u),vec2<u32>(4278190081u,65793u),vec2<u32>(4278190591u,65793u),vec2<u32>(16776960u,65793u),vec2<u32>(16711935u,65793u),vec2<u32>(16711936u,65793u),vec2<u32>(65535u,65793u),vec2<u32>(0u,65793u),vec2<u32>(511u,65793u),vec2<u32>(257u,65793u),vec2<u32>(65791u,65793u),vec2<u32>(65536u,65793u),vec2<u32>(65792u,65793u),vec2<u32>(33488897u,65793u),vec2<u32>(16777471u,65793u),vec2<u32>(16777727u,65793u),vec2<u32>(16777473u,65793u),vec2<u32>(16842753u,65793u),vec2<u32>(4294967295u,33554431u),vec2<u32>(4294967041u,33554431u),vec2<u32>(4294902271u,33554431u),vec2<u32>(4294902017u,33554431u),vec2<u32>(4278321151u,33554431u),vec2<u32>(4278320897u,33554431u),vec2<u32>(4278256127u,33554431u),vec2<u32>(4278255873u,33554431u),vec2<u32>(16711680u,33554431u),vec2<u32>(65535u,33554431u),vec2<u32>(65280u,33554431u),vec2<u32>(255u,33554431u),vec2<u32>(1u,33554431u),vec2<u32>(256u,33554431u),vec2<u32>(65536u,33554431u),vec2<u32>(33554431u,33554431u),vec2<u32>(33554177u,33554431u),vec2<u32>(33489407u,33554431u),vec2<u32>(33489153u,33554431u),vec2<u32>(16777216u,33554431u),vec2<u32>(16908287u,33554431u),vec2<u32>(16908033u,33554431u),vec2<u32>(16843263u,33554431u),vec2<u32>(16843009u,33554431u),vec2<u32>(4294901760u,33554176u),vec2<u32>(4278255360u,33554176u),vec2<u32>(4278190335u,33554176u),vec2<u32>(4278190081u,33554176u),vec2<u32>(4278190336u,33554176u),vec2<u32>(4278255616u,33554176u),vec2<u32>(16776960u,33554176u),vec2<u32>(16711935u,33554176u),vec2<u32>(16711936u,33554176u),vec2<u32>(65535u,33554176u),vec2<u32>(65281u,33554176u),vec2<u32>(0u,33554176u),vec2<u32>(1u,33554176u),vec2<u32>(511u,33554176u),vec2<u32>(256u,33554176u),vec2<u32>(65791u,33554176u),vec2<u32>(65537u,33554176u),vec2<u32>(65792u,33554176u),vec2<u32>(33488896u,33554176u),vec2<u32>(33489152u,33554176u),vec2<u32>(16777471u,33554176u),vec2<u32>(16777217u,33554176u),vec2<u32>(16777472u,33554176u),vec2<u32>(16842752u,33554176u),vec2<u32>(4294967295u,33554177u),vec2<u32>(4294967041u,33554177u),vec2<u32>(4294902271u,33554177u),vec2<u32>(4294902017u,33554177u),vec2<u32>(4278190080u,33554177u),vec2<u32>(4278321151u,33554177u),vec2<u32>(4278320897u,33554177u),vec2<u32>(4278256127u,33554177u),vec2<u32>(4278255873u,33554177u),vec2<u32>(65280u,33554177u),vec2<u32>(255u,33554177u),vec2<u32>(256u,33554177u),vec2<u32>(65536u,33554177u),vec2<u32>(33554431u,33554177u),vec2<u32>(33554177u,33554177u),vec2<u32>(33489407u,33554177u),vec2<u32>(33489153u,33554177u),vec2<u32>(16777216u,33554177u),vec2<u32>(16908287u,33554177u),vec2<u32>(16908033u,33554177u),vec2<u32>(16843263u,33554177u),vec2<u32>(16843009u,33554177u),vec2<u32>(4278190335u,33489151u),vec2<u32>(4278190336u,33489151u),vec2<u32>(16776960u,33489151u),vec2<u32>(16711935u,33489151u),vec2<u32>(65280u,33489151u),vec2<u32>(0u,33489151u),vec2<u32>(257u,33489151u),vec2<u32>(130816u,33489151u),vec2<u32>(65791u,33489151u),vec2<u32>(65792u,33489151u),vec2<u32>(16777471u,33489151u),vec2<u32>(16777472u,33489151u),vec2<u32>(4294967040u,33488896u),vec2<u32>(4294902016u,33488896u),vec2<u32>(4278255361u,33488896u),vec2<u32>(4278190080u,33488896u),vec2<u32>(4278190337u,33488896u),vec2<u32>(4278255617u,33488896u),vec2<u32>(4278255872u,33488896u),vec2<u32>(16777215u,33488896u),vec2<u32>(16776960u,33488896u),vec2<u32>(16711680u,33488896u),vec2<u32>(16712191u,33488896u),vec2<u32>(65280u,33488896u),vec2<u32>(255u,33488896u),vec2<u32>(0u,33488896u),vec2<u32>(1u,33488896u),vec2<u32>(256u,33488896u),vec2<u32>(257u,33488896u),vec2<u32>(65536u,33488896u),vec2<u32>(65537u,33488896u),vec2<u32>(66047u,33488896u),vec2<u32>(65793u,33488896u),vec2<u32>(33554176u,33488896u),vec2<u32>(33489151u,33488896u),vec2<u32>(33488897u,33488896u),vec2<u32>(33489152u,33488896u),vec2<u32>(16842751u,33488896u),vec2<u32>(16842497u,33488896u),vec2<u32>(16777216u,33488896u),vec2<u32>(16777727u,33488896u),vec2<u32>(16842753u,33488896u),vec2<u32>(4278255360u,33488897u),vec2<u32>(4278190081u,33488897u),vec2<u32>(4278190336u,33488897u),vec2<u32>(4278255616u,33488897u),vec2<u32>(16776960u,33488897u),vec2<u32>(16711935u,33488897u),vec2<u32>(16711936u,33488897u),vec2<u32>(16711937u,33488897u),vec2<u32>(65535u,33488897u),vec2<u32>(0u,33488897u),vec2<u32>(256u,33488897u),vec2<u32>(257u,33488897u),vec2<u32>(130816u,33488897u),vec2<u32>(65537u,33488897u),vec2<u32>(65793u,33488897u),vec2<u32>(33488896u,33488897u),vec2<u32>(16842496u,33488897u),vec2<u32>(16777473u,33488897u),vec2<u32>(16843007u,33488897u),vec2<u32>(4294967295u,33489407u),vec2<u32>(4294967041u,33489407u),vec2<u32>(4294902271u,33489407u),vec2<u32>(4294902017u,33489407u),vec2<u32>(4278190080u,33489407u),vec2<u32>(4278321151u,33489407u),vec2<u32>(4278320897u,33489407u),vec2<u32>(4278256127u,33489407u),vec2<u32>(4278255873u,33489407u),vec2<u32>(16776960u,33489407u),vec2<u32>(16711680u,33489407u),vec2<u32>(65280u,33489407u),vec2<u32>(255u,33489407u),vec2<u32>(256u,33489407u),vec2<u32>(65536u,33489407u),vec2<u32>(65792u,33489407u),vec2<u32>(33554431u,33489407u),vec2<u32>(33554177u,33489407u),vec2<u32>(33489407u,33489407u),vec2<u32>(33489153u,33489407u),vec2<u32>(16777216u,33489407u),vec2<u32>(16908287u,33489407u),vec2<u32>(16908033u,33489407u),vec2<u32>(16843263u,33489407u),vec2<u32>(16843009u,33489407u),vec2<u32>(4294901760u,33489152u),vec2<u32>(4294901761u,33489152u),vec2<u32>(4278255360u,33489152u),vec2<u32>(4278190335u,33489152u),vec2<u32>(4278190081u,33489152u),vec2<u32>(4278255616u,33489152u),vec2<u32>(16776960u,33489152u),vec2<u32>(16711935u,33489152u),vec2<u32>(16711681u,33489152u),vec2<u32>(16711936u,33489152u),vec2<u32>(65535u,33489152u),vec2<u32>(65281u,33489152u),vec2<u32>(0u,33489152u),vec2<u32>(257u,33489152u),vec2<u32>(130816u,33489152u),vec2<u32>(65791u,33489152u),vec2<u32>(33488896u,33489152u),vec2<u32>(16777217u,33489152u),vec2<u32>(16777472u,33489152u),vec2<u32>(16842752u,33489152u),vec2<u32>(4294967295u,33489153u),vec2<u32>(4294967041u,33489153u),vec2<u32>(4294902271u,33489153u),vec2<u32>(4294902017u,33489153u),vec2<u32>(4278190080u,33489153u),vec2<u32>(4278321151u,33489153u),vec2<u32>(4278320897u,33489153u),vec2<u32>(4278256127u,33489153u),vec2<u32>(4278255873u,33489153u),vec2<u32>(16711680u,33489153u),vec2<u32>(65280u,33489153u),vec2<u32>(255u,33489153u),vec2<u32>(1u,33489153u),vec2<u32>(33554431u,33489153u),vec2<u32>(33554177u,33489153u),vec2<u32>(33489407u,33489153u),vec2<u32>(33489153u,33489153u),vec2<u32>(16777216u,33489153u),vec2<u32>(16908287u,33489153u),vec2<u32>(16908033u,33489153u),vec2<u32>(16843263u,33489153u),vec2<u32>(16843009u,33489153u),vec2<u32>(4294901760u,16842751u),vec2<u32>(4278255360u,16842751u),vec2<u32>(4278190081u,16842751u),vec2<u32>(4278190591u,16842751u),vec2<u32>(4278190336u,16842751u),vec2<u32>(4278255616u,16842751u),vec2<u32>(16776960u,16842751u),vec2<u32>(16711681u,16842751u),vec2<u32>(16711936u,16842751u),vec2<u32>(0u,16842751u),vec2<u32>(511u,16842751u),vec2<u32>(257u,16842751u),vec2<u32>(65792u,16842751u),vec2<u32>(65793u,16842751u),vec2<u32>(33488896u,16842751u),vec2<u32>(16842496u,16842751u),vec2<u32>(16777471u,16842751u),vec2<u32>(16777217u,16842751u),vec2<u32>(16777472u,16842751u),vec2<u32>(16842752u,16842751u),vec2<u32>(4294967040u,16842496u),vec2<u32>(4294902015u,16842496u),vec2<u32>(4294901761u,16842496u),vec2<u32>(4294902016u,16842496u),vec2<u32>(4278255615u,16842496u),vec2<u32>(4278190080u,16842496u),vec2<u32>(4278190591u,16842496u),vec2<u32>(4278190337u,16842496u),vec2<u32>(4278320896u,16842496u),vec2<u32>(4278255871u,16842496u),vec2<u32>(4278255617u,16842496u),vec2<u32>(4278255872u,16842496u),vec2<u32>(16777215u,16842496u),vec2<u32>(16711680u,16842496u),vec2<u32>(65535u,16842496u),vec2<u32>(65280u,16842496u),vec2<u32>(255u,16842496u),vec2<u32>(0u,16842496u),vec2<u32>(1u,16842496u),vec2<u32>(256u,16842496u),vec2<u32>(130817u,16842496u),vec2<u32>(65536u,16842496u),vec2<u32>(33489151u,16842496u),vec2<u32>(33488897u,16842496u),vec2<u32>(16842497u,16842496u),vec2<u32>(16777216u,16842496u),vec2<u32>(16777727u,16842496u),vec2<u32>(16908032u,16842496u),vec2<u32>(16843007u,16842496u),vec2<u32>(16842753u,16842496u),vec2<u32>(16843008u,16842496u),vec2<u32>(4294901760u,16842497u),vec2<u32>(4278255360u,16842497u),vec2<u32>(4278190335u,16842497u),vec2<u32>(4278190336u,16842497u),vec2<u32>(4278255616u,16842497u),vec2<u32>(16711935u,16842497u),vec2<u32>(16711681u,16842497u),vec2<u32>(16711936u,16842497u),vec2<u32>(65535u,16842497u),vec2<u32>(65281u,16842497u),vec2<u32>(0u,16842497u),vec2<u32>(511u,16842497u),vec2<u32>(65537u,16842497u),vec2<u32>(65792u,16842497u),vec2<u32>(33488896u,16842497u),vec2<u32>(16777471u,16842497u),vec2<u32>(16777217u,16842497u),vec2<u32>(16843008u,16842497u),vec2<u32>(4294967040u,16777471u),vec2<u32>(4294902015u,16777471u),vec2<u32>(4294901761u,16777471u),vec2<u32>(4278255615u,16777471u),vec2<u32>(4278190080u,16777471u),vec2<u32>(4278190591u,16777471u),vec2<u32>(4278255617u,16777471u),vec2<u32>(16777215u,16777471u),vec2<u32>(16711937u,16777471u),vec2<u32>(65280u,16777471u),vec2<u32>(255u,16777471u),vec2<u32>(0u,16777471u),vec2<u32>(1u,16777471u),vec2<u32>(511u,16777471u),vec2<u32>(256u,16777471u),vec2<u32>(131071u,16777471u),vec2<u32>(130816u,16777471u),vec2<u32>(130817u,16777471u),vec2<u32>(65536u,16777471u),vec2<u32>(33489151u,16777471u),vec2<u32>(33488897u,16777471u),vec2<u32>(16842497u,16777471u),vec2<u32>(16777471u,16777471u),vec2<u32>(16777216u,16777471u),vec2<u32>(16777727u,16777471u),vec2<u32>(16908032u,16777471u),vec2<u32>(16843008u,16777471u),vec2<u32>(4294967295u,16777216u),vec2<u32>(4294901760u,16777216u),vec2<u32>(4294902271u,16777216u),vec2<u32>(4294902017u,16777216u),vec2<u32>(4278255615u,16777216u),vec2<u32>(4278255360u,16777216u),vec2<u32>(4278190335u,16777216u),vec2<u32>(4278190080u,16777216u),vec2<u32>(4278190081u,16777216u),vec2<u32>(4278190336u,16777216u),vec2<u32>(4278320896u,16777216u),vec2<u32>(4278255616u,16777216u),vec2<u32>(4278255872u,16777216u),vec2<u32>(4278255873u,16777216u),vec2<u32>(16776960u,16777216u),vec2<u32>(16711935u,16777216u),vec2<u32>(16711680u,16777216u),vec2<u32>(16711681u,16777216u),vec2<u32>(16711936u,16777216u),vec2<u32>(65535u,16777216u),vec2<u32>(65280u,16777216u),vec2<u32>(65281u,16777216u),vec2<u32>(255u,16777216u),vec2<u32>(0u,16777216u),vec2<u32>(1u,16777216u),vec2<u32>(511u,16777216u),vec2<u32>(256u,16777216u),vec2<u32>(257u,16777216u),vec2<u32>(130816u,16777216u),vec2<u32>(65791u,16777216u),vec2<u32>(65536u,16777216u),vec2<u32>(65537u,16777216u),vec2<u32>(65792u,16777216u),vec2<u32>(33554176u,16777216u),vec2<u32>(33488896u,16777216u),vec2<u32>(33489407u,16777216u),vec2<u32>(16842496u,16777216u),vec2<u32>(16842497u,16777216u),vec2<u32>(16777471u,16777216u),vec2<u32>(16777216u,16777216u),vec2<u32>(16777217u,16777216u),vec2<u32>(16777472u,16777216u),vec2<u32>(16777473u,16777216u),vec2<u32>(16908287u,16777216u),vec2<u32>(16908033u,16777216u),vec2<u32>(16842752u,16777216u),vec2<u32>(16843263u,16777216u),vec2<u32>(16843009u,16777216u),vec2<u32>(4294967040u,16777217u),vec2<u32>(4294902015u,16777217u),vec2<u32>(4278255615u,16777217u),vec2<u32>(4278190080u,16777217u),vec2<u32>(4278190336u,16777217u),vec2<u32>(4278321151u,16777217u),vec2<u32>(4278255617u,16777217u),vec2<u32>(4278255872u,16777217u),vec2<u32>(16711680u,16777217u),vec2<u32>(16712191u,16777217u),vec2<u32>(16711936u,16777217u),vec2<u32>(65280u,16777217u),vec2<u32>(65281u,16777217u),vec2<u32>(0u,16777217u),vec2<u32>(1u,16777217u),vec2<u32>(256u,16777217u),vec2<u32>(65536u,16777217u),vec2<u32>(66047u,16777217u),vec2<u32>(33554177u,16777217u),vec2<u32>(33489151u,16777217u),vec2<u32>(33489152u,16777217u),vec2<u32>(33489153u,16777217u),vec2<u32>(16842497u,16777217u),vec2<u32>(16777471u,16777217u),vec2<u32>(16777216u,16777217u),vec2<u32>(16843007u,16777217u),vec2<u32>(16842753u,16777217u),vec2<u32>(16843008u,16777217u),vec2<u32>(4294901760u,16777727u),vec2<u32>(4278190081u,16777727u),vec2<u32>(4278190336u,16777727u),vec2<u32>(4278255616u,16777727u),vec2<u32>(16776960u,16777727u),vec2<u32>(16711681u,16777727u),vec2<u32>(65535u,16777727u),vec2<u32>(65281u,16777727u),vec2<u32>(0u,16777727u),vec2<u32>(1u,16777727u),vec2<u32>(257u,16777727u),vec2<u32>(65791u,16777727u),vec2<u32>(65536u,16777727u),vec2<u32>(33488896u,16777727u),vec2<u32>(16842496u,16777727u),vec2<u32>(16777217u,16777727u),vec2<u32>(16777472u,16777727u),vec2<u32>(16842752u,16777727u),vec2<u32>(4294902015u,16777472u),vec2<u32>(4294901761u,16777472u),vec2<u32>(4294902016u,16777472u),vec2<u32>(4278255615u,16777472u),vec2<u32>(4278255361u,16777472u),vec2<u32>(4278190080u,16777472u),vec2<u32>(4278190591u,16777472u),vec2<u32>(4278190337u,16777472u),vec2<u32>(4278321151u,16777472u),vec2<u32>(4278320896u,16777472u),vec2<u32>(4278255871u,16777472u),vec2<u32>(4278255617u,16777472u),vec2<u32>(16777215u,16777472u),vec2<u32>(16776961u,16777472u),vec2<u32>(16711680u,16777472u),vec2<u32>(16712191u,16777472u),vec2<u32>(16711937u,16777472u),vec2<u32>(65280u,16777472u),vec2<u32>(255u,16777472u),vec2<u32>(0u,16777472u),vec2<u32>(1u,16777472u),vec2<u32>(256u,16777472u),vec2<u32>(130817u,16777472u),vec2<u32>(65536u,16777472u),vec2<u32>(65537u,16777472u),vec2<u32>(65793u,16777472u),vec2<u32>(33554176u,16777472u),vec2<u32>(33489151u,16777472u),vec2<u32>(16842751u,16777472u),vec2<u32>(16842497u,16777472u),vec2<u32>(16777216u,16777472u),vec2<u32>(16777473u,16777472u),vec2<u32>(16908032u,16777472u),vec2<u32>(16842753u,16777472u),vec2<u32>(4294901760u,16777473u),vec2<u32>(4278190080u,16777473u),vec2<u32>(4278255616u,16777473u),vec2<u32>(16711935u,16777473u),vec2<u32>(16711681u,16777473u),vec2<u32>(16711936u,16777473u),vec2<u32>(65535u,16777473u),vec2<u32>(0u,16777473u),vec2<u32>(511u,16777473u),vec2<u32>(130816u,16777473u),vec2<u32>(33488896u,16777473u),vec2<u32>(16842496u,16777473u),vec2<u32>(16777471u,16777473u),vec2<u32>(16777216u,16777473u),vec2<u32>(16777217u,16777473u),vec2<u32>(4294967295u,16908287u),vec2<u32>(4294967041u,16908287u),vec2<u32>(4294902271u,16908287u),vec2<u32>(4294902017u,16908287u),vec2<u32>(4278190080u,16908287u),vec2<u32>(4278321151u,16908287u),vec2<u32>(4278320897u,16908287u),vec2<u32>(4278256127u,16908287u),vec2<u32>(4278255873u,16908287u),vec2<u32>(16711680u,16908287u),vec2<u32>(65280u,16908287u),vec2<u32>(255u,16908287u),vec2<u32>(1u,16908287u),vec2<u32>(256u,16908287u),vec2<u32>(33554431u,16908287u),vec2<u32>(33554177u,16908287u),vec2<u32>(33489407u,16908287u),vec2<u32>(33489153u,16908287u),vec2<u32>(16777216u,16908287u),vec2<u32>(16908287u,16908287u),vec2<u32>(16908033u,16908287u),vec2<u32>(16843263u,16908287u),vec2<u32>(16843009u,16908287u),vec2<u32>(4294901760u,16908032u),vec2<u32>(4294902016u,16908032u),vec2<u32>(4278255360u,16908032u),vec2<u32>(4278190335u,16908032u),vec2<u32>(4278190081u,16908032u),vec2<u32>(4278190336u,16908032u),vec2<u32>(4278190337u,16908032u),vec2<u32>(16711681u,16908032u),vec2<u32>(16711936u,16908032u),vec2<u32>(65280u,16908032u),vec2<u32>(0u,16908032u),vec2<u32>(511u,16908032u),vec2<u32>(257u,16908032u),vec2<u32>(130816u,16908032u),vec2<u32>(65791u,16908032u),vec2<u32>(33488896u,16908032u),vec2<u32>(16842751u,16908032u),vec2<u32>(16842497u,16908032u),vec2<u32>(16777217u,16908032u),vec2<u32>(16777472u,16908032u),vec2<u32>(4294967041u,16908033u),vec2<u32>(4294902271u,16908033u),vec2<u32>(4294902017u,16908033u),vec2<u32>(4278255615u,16908033u),vec2<u32>(4278190336u,16908033u),vec2<u32>(4278320897u,16908033u),vec2<u32>(4278256127u,16908033u),vec2<u32>(4278255873u,16908033u),vec2<u32>(16711680u,16908033u),vec2<u32>(65280u,16908033u),vec2<u32>(1u,16908033u),vec2<u32>(256u,16908033u),vec2<u32>(65536u,16908033u),vec2<u32>(33554431u,16908033u),vec2<u32>(33554177u,16908033u),vec2<u32>(33489407u,16908033u),vec2<u32>(33489153u,16908033u),vec2<u32>(16777216u,16908033u),vec2<u32>(16908287u,16908033u),vec2<u32>(16908033u,16908033u),vec2<u32>(16843263u,16908033u),vec2<u32>(16843009u,16908033u),vec2<u32>(4278190336u,16843007u),vec2<u32>(4278255616u,16843007u),vec2<u32>(16776960u,16843007u),vec2<u32>(16711935u,16843007u),vec2<u32>(65535u,16843007u),vec2<u32>(255u,16843007u),vec2<u32>(0u,16843007u),vec2<u32>(511u,16843007u),vec2<u32>(257u,16843007u),vec2<u32>(130816u,16843007u),vec2<u32>(65536u,16843007u),vec2<u32>(65537u,16843007u),vec2<u32>(66047u,16843007u),vec2<u32>(65792u,16843007u),vec2<u32>(33488896u,16843007u),vec2<u32>(4294901761u,16842752u),vec2<u32>(4294902016u,16842752u),vec2<u32>(4278255615u,16842752u),vec2<u32>(4278255361u,16842752u),vec2<u32>(4278190080u,16842752u),vec2<u32>(4278190591u,16842752u),vec2<u32>(4278255617u,16842752u),vec2<u32>(4278255872u,16842752u),vec2<u32>(16776961u,16842752u),vec2<u32>(16711680u,16842752u),vec2<u32>(65280u,16842752u),vec2<u32>(255u,16842752u),vec2<u32>(0u,16842752u),vec2<u32>(1u,16842752u),vec2<u32>(256u,16842752u),vec2<u32>(65536u,16842752u),vec2<u32>(65793u,16842752u),vec2<u32>(33554176u,16842752u),vec2<u32>(33489151u,16842752u),vec2<u32>(33488896u,16842752u),vec2<u32>(33488897u,16842752u),vec2<u32>(33489152u,16842752u),vec2<u32>(16842497u,16842752u),vec2<u32>(16777216u,16842752u),vec2<u32>(16777727u,16842752u),vec2<u32>(4294901760u,16842753u),vec2<u32>(4278255360u,16842753u),vec2<u32>(4278190081u,16842753u),vec2<u32>(4278190337u,16842753u),vec2<u32>(4278320896u,16842753u),vec2<u32>(4278255616u,16842753u),vec2<u32>(16711935u,16842753u),vec2<u32>(16711681u,16842753u),vec2<u32>(16711937u,16842753u),vec2<u32>(65281u,16842753u),vec2<u32>(0u,16842753u),vec2<u32>(1u,16842753u),vec2<u32>(511u,16842753u),vec2<u32>(131071u,16842753u),vec2<u32>(130817u,16842753u),vec2<u32>(33488897u,16842753u),vec2<u32>(16842751u,16842753u),vec2<u32>(16777216u,16842753u),vec2<u32>(16777217u,16842753u),vec2<u32>(16777472u,16842753u),vec2<u32>(16908032u,16842753u),vec2<u32>(16843007u,16842753u),vec2<u32>(16842753u,16842753u),vec2<u32>(4294967295u,16843263u),vec2<u32>(4294967041u,16843263u),vec2<u32>(4294902271u,16843263u),vec2<u32>(4294902017u,16843263u),vec2<u32>(4278321151u,16843263u),vec2<u32>(4278320897u,16843263u),vec2<u32>(4278256127u,16843263u),vec2<u32>(4278255873u,16843263u),vec2<u32>(65280u,16843263u),vec2<u32>(255u,16843263u),vec2<u32>(1u,16843263u),vec2<u32>(256u,16843263u),vec2<u32>(33554431u,16843263u),vec2<u32>(33554177u,16843263u),vec2<u32>(33489407u,16843263u),vec2<u32>(33489153u,16843263u),vec2<u32>(16777216u,16843263u),vec2<u32>(16908287u,16843263u),vec2<u32>(16908033u,16843263u),vec2<u32>(16843263u,16843263u),vec2<u32>(16843009u,16843263u),vec2<u32>(4294901760u,16843008u),vec2<u32>(4278190335u,16843008u),vec2<u32>(4278190336u,16843008u),vec2<u32>(4278320896u,16843008u),vec2<u32>(4278255616u,16843008u),vec2<u32>(16776960u,16843008u),vec2<u32>(65535u,16843008u),vec2<u32>(0u,16843008u),vec2<u32>(257u,16843008u),vec2<u32>(130816u,16843008u),vec2<u32>(65537u,16843008u),vec2<u32>(65792u,16843008u),vec2<u32>(16842751u,16843008u),vec2<u32>(16777217u,16843008u),vec2<u32>(4294967295u,16843009u),vec2<u32>(4294967041u,16843009u),vec2<u32>(4294902271u,16843009u),vec2<u32>(4294902017u,16843009u),vec2<u32>(4278321151u,16843009u),vec2<u32>(4278320897u,16843009u),vec2<u32>(4278256127u,16843009u),vec2<u32>(4278255873u,16843009u),vec2<u32>(65280u,16843009u),vec2<u32>(255u,16843009u),vec2<u32>(1u,16843009u),vec2<u32>(33554431u,16843009u),vec2<u32>(33554177u,16843009u),vec2<u32>(33489407u,16843009u),vec2<u32>(33489153u,16843009u),vec2<u32>(16777216u,16843009u),vec2<u32>(16908287u,16843009u),vec2<u32>(16908033u,16843009u),vec2<u32>(16843263u,16843009u));"],["ops/_shared/iq-grid-iq1s.498a1afa8d.wgsl.jinja",`{% set iqSfx = "" %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,2047>=array<vec2<u32>,2047>(
{% include "ops/_shared/iq-grid-iq1s.4a6f38db3c.wgsl.jinja" %}const IQ_GRID_1{{ iqSfx }}: array<vec2<u32>, 1> = array<vec2<u32>, 1>(
vec2<u32>(16843009u,16843009u));fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{if(i<2047u){return IQ_GRID_0{{ iqSfx }}[i];}return IQ_GRID_1{{ iqSfx }}[i-2047u];}`],["ops/_shared/iq-grid-iq2xxs.87768952ad.wgsl.jinja","vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(137046827u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721955627u,134744072u),vec2<u32>(724240427u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723069704u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(137037867u,134744107u),vec2<u32>(721946667u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(724244744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(419961131u,134748441u),vec2<u32>(422259481u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(723060761u,134748441u),vec2<u32>(134753049u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(724244744u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(137046792u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421070891u,134753032u),vec2<u32>(721955592u,134753032u),vec2<u32>(134748424u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(134744107u,134753067u),vec2<u32>(135862536u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(723065113u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(421079851u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(137042184u,135858219u),vec2<u32>(419961113u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(137042201u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(135867144u,135862553u),vec2<u32>(422250539u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(135862571u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(721946649u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(724240392u,135867161u),vec2<u32>(421070873u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134753067u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(422250521u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(721946667u,137037832u),vec2<u32>(137046809u,137037849u),vec2<u32>(419965704u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(134744107u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(421075243u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(422254856u,137042201u),vec2<u32>(723060744u,137042219u),vec2<u32>(134753032u,137046792u),vec2<u32>(137037832u,137046792u),vec2<u32>(723065096u,137046792u),vec2<u32>(419961096u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(421075243u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(422250521u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(419965704u,419956779u),vec2<u32>(421075243u,419956779u),vec2<u32>(422259464u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(723069721u,419961096u),vec2<u32>(135858219u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(421079816u,419965721u),vec2<u32>(422250521u,419965721u),vec2<u32>(721946667u,419965721u),vec2<u32>(419961113u,419965739u),vec2<u32>(723060744u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135867161u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(721955592u,421070856u),vec2<u32>(134748424u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(724244744u,421070873u),vec2<u32>(723060761u,421070891u),vec2<u32>(723060744u,421075208u),vec2<u32>(723060779u,421075208u),vec2<u32>(134753067u,421075225u),vec2<u32>(134744089u,421075243u),vec2<u32>(421075208u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(135858201u,421079816u),vec2<u32>(135867161u,421079816u),vec2<u32>(422254856u,421079816u),vec2<u32>(419956744u,421079833u),vec2<u32>(134753032u,421079851u),vec2<u32>(134748424u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(422259464u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(421075225u,422250521u),vec2<u32>(135867144u,422250539u),vec2<u32>(422250504u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(134748441u,422254856u),vec2<u32>(135858184u,422254873u),vec2<u32>(135858219u,422254873u),vec2<u32>(721950984u,422254873u),vec2<u32>(419956779u,422259464u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134753067u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(721946667u,721946632u),vec2<u32>(134748424u,721946649u),vec2<u32>(135867144u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(135858201u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(419961131u,721951001u),vec2<u32>(723065096u,721951001u),vec2<u32>(134753049u,721951019u),vec2<u32>(419956744u,721951019u),vec2<u32>(422250504u,721951019u),vec2<u32>(134744107u,721955592u),vec2<u32>(134748424u,721955609u),vec2<u32>(135858201u,721955627u),vec2<u32>(134748424u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(137042184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(724240409u,723060744u),vec2<u32>(135862571u,723060761u),vec2<u32>(721946632u,723060761u),vec2<u32>(419961113u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(137037867u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070873u,723065113u),vec2<u32>(721946649u,723069704u),vec2<u32>(137037832u,723069721u),vec2<u32>(134744107u,724240392u),vec2<u32>(421070856u,724240392u),vec2<u32>(721951001u,724240392u),vec2<u32>(134753049u,724240409u),vec2<u32>(134744072u,724240427u),vec2<u32>(135867144u,724244744u),vec2<u32>(421070856u,724249352u),vec2<u32>(134748424u,724249369u));"],["ops/_shared/iq-grid-iq2xxs.cf9c8c7724.wgsl.jinja",`{% set iqSfx = "" %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,256>=array<vec2<u32>,256>(
{% include "ops/_shared/iq-grid-iq2xxs.87768952ad.wgsl.jinja" %}fn iq_grid{{ iqSfx }}(i: u32) -> vec2<u32> {
return IQ_GRID_0{{ iqSfx }}[i];}`],["ops/_shared/iq-grid-iq2xs.79d6ac37c3.wgsl.jinja","vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(135862571u,134744072u),vec2<u32>(135867161u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137042201u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(419961131u,134744072u),vec2<u32>(419965721u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421070891u,134744072u),vec2<u32>(421075225u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721951001u,134744072u),vec2<u32>(721955592u,134744072u),vec2<u32>(723060761u,134744072u),vec2<u32>(723065096u,134744072u),vec2<u32>(723069721u,134744072u),vec2<u32>(724240392u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(134748459u,134744089u),vec2<u32>(134753049u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135858219u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(135867144u,134744089u),vec2<u32>(135867179u,134744089u),vec2<u32>(137037849u,134744089u),vec2<u32>(137042184u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(419956779u,134744089u),vec2<u32>(419961113u,134744089u),vec2<u32>(419965704u,134744089u),vec2<u32>(421070873u,134744089u),vec2<u32>(421075208u,134744089u),vec2<u32>(422250504u,134744089u),vec2<u32>(422259464u,134744089u),vec2<u32>(721946649u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723060744u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(134748441u,134744107u),vec2<u32>(134753032u,134744107u),vec2<u32>(135858201u,134744107u),vec2<u32>(135862536u,134744107u),vec2<u32>(137037832u,134744107u),vec2<u32>(419956761u,134744107u),vec2<u32>(419961096u,134744107u),vec2<u32>(421070856u,134744107u),vec2<u32>(421075225u,134744107u),vec2<u32>(721946632u,134744107u),vec2<u32>(721955627u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(134748459u,134748424u),vec2<u32>(134753049u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(135858219u,134748424u),vec2<u32>(135862553u,134748424u),vec2<u32>(135867144u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419961113u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(421070873u,134748424u),vec2<u32>(421075208u,134748424u),vec2<u32>(421075243u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134748441u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(135858201u,134748441u),vec2<u32>(135862536u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(419956761u,134748441u),vec2<u32>(419961096u,134748441u),vec2<u32>(421070856u,134748441u),vec2<u32>(422250521u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(134744089u,134748459u),vec2<u32>(134748424u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(137042219u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(419956779u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134744107u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(134753067u,134753032u),vec2<u32>(135858201u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(137037832u,134753032u),vec2<u32>(137042201u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421079816u,134753032u),vec2<u32>(721946632u,134753032u),vec2<u32>(724240392u,134753032u),vec2<u32>(724249387u,134753032u),vec2<u32>(134744089u,134753049u),vec2<u32>(134748424u,134753049u),vec2<u32>(135858184u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(721946649u,134753049u),vec2<u32>(721955609u,134753049u),vec2<u32>(134744072u,134753067u),vec2<u32>(137037832u,134753067u),vec2<u32>(137046792u,134753067u),vec2<u32>(723065131u,134753067u),vec2<u32>(724240392u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(134748459u,135858184u),vec2<u32>(134753049u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(135858219u,135858184u),vec2<u32>(135862553u,135858184u),vec2<u32>(135867144u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(137042184u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(419956779u,135858184u),vec2<u32>(419961113u,135858184u),vec2<u32>(419965704u,135858184u),vec2<u32>(421070873u,135858184u),vec2<u32>(421075208u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(422259499u,135858184u),vec2<u32>(721946649u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134744107u,135858201u),vec2<u32>(134748441u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(135858201u,135858201u),vec2<u32>(135862536u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(419956761u,135858201u),vec2<u32>(419961096u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(723065096u,135858201u),vec2<u32>(723065131u,135858201u),vec2<u32>(134744089u,135858219u),vec2<u32>(134748424u,135858219u),vec2<u32>(134748459u,135858219u),vec2<u32>(135858184u,135858219u),vec2<u32>(419956744u,135858219u),vec2<u32>(422250504u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134744107u,135862536u),vec2<u32>(134748441u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(135858201u,135862536u),vec2<u32>(135862536u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(419956761u,135862536u),vec2<u32>(419961096u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(421070856u,135862536u),vec2<u32>(422254856u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(134744089u,135862553u),vec2<u32>(134748424u,135862553u),vec2<u32>(135858184u,135862553u),vec2<u32>(419956744u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(135862536u,135862571u),vec2<u32>(419965721u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(135858219u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(421075208u,135867144u),vec2<u32>(721951019u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(422254891u,135867161u),vec2<u32>(421070873u,135867179u),vec2<u32>(724249369u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134748441u,137037832u),vec2<u32>(134753032u,137037832u),vec2<u32>(134753067u,137037832u),vec2<u32>(135858201u,137037832u),vec2<u32>(135862536u,137037832u),vec2<u32>(137037832u,137037832u),vec2<u32>(419956761u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(421070856u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(724240392u,137037832u),vec2<u32>(134744089u,137037849u),vec2<u32>(134748424u,137037849u),vec2<u32>(135858184u,137037849u),vec2<u32>(419956744u,137037849u),vec2<u32>(419965704u,137037849u),vec2<u32>(422254873u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(137037867u,137037867u),vec2<u32>(721946632u,137037867u),vec2<u32>(724249352u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(137046809u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(421070891u,137042201u),vec2<u32>(723069721u,137042201u),vec2<u32>(134744089u,137042219u),vec2<u32>(135867179u,137042219u),vec2<u32>(724244779u,137042219u),vec2<u32>(134744072u,137046792u),vec2<u32>(134753032u,137046792u),vec2<u32>(134753067u,137046792u),vec2<u32>(137037832u,137046792u),vec2<u32>(421075225u,137046792u),vec2<u32>(721955592u,137046792u),vec2<u32>(724240427u,137046792u),vec2<u32>(422259464u,137046809u),vec2<u32>(723060744u,137046809u),vec2<u32>(134753032u,137046827u),vec2<u32>(137037832u,137046827u),vec2<u32>(721946667u,137046827u),vec2<u32>(721955592u,137046827u),vec2<u32>(721955627u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(134748459u,419956744u),vec2<u32>(134753049u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135858219u,419956744u),vec2<u32>(135862553u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419956779u,419956744u),vec2<u32>(419961113u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(419965739u,419956744u),vec2<u32>(421070873u,419956744u),vec2<u32>(421075208u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(422254873u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(134744107u,419956761u),vec2<u32>(134748441u,419956761u),vec2<u32>(134753032u,419956761u),vec2<u32>(135858201u,419956761u),vec2<u32>(135862536u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(419956761u,419956761u),vec2<u32>(419961096u,419956761u),vec2<u32>(421070856u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(724240427u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(134748424u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(135858219u,419956779u),vec2<u32>(137046809u,419956779u),vec2<u32>(419956744u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134744107u,419961096u),vec2<u32>(134748441u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(135858201u,419961096u),vec2<u32>(135862536u,419961096u),vec2<u32>(135867161u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(419956761u,419961096u),vec2<u32>(419961096u,419961096u),vec2<u32>(421070856u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(723065096u,419961096u),vec2<u32>(134744089u,419961113u),vec2<u32>(134748424u,419961113u),vec2<u32>(135858184u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(419956744u,419961113u),vec2<u32>(723069739u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134753067u,419961131u),vec2<u32>(419961096u,419961131u),vec2<u32>(421070856u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(421075208u,419965704u),vec2<u32>(422250539u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(135858201u,419965721u),vec2<u32>(419961096u,419965721u),vec2<u32>(421070856u,419965721u),vec2<u32>(422259481u,419965721u),vec2<u32>(134748424u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134744107u,421070856u),vec2<u32>(134748441u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135862536u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(137046792u,421070856u),vec2<u32>(419956761u,421070856u),vec2<u32>(419961096u,421070856u),vec2<u32>(421070856u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(134744089u,421070873u),vec2<u32>(134748424u,421070873u),vec2<u32>(135858184u,421070873u),vec2<u32>(135862553u,421070873u),vec2<u32>(419956744u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(134744072u,421070891u),vec2<u32>(419961096u,421070891u),vec2<u32>(724249387u,421070891u),vec2<u32>(134744089u,421075208u),vec2<u32>(134748424u,421075208u),vec2<u32>(135858184u,421075208u),vec2<u32>(137037849u,421075208u),vec2<u32>(419956744u,421075208u),vec2<u32>(422250504u,421075208u),vec2<u32>(721946649u,421075208u),vec2<u32>(724240409u,421075208u),vec2<u32>(134744072u,421075225u),vec2<u32>(134753032u,421075225u),vec2<u32>(721946632u,421075225u),vec2<u32>(721955592u,421075225u),vec2<u32>(137037849u,421075243u),vec2<u32>(422259464u,421075243u),vec2<u32>(724240409u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(135862536u,421079816u),vec2<u32>(419956761u,421079816u),vec2<u32>(421070856u,421079816u),vec2<u32>(723069721u,421079816u),vec2<u32>(135867179u,421079833u),vec2<u32>(419956744u,421079833u),vec2<u32>(419956779u,421079833u),vec2<u32>(721951001u,421079851u),vec2<u32>(134744089u,422250504u),vec2<u32>(134748424u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(421075208u,422250504u),vec2<u32>(422250539u,422250504u),vec2<u32>(721951019u,422250504u),vec2<u32>(724249369u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(137042184u,422250539u),vec2<u32>(419965739u,422250539u),vec2<u32>(723060779u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(135862571u,422254856u),vec2<u32>(135858184u,422254873u),vec2<u32>(419956744u,422254873u),vec2<u32>(419961113u,422254873u),vec2<u32>(724244744u,422254873u),vec2<u32>(134744089u,422259464u),vec2<u32>(422259499u,422259464u),vec2<u32>(137042201u,422259481u),vec2<u32>(134748459u,422259499u),vec2<u32>(421075208u,422259499u),vec2<u32>(422250539u,422259499u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134748441u,721946632u),vec2<u32>(134753032u,721946632u),vec2<u32>(135858201u,721946632u),vec2<u32>(135862536u,721946632u),vec2<u32>(137037832u,721946632u),vec2<u32>(137046827u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(419961096u,721946632u),vec2<u32>(421070856u,721946632u),vec2<u32>(721946632u,721946632u),vec2<u32>(721946667u,721946632u),vec2<u32>(724249352u,721946632u),vec2<u32>(724249387u,721946632u),vec2<u32>(134744089u,721946649u),vec2<u32>(134748424u,721946649u),vec2<u32>(134748459u,721946649u),vec2<u32>(135858184u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(421070873u,721946649u),vec2<u32>(421079833u,721946649u),vec2<u32>(134744072u,721946667u),vec2<u32>(137037832u,721946667u),vec2<u32>(721946632u,721946667u),vec2<u32>(721946667u,721946667u),vec2<u32>(724240392u,721946667u),vec2<u32>(724249352u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135858219u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(721955609u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(419961096u,721951001u),vec2<u32>(724244761u,721951001u),vec2<u32>(135867144u,721951019u),vec2<u32>(422259499u,721951019u),vec2<u32>(134744072u,721955592u),vec2<u32>(134753032u,721955592u),vec2<u32>(137042201u,721955592u),vec2<u32>(421079851u,721955592u),vec2<u32>(721946632u,721955592u),vec2<u32>(721946667u,721955592u),vec2<u32>(724249352u,721955592u),vec2<u32>(134748459u,721955609u),vec2<u32>(137037867u,721955627u),vec2<u32>(721946632u,721955627u),vec2<u32>(721955592u,721955627u),vec2<u32>(723065131u,721955627u),vec2<u32>(724249352u,721955627u),vec2<u32>(134744089u,723060744u),vec2<u32>(134748424u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(421075243u,723060744u),vec2<u32>(721950984u,723060744u),vec2<u32>(134744072u,723060761u),vec2<u32>(137037867u,723060761u),vec2<u32>(422254856u,723060761u),vec2<u32>(421075243u,723060779u),vec2<u32>(721955609u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(134748441u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070856u,723065096u),vec2<u32>(421079816u,723065096u),vec2<u32>(137046809u,723065113u),vec2<u32>(723060744u,723065113u),vec2<u32>(723060779u,723065113u),vec2<u32>(419956761u,723065131u),vec2<u32>(421070873u,723069704u),vec2<u32>(724244779u,723069704u),vec2<u32>(419965721u,723069721u),vec2<u32>(135862553u,723069739u),vec2<u32>(422250504u,723069739u),vec2<u32>(134744072u,724240392u),vec2<u32>(134744107u,724240392u),vec2<u32>(134753032u,724240392u),vec2<u32>(134753067u,724240392u),vec2<u32>(137037832u,724240392u),vec2<u32>(137046827u,724240392u),vec2<u32>(724240392u,724240392u),vec2<u32>(421070873u,724240409u),vec2<u32>(421079833u,724240409u),vec2<u32>(724244779u,724240409u),vec2<u32>(134744072u,724240427u),vec2<u32>(134744107u,724240427u),vec2<u32>(134753032u,724240427u),vec2<u32>(137046827u,724240427u),vec2<u32>(721946632u,724240427u),vec2<u32>(724240392u,724240427u),vec2<u32>(419956744u,724244744u),vec2<u32>(723065113u,724244744u),vec2<u32>(422254873u,724244779u),vec2<u32>(723069704u,724244779u),vec2<u32>(134753067u,724249352u),vec2<u32>(137037832u,724249352u),vec2<u32>(137037867u,724249352u),vec2<u32>(137046792u,724249352u),vec2<u32>(724240392u,724249352u),vec2<u32>(724249352u,724249352u),vec2<u32>(134748424u,724249369u),vec2<u32>(721950984u,724249369u),vec2<u32>(721951019u,724249369u),vec2<u32>(137046792u,724249387u),vec2<u32>(137046827u,724249387u),vec2<u32>(723060761u,724249387u),vec2<u32>(724249387u,724249387u));"],["ops/_shared/iq-grid-iq2xs.fc760188d7.wgsl.jinja",`{% set iqSfx = "" %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,512>=array<vec2<u32>,512>(
{% include "ops/_shared/iq-grid-iq2xs.79d6ac37c3.wgsl.jinja" %}fn iq_grid{{ iqSfx }}(i: u32) -> vec2<u32> {
return IQ_GRID_0{{ iqSfx }}[i];}`],["ops/_shared/iq-grid-iq2s.1d969b24af.wgsl.jinja","vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(135862571u,134744072u),vec2<u32>(135867161u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137042201u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(419961131u,134744072u),vec2<u32>(419965721u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421070891u,134744072u),vec2<u32>(421075225u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(422254891u,134744072u),vec2<u32>(422259481u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721951001u,134744072u),vec2<u32>(721955592u,134744072u),vec2<u32>(723060761u,134744072u),vec2<u32>(723065096u,134744072u),vec2<u32>(724240392u,134744072u),vec2<u32>(724244761u,134744072u),vec2<u32>(724249387u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(134748459u,134744089u),vec2<u32>(134753049u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135858219u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(135867144u,134744089u),vec2<u32>(137037849u,134744089u),vec2<u32>(137042184u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(419956779u,134744089u),vec2<u32>(419961113u,134744089u),vec2<u32>(419965704u,134744089u),vec2<u32>(421070873u,134744089u),vec2<u32>(421075208u,134744089u),vec2<u32>(421075243u,134744089u),vec2<u32>(421079833u,134744089u),vec2<u32>(422250504u,134744089u),vec2<u32>(422254873u,134744089u),vec2<u32>(422259464u,134744089u),vec2<u32>(721946649u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723060744u,134744089u),vec2<u32>(723060779u,134744089u),vec2<u32>(723065113u,134744089u),vec2<u32>(724240409u,134744089u),vec2<u32>(724244744u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(134748441u,134744107u),vec2<u32>(134753032u,134744107u),vec2<u32>(135858201u,134744107u),vec2<u32>(135862536u,134744107u),vec2<u32>(137037832u,134744107u),vec2<u32>(137046827u,134744107u),vec2<u32>(419956761u,134744107u),vec2<u32>(419961096u,134744107u),vec2<u32>(419961131u,134744107u),vec2<u32>(419965721u,134744107u),vec2<u32>(421070856u,134744107u),vec2<u32>(421075225u,134744107u),vec2<u32>(721946632u,134744107u),vec2<u32>(721951001u,134744107u),vec2<u32>(721955627u,134744107u),vec2<u32>(723065096u,134744107u),vec2<u32>(724240427u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(134748459u,134748424u),vec2<u32>(134753049u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(135858219u,134748424u),vec2<u32>(135862553u,134748424u),vec2<u32>(135867144u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(137042219u,134748424u),vec2<u32>(137046809u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419961113u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(419965739u,134748424u),vec2<u32>(421070873u,134748424u),vec2<u32>(421075208u,134748424u),vec2<u32>(421075243u,134748424u),vec2<u32>(421079833u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(422250539u,134748424u),vec2<u32>(422254873u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(721951019u,134748424u),vec2<u32>(721955609u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(723065113u,134748424u),vec2<u32>(723069704u,134748424u),vec2<u32>(724240409u,134748424u),vec2<u32>(724244744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134748441u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(134753067u,134748441u),vec2<u32>(135858201u,134748441u),vec2<u32>(135862536u,134748441u),vec2<u32>(135862571u,134748441u),vec2<u32>(135867161u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(137042201u,134748441u),vec2<u32>(137046792u,134748441u),vec2<u32>(419956761u,134748441u),vec2<u32>(419961096u,134748441u),vec2<u32>(419961131u,134748441u),vec2<u32>(419965721u,134748441u),vec2<u32>(421070856u,134748441u),vec2<u32>(421070891u,134748441u),vec2<u32>(421075225u,134748441u),vec2<u32>(421079816u,134748441u),vec2<u32>(422250521u,134748441u),vec2<u32>(422254856u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(721946667u,134748441u),vec2<u32>(721951001u,134748441u),vec2<u32>(721955592u,134748441u),vec2<u32>(723060761u,134748441u),vec2<u32>(723065096u,134748441u),vec2<u32>(724240392u,134748441u),vec2<u32>(134744089u,134748459u),vec2<u32>(134748424u,134748459u),vec2<u32>(134748459u,134748459u),vec2<u32>(134753049u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(135862553u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(419961113u,134748459u),vec2<u32>(419965704u,134748459u),vec2<u32>(421070873u,134748459u),vec2<u32>(421075208u,134748459u),vec2<u32>(422250504u,134748459u),vec2<u32>(721946649u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(723060744u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134744107u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(135858201u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(135862571u,134753032u),vec2<u32>(135867161u,134753032u),vec2<u32>(137037832u,134753032u),vec2<u32>(137042201u,134753032u),vec2<u32>(137046827u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(419961131u,134753032u),vec2<u32>(419965721u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421070891u,134753032u),vec2<u32>(421075225u,134753032u),vec2<u32>(421079816u,134753032u),vec2<u32>(422250521u,134753032u),vec2<u32>(422254856u,134753032u),vec2<u32>(721946632u,134753032u),vec2<u32>(721951001u,134753032u),vec2<u32>(723065096u,134753032u),vec2<u32>(724249387u,134753032u),vec2<u32>(134744089u,134753049u),vec2<u32>(134748424u,134753049u),vec2<u32>(135858184u,134753049u),vec2<u32>(135858219u,134753049u),vec2<u32>(135862553u,134753049u),vec2<u32>(135867144u,134753049u),vec2<u32>(137037849u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(419961113u,134753049u),vec2<u32>(419965704u,134753049u),vec2<u32>(421070873u,134753049u),vec2<u32>(421075208u,134753049u),vec2<u32>(422250504u,134753049u),vec2<u32>(721946649u,134753049u),vec2<u32>(723060744u,134753049u),vec2<u32>(134744072u,134753067u),vec2<u32>(135858201u,134753067u),vec2<u32>(135862536u,134753067u),vec2<u32>(137037867u,134753067u),vec2<u32>(137046792u,134753067u),vec2<u32>(137046827u,134753067u),vec2<u32>(421070856u,134753067u),vec2<u32>(723069721u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(134748459u,135858184u),vec2<u32>(134753049u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(135858219u,135858184u),vec2<u32>(135862553u,135858184u),vec2<u32>(135867144u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(137042184u,135858184u),vec2<u32>(137042219u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(419956779u,135858184u),vec2<u32>(419961113u,135858184u),vec2<u32>(419965704u,135858184u),vec2<u32>(421070873u,135858184u),vec2<u32>(421075208u,135858184u),vec2<u32>(421075243u,135858184u),vec2<u32>(421079833u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(422250539u,135858184u),vec2<u32>(422254873u,135858184u),vec2<u32>(422259464u,135858184u),vec2<u32>(721946649u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(721951019u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(723065113u,135858184u),vec2<u32>(723069704u,135858184u),vec2<u32>(724240409u,135858184u),vec2<u32>(724244744u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134744107u,135858201u),vec2<u32>(134748441u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(134753067u,135858201u),vec2<u32>(135858201u,135858201u),vec2<u32>(135862536u,135858201u),vec2<u32>(135862571u,135858201u),vec2<u32>(135867161u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(137037867u,135858201u),vec2<u32>(137042201u,135858201u),vec2<u32>(137046792u,135858201u),vec2<u32>(419956761u,135858201u),vec2<u32>(419961096u,135858201u),vec2<u32>(419961131u,135858201u),vec2<u32>(419965721u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(421070891u,135858201u),vec2<u32>(421075225u,135858201u),vec2<u32>(421079816u,135858201u),vec2<u32>(422250521u,135858201u),vec2<u32>(422254856u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(721946667u,135858201u),vec2<u32>(721951001u,135858201u),vec2<u32>(721955592u,135858201u),vec2<u32>(723060761u,135858201u),vec2<u32>(723065096u,135858201u),vec2<u32>(134744089u,135858219u),vec2<u32>(134748424u,135858219u),vec2<u32>(134753049u,135858219u),vec2<u32>(135858184u,135858219u),vec2<u32>(135862553u,135858219u),vec2<u32>(137037849u,135858219u),vec2<u32>(137042184u,135858219u),vec2<u32>(419956744u,135858219u),vec2<u32>(419961113u,135858219u),vec2<u32>(421070873u,135858219u),vec2<u32>(421075208u,135858219u),vec2<u32>(721946649u,135858219u),vec2<u32>(721950984u,135858219u),vec2<u32>(723060744u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134744107u,135862536u),vec2<u32>(134748441u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(135858201u,135862536u),vec2<u32>(135862536u,135862536u),vec2<u32>(135862571u,135862536u),vec2<u32>(135867161u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(137042201u,135862536u),vec2<u32>(137046792u,135862536u),vec2<u32>(419956761u,135862536u),vec2<u32>(419961096u,135862536u),vec2<u32>(419961131u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(421070856u,135862536u),vec2<u32>(421070891u,135862536u),vec2<u32>(421075225u,135862536u),vec2<u32>(421079816u,135862536u),vec2<u32>(422250521u,135862536u),vec2<u32>(422254856u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(721946667u,135862536u),vec2<u32>(721951001u,135862536u),vec2<u32>(721955592u,135862536u),vec2<u32>(723060761u,135862536u),vec2<u32>(723065096u,135862536u),vec2<u32>(724240392u,135862536u),vec2<u32>(134744089u,135862553u),vec2<u32>(134748424u,135862553u),vec2<u32>(134748459u,135862553u),vec2<u32>(134753049u,135862553u),vec2<u32>(135858184u,135862553u),vec2<u32>(135858219u,135862553u),vec2<u32>(135862553u,135862553u),vec2<u32>(135867144u,135862553u),vec2<u32>(137037849u,135862553u),vec2<u32>(137042184u,135862553u),vec2<u32>(419956744u,135862553u),vec2<u32>(419956779u,135862553u),vec2<u32>(419961113u,135862553u),vec2<u32>(419965704u,135862553u),vec2<u32>(421070873u,135862553u),vec2<u32>(421075208u,135862553u),vec2<u32>(422250504u,135862553u),vec2<u32>(721946649u,135862553u),vec2<u32>(721950984u,135862553u),vec2<u32>(723060744u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(134748441u,135862571u),vec2<u32>(134753032u,135862571u),vec2<u32>(135858201u,135862571u),vec2<u32>(135862536u,135862571u),vec2<u32>(137037832u,135862571u),vec2<u32>(419956761u,135862571u),vec2<u32>(419961096u,135862571u),vec2<u32>(421070856u,135862571u),vec2<u32>(721946632u,135862571u),vec2<u32>(724249387u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(134748459u,135867144u),vec2<u32>(134753049u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(135862553u,135867144u),vec2<u32>(135867144u,135867144u),vec2<u32>(137037849u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(419956779u,135867144u),vec2<u32>(419961113u,135867144u),vec2<u32>(419965704u,135867144u),vec2<u32>(421070873u,135867144u),vec2<u32>(421075208u,135867144u),vec2<u32>(422250504u,135867144u),vec2<u32>(721946649u,135867144u),vec2<u32>(721950984u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134744107u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(134753032u,135867161u),vec2<u32>(135858201u,135867161u),vec2<u32>(135862536u,135867161u),vec2<u32>(137037832u,135867161u),vec2<u32>(419956761u,135867161u),vec2<u32>(419961096u,135867161u),vec2<u32>(421070856u,135867161u),vec2<u32>(422259481u,135867161u),vec2<u32>(724240427u,135867161u),vec2<u32>(134748424u,135867179u),vec2<u32>(135858184u,135867179u),vec2<u32>(419956744u,135867179u),vec2<u32>(421075243u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134748441u,137037832u),vec2<u32>(134753032u,137037832u),vec2<u32>(135858201u,137037832u),vec2<u32>(135862536u,137037832u),vec2<u32>(135862571u,137037832u),vec2<u32>(135867161u,137037832u),vec2<u32>(137037832u,137037832u),vec2<u32>(137042201u,137037832u),vec2<u32>(137046827u,137037832u),vec2<u32>(419956761u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(421070856u,137037832u),vec2<u32>(421070891u,137037832u),vec2<u32>(421075225u,137037832u),vec2<u32>(422254856u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(721955627u,137037832u),vec2<u32>(723065096u,137037832u),vec2<u32>(724249387u,137037832u),vec2<u32>(134744089u,137037849u),vec2<u32>(134748424u,137037849u),vec2<u32>(135858184u,137037849u),vec2<u32>(135858219u,137037849u),vec2<u32>(135862553u,137037849u),vec2<u32>(137037849u,137037849u),vec2<u32>(419956744u,137037849u),vec2<u32>(419956779u,137037849u),vec2<u32>(419961113u,137037849u),vec2<u32>(421070873u,137037849u),vec2<u32>(421075208u,137037849u),vec2<u32>(422250504u,137037849u),vec2<u32>(721946649u,137037849u),vec2<u32>(721950984u,137037849u),vec2<u32>(723060744u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(134753067u,137037867u),vec2<u32>(137037867u,137037867u),vec2<u32>(137046792u,137037867u),vec2<u32>(137046827u,137037867u),vec2<u32>(419961096u,137037867u),vec2<u32>(421070856u,137037867u),vec2<u32>(721955592u,137037867u),vec2<u32>(721955627u,137037867u),vec2<u32>(724249352u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(134748459u,137042184u),vec2<u32>(134753049u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(135862553u,137042184u),vec2<u32>(135867144u,137042184u),vec2<u32>(137037849u,137042184u),vec2<u32>(137042184u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(419956779u,137042184u),vec2<u32>(419961113u,137042184u),vec2<u32>(419965704u,137042184u),vec2<u32>(421070873u,137042184u),vec2<u32>(421075208u,137042184u),vec2<u32>(422250504u,137042184u),vec2<u32>(721946649u,137042184u),vec2<u32>(721950984u,137042184u),vec2<u32>(723060744u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(134748441u,137042201u),vec2<u32>(134753032u,137042201u),vec2<u32>(135858201u,137042201u),vec2<u32>(135862536u,137042201u),vec2<u32>(137037832u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(419961096u,137042201u),vec2<u32>(421070856u,137042201u),vec2<u32>(422254891u,137042201u),vec2<u32>(721946632u,137042201u),vec2<u32>(134744089u,137042219u),vec2<u32>(134748424u,137042219u),vec2<u32>(135858184u,137042219u),vec2<u32>(419956744u,137042219u),vec2<u32>(421079833u,137042219u),vec2<u32>(134744072u,137046792u),vec2<u32>(134748441u,137046792u),vec2<u32>(135858201u,137046792u),vec2<u32>(135862536u,137046792u),vec2<u32>(419956761u,137046792u),vec2<u32>(419961096u,137046792u),vec2<u32>(421070856u,137046792u),vec2<u32>(721955627u,137046792u),vec2<u32>(724249387u,137046792u),vec2<u32>(134744089u,137046809u),vec2<u32>(134748424u,137046809u),vec2<u32>(135858184u,137046809u),vec2<u32>(723065113u,137046809u),vec2<u32>(134753067u,137046827u),vec2<u32>(137037867u,137046827u),vec2<u32>(422254856u,137046827u),vec2<u32>(721955592u,137046827u),vec2<u32>(721955627u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(134748459u,419956744u),vec2<u32>(134753049u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135858219u,419956744u),vec2<u32>(135862553u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(135867179u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(137042219u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419956779u,419956744u),vec2<u32>(419961113u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(419965739u,419956744u),vec2<u32>(421070873u,419956744u),vec2<u32>(421075208u,419956744u),vec2<u32>(421075243u,419956744u),vec2<u32>(421079833u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(422250539u,419956744u),vec2<u32>(422254873u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(723065113u,419956744u),vec2<u32>(723069704u,419956744u),vec2<u32>(724240409u,419956744u),vec2<u32>(724244744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(134744107u,419956761u),vec2<u32>(134748441u,419956761u),vec2<u32>(134753032u,419956761u),vec2<u32>(135858201u,419956761u),vec2<u32>(135862536u,419956761u),vec2<u32>(135862571u,419956761u),vec2<u32>(135867161u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(137037867u,419956761u),vec2<u32>(137042201u,419956761u),vec2<u32>(419956761u,419956761u),vec2<u32>(419961096u,419956761u),vec2<u32>(419961131u,419956761u),vec2<u32>(419965721u,419956761u),vec2<u32>(421070856u,419956761u),vec2<u32>(421070891u,419956761u),vec2<u32>(421075225u,419956761u),vec2<u32>(421079816u,419956761u),vec2<u32>(422250521u,419956761u),vec2<u32>(422254856u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721946667u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(721955592u,419956761u),vec2<u32>(723060761u,419956761u),vec2<u32>(723065096u,419956761u),vec2<u32>(724240392u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(134748424u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(135858219u,419956779u),vec2<u32>(135862553u,419956779u),vec2<u32>(135867144u,419956779u),vec2<u32>(137042184u,419956779u),vec2<u32>(419956744u,419956779u),vec2<u32>(419961113u,419956779u),vec2<u32>(419965704u,419956779u),vec2<u32>(421070873u,419956779u),vec2<u32>(421075208u,419956779u),vec2<u32>(422250504u,419956779u),vec2<u32>(721946649u,419956779u),vec2<u32>(721950984u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134744107u,419961096u),vec2<u32>(134748441u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(134753067u,419961096u),vec2<u32>(135858201u,419961096u),vec2<u32>(135862536u,419961096u),vec2<u32>(135862571u,419961096u),vec2<u32>(135867161u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(137037867u,419961096u),vec2<u32>(137042201u,419961096u),vec2<u32>(137046792u,419961096u),vec2<u32>(419956761u,419961096u),vec2<u32>(419961096u,419961096u),vec2<u32>(419961131u,419961096u),vec2<u32>(419965721u,419961096u),vec2<u32>(421070856u,419961096u),vec2<u32>(421070891u,419961096u),vec2<u32>(421075225u,419961096u),vec2<u32>(421079816u,419961096u),vec2<u32>(422250521u,419961096u),vec2<u32>(422254856u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(721946667u,419961096u),vec2<u32>(721951001u,419961096u),vec2<u32>(721955592u,419961096u),vec2<u32>(723060761u,419961096u),vec2<u32>(723065096u,419961096u),vec2<u32>(724240392u,419961096u),vec2<u32>(134744089u,419961113u),vec2<u32>(134748424u,419961113u),vec2<u32>(134748459u,419961113u),vec2<u32>(134753049u,419961113u),vec2<u32>(135858184u,419961113u),vec2<u32>(135858219u,419961113u),vec2<u32>(135862553u,419961113u),vec2<u32>(135867144u,419961113u),vec2<u32>(137037849u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(419956744u,419961113u),vec2<u32>(419956779u,419961113u),vec2<u32>(419961113u,419961113u),vec2<u32>(419965704u,419961113u),vec2<u32>(421070873u,419961113u),vec2<u32>(421075208u,419961113u),vec2<u32>(422250504u,419961113u),vec2<u32>(422259499u,419961113u),vec2<u32>(721946649u,419961113u),vec2<u32>(721950984u,419961113u),vec2<u32>(723060744u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134744107u,419961131u),vec2<u32>(134748441u,419961131u),vec2<u32>(134753032u,419961131u),vec2<u32>(135858201u,419961131u),vec2<u32>(135862536u,419961131u),vec2<u32>(137037832u,419961131u),vec2<u32>(419956761u,419961131u),vec2<u32>(419961096u,419961131u),vec2<u32>(421070856u,419961131u),vec2<u32>(721946632u,419961131u),vec2<u32>(724244761u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(134753049u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(135858219u,419965704u),vec2<u32>(135862553u,419965704u),vec2<u32>(135867144u,419965704u),vec2<u32>(137037849u,419965704u),vec2<u32>(137042184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419956779u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(419965704u,419965704u),vec2<u32>(421070873u,419965704u),vec2<u32>(421075208u,419965704u),vec2<u32>(422250504u,419965704u),vec2<u32>(721950984u,419965704u),vec2<u32>(723060744u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(134744107u,419965721u),vec2<u32>(134748441u,419965721u),vec2<u32>(134753032u,419965721u),vec2<u32>(135858201u,419965721u),vec2<u32>(135862536u,419965721u),vec2<u32>(137037832u,419965721u),vec2<u32>(419956761u,419965721u),vec2<u32>(419961096u,419965721u),vec2<u32>(421070856u,419965721u),vec2<u32>(721946632u,419965721u),vec2<u32>(723065131u,419965721u),vec2<u32>(134744089u,419965739u),vec2<u32>(134748424u,419965739u),vec2<u32>(135858184u,419965739u),vec2<u32>(419956744u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134744107u,421070856u),vec2<u32>(134748441u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135862536u,421070856u),vec2<u32>(135862571u,421070856u),vec2<u32>(135867161u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(137037867u,421070856u),vec2<u32>(137042201u,421070856u),vec2<u32>(137046792u,421070856u),vec2<u32>(419956761u,421070856u),vec2<u32>(419961096u,421070856u),vec2<u32>(419961131u,421070856u),vec2<u32>(419965721u,421070856u),vec2<u32>(421070856u,421070856u),vec2<u32>(421070891u,421070856u),vec2<u32>(421075225u,421070856u),vec2<u32>(421079816u,421070856u),vec2<u32>(422250521u,421070856u),vec2<u32>(422254856u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(721946667u,421070856u),vec2<u32>(721951001u,421070856u),vec2<u32>(721955592u,421070856u),vec2<u32>(723060761u,421070856u),vec2<u32>(723065096u,421070856u),vec2<u32>(134744089u,421070873u),vec2<u32>(134748424u,421070873u),vec2<u32>(134748459u,421070873u),vec2<u32>(134753049u,421070873u),vec2<u32>(135858184u,421070873u),vec2<u32>(135858219u,421070873u),vec2<u32>(135862553u,421070873u),vec2<u32>(135867144u,421070873u),vec2<u32>(137037849u,421070873u),vec2<u32>(137042184u,421070873u),vec2<u32>(419956744u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(419961113u,421070873u),vec2<u32>(419965704u,421070873u),vec2<u32>(421070873u,421070873u),vec2<u32>(421075208u,421070873u),vec2<u32>(422250504u,421070873u),vec2<u32>(721946649u,421070873u),vec2<u32>(721950984u,421070873u),vec2<u32>(723060744u,421070873u),vec2<u32>(134744072u,421070891u),vec2<u32>(134748441u,421070891u),vec2<u32>(134753032u,421070891u),vec2<u32>(135858201u,421070891u),vec2<u32>(135862536u,421070891u),vec2<u32>(137037832u,421070891u),vec2<u32>(419956761u,421070891u),vec2<u32>(419961096u,421070891u),vec2<u32>(421070856u,421070891u),vec2<u32>(422259481u,421070891u),vec2<u32>(721946632u,421070891u),vec2<u32>(134744089u,421075208u),vec2<u32>(134748424u,421075208u),vec2<u32>(134748459u,421075208u),vec2<u32>(134753049u,421075208u),vec2<u32>(135858184u,421075208u),vec2<u32>(135858219u,421075208u),vec2<u32>(135862553u,421075208u),vec2<u32>(135867144u,421075208u),vec2<u32>(137037849u,421075208u),vec2<u32>(137042184u,421075208u),vec2<u32>(419956744u,421075208u),vec2<u32>(419956779u,421075208u),vec2<u32>(419961113u,421075208u),vec2<u32>(419965704u,421075208u),vec2<u32>(421070873u,421075208u),vec2<u32>(421075208u,421075208u),vec2<u32>(422250504u,421075208u),vec2<u32>(721946649u,421075208u),vec2<u32>(721950984u,421075208u),vec2<u32>(723060744u,421075208u),vec2<u32>(134744072u,421075225u),vec2<u32>(134744107u,421075225u),vec2<u32>(134748441u,421075225u),vec2<u32>(134753032u,421075225u),vec2<u32>(135858201u,421075225u),vec2<u32>(135862536u,421075225u),vec2<u32>(137037832u,421075225u),vec2<u32>(419956761u,421075225u),vec2<u32>(419961096u,421075225u),vec2<u32>(421070856u,421075225u),vec2<u32>(721946632u,421075225u),vec2<u32>(134744089u,421075243u),vec2<u32>(134748424u,421075243u),vec2<u32>(135858184u,421075243u),vec2<u32>(137042219u,421075243u),vec2<u32>(419956744u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(134744107u,421079816u),vec2<u32>(134748441u,421079816u),vec2<u32>(134753032u,421079816u),vec2<u32>(135858201u,421079816u),vec2<u32>(135862536u,421079816u),vec2<u32>(137037832u,421079816u),vec2<u32>(419956761u,421079816u),vec2<u32>(419961096u,421079816u),vec2<u32>(421070856u,421079816u),vec2<u32>(421079851u,421079816u),vec2<u32>(721946632u,421079816u),vec2<u32>(134744089u,421079833u),vec2<u32>(134748424u,421079833u),vec2<u32>(135858184u,421079833u),vec2<u32>(419956744u,421079833u),vec2<u32>(134744072u,421079851u),vec2<u32>(135867161u,421079851u),vec2<u32>(721951001u,421079851u),vec2<u32>(724249352u,421079851u),vec2<u32>(134744089u,422250504u),vec2<u32>(134748424u,422250504u),vec2<u32>(134748459u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(135858219u,422250504u),vec2<u32>(135862553u,422250504u),vec2<u32>(135867144u,422250504u),vec2<u32>(137037849u,422250504u),vec2<u32>(137042184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(419961113u,422250504u),vec2<u32>(419965704u,422250504u),vec2<u32>(421070873u,422250504u),vec2<u32>(421075208u,422250504u),vec2<u32>(422250504u,422250504u),vec2<u32>(721950984u,422250504u),vec2<u32>(723060744u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(134744107u,422250521u),vec2<u32>(134748441u,422250521u),vec2<u32>(134753032u,422250521u),vec2<u32>(135858201u,422250521u),vec2<u32>(135862536u,422250521u),vec2<u32>(137037832u,422250521u),vec2<u32>(419956761u,422250521u),vec2<u32>(419961096u,422250521u),vec2<u32>(421070856u,422250521u),vec2<u32>(721946632u,422250521u),vec2<u32>(723069721u,422250521u),vec2<u32>(134748424u,422250539u),vec2<u32>(135858184u,422250539u),vec2<u32>(419956744u,422250539u),vec2<u32>(421075243u,422250539u),vec2<u32>(724240409u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(134748441u,422254856u),vec2<u32>(134753032u,422254856u),vec2<u32>(135858201u,422254856u),vec2<u32>(135862536u,422254856u),vec2<u32>(137037832u,422254856u),vec2<u32>(419956761u,422254856u),vec2<u32>(419961096u,422254856u),vec2<u32>(421070856u,422254856u),vec2<u32>(721946632u,422254856u),vec2<u32>(134744089u,422254873u),vec2<u32>(134748424u,422254873u),vec2<u32>(135858184u,422254873u),vec2<u32>(419956744u,422254873u),vec2<u32>(419965739u,422254873u),vec2<u32>(422259464u,422254873u),vec2<u32>(723060779u,422254873u),vec2<u32>(134744072u,422254891u),vec2<u32>(723065096u,422254891u),vec2<u32>(134744089u,422259464u),vec2<u32>(134748424u,422259464u),vec2<u32>(135858184u,422259464u),vec2<u32>(422254873u,422259464u),vec2<u32>(723069704u,422259464u),vec2<u32>(134744072u,422259481u),vec2<u32>(137046827u,422259481u),vec2<u32>(419956779u,422259499u),vec2<u32>(724240409u,422259499u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134748441u,721946632u),vec2<u32>(134753032u,721946632u),vec2<u32>(135858201u,721946632u),vec2<u32>(135862536u,721946632u),vec2<u32>(135867161u,721946632u),vec2<u32>(137037832u,721946632u),vec2<u32>(137042201u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(419961096u,721946632u),vec2<u32>(421070856u,721946632u),vec2<u32>(421070891u,721946632u),vec2<u32>(421075225u,721946632u),vec2<u32>(421079816u,721946632u),vec2<u32>(422250521u,721946632u),vec2<u32>(721946632u,721946632u),vec2<u32>(721951001u,721946632u),vec2<u32>(723060761u,721946632u),vec2<u32>(723065096u,721946632u),vec2<u32>(134744089u,721946649u),vec2<u32>(134748424u,721946649u),vec2<u32>(134753049u,721946649u),vec2<u32>(135858184u,721946649u),vec2<u32>(135858219u,721946649u),vec2<u32>(135862553u,721946649u),vec2<u32>(135867144u,721946649u),vec2<u32>(137037849u,721946649u),vec2<u32>(137042184u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(419956779u,721946649u),vec2<u32>(419961113u,721946649u),vec2<u32>(419965704u,721946649u),vec2<u32>(421070873u,721946649u),vec2<u32>(421075208u,721946649u),vec2<u32>(721946649u,721946649u),vec2<u32>(721950984u,721946649u),vec2<u32>(723060744u,721946649u),vec2<u32>(724249369u,721946649u),vec2<u32>(134744072u,721946667u),vec2<u32>(134748441u,721946667u),vec2<u32>(134753067u,721946667u),vec2<u32>(135858201u,721946667u),vec2<u32>(135862536u,721946667u),vec2<u32>(419956761u,721946667u),vec2<u32>(419961096u,721946667u),vec2<u32>(421070856u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(134748459u,721950984u),vec2<u32>(134753049u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135858219u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(135867144u,721950984u),vec2<u32>(137037849u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(419956779u,721950984u),vec2<u32>(419961113u,721950984u),vec2<u32>(419965704u,721950984u),vec2<u32>(421070873u,721950984u),vec2<u32>(421075208u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(721946649u,721950984u),vec2<u32>(721950984u,721950984u),vec2<u32>(723060744u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(134744107u,721951001u),vec2<u32>(134748441u,721951001u),vec2<u32>(134753032u,721951001u),vec2<u32>(135858201u,721951001u),vec2<u32>(135862536u,721951001u),vec2<u32>(137037832u,721951001u),vec2<u32>(419956761u,721951001u),vec2<u32>(419961096u,721951001u),vec2<u32>(421070856u,721951001u),vec2<u32>(721946632u,721951001u),vec2<u32>(721955627u,721951001u),vec2<u32>(134744089u,721951019u),vec2<u32>(134748424u,721951019u),vec2<u32>(135858184u,721951019u),vec2<u32>(137046809u,721951019u),vec2<u32>(419956744u,721951019u),vec2<u32>(134744072u,721955592u),vec2<u32>(134748441u,721955592u),vec2<u32>(135858201u,721955592u),vec2<u32>(135862536u,721955592u),vec2<u32>(419956761u,721955592u),vec2<u32>(419961096u,721955592u),vec2<u32>(421070856u,721955592u),vec2<u32>(724240427u,721955592u),vec2<u32>(134744089u,721955609u),vec2<u32>(134748424u,721955609u),vec2<u32>(419956744u,721955609u),vec2<u32>(422254873u,721955609u),vec2<u32>(137037867u,721955627u),vec2<u32>(421079816u,721955627u),vec2<u32>(421079851u,721955627u),vec2<u32>(721946667u,721955627u),vec2<u32>(724240427u,721955627u),vec2<u32>(134744089u,723060744u),vec2<u32>(134748424u,723060744u),vec2<u32>(134753049u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(135858219u,723060744u),vec2<u32>(135862553u,723060744u),vec2<u32>(135867144u,723060744u),vec2<u32>(137042184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(419956779u,723060744u),vec2<u32>(419961113u,723060744u),vec2<u32>(419965704u,723060744u),vec2<u32>(421070873u,723060744u),vec2<u32>(421075208u,723060744u),vec2<u32>(422250504u,723060744u),vec2<u32>(721946649u,723060744u),vec2<u32>(721950984u,723060744u),vec2<u32>(723060744u,723060744u),vec2<u32>(134744072u,723060761u),vec2<u32>(134748441u,723060761u),vec2<u32>(135858201u,723060761u),vec2<u32>(135862536u,723060761u),vec2<u32>(419956761u,723060761u),vec2<u32>(419961096u,723060761u),vec2<u32>(421070856u,723060761u),vec2<u32>(421079851u,723060761u),vec2<u32>(134744089u,723060779u),vec2<u32>(134748424u,723060779u),vec2<u32>(135858184u,723060779u),vec2<u32>(419956744u,723060779u),vec2<u32>(724244779u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(134744107u,723065096u),vec2<u32>(134748441u,723065096u),vec2<u32>(134753032u,723065096u),vec2<u32>(135858201u,723065096u),vec2<u32>(135862536u,723065096u),vec2<u32>(137037832u,723065096u),vec2<u32>(419956761u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070856u,723065096u),vec2<u32>(721946632u,723065096u),vec2<u32>(723065131u,723065096u),vec2<u32>(134744089u,723065113u),vec2<u32>(134748424u,723065113u),vec2<u32>(135858184u,723065113u),vec2<u32>(419956744u,723065113u),vec2<u32>(723069704u,723065113u),vec2<u32>(724240409u,723065113u),vec2<u32>(134744072u,723065131u),vec2<u32>(419961131u,723065131u),vec2<u32>(422254856u,723065131u),vec2<u32>(134744089u,723069704u),vec2<u32>(134748424u,723069704u),vec2<u32>(135858184u,723069704u),vec2<u32>(137042219u,723069704u),vec2<u32>(419956744u,723069704u),vec2<u32>(724249369u,723069704u),vec2<u32>(134744072u,723069721u),vec2<u32>(419965721u,723069721u),vec2<u32>(421070891u,723069721u),vec2<u32>(723060744u,723069739u),vec2<u32>(134744072u,724240392u),vec2<u32>(134748441u,724240392u),vec2<u32>(134753067u,724240392u),vec2<u32>(135862536u,724240392u),vec2<u32>(137037867u,724240392u),vec2<u32>(137046827u,724240392u),vec2<u32>(419956761u,724240392u),vec2<u32>(419961096u,724240392u),vec2<u32>(421070856u,724240392u),vec2<u32>(724240427u,724240392u),vec2<u32>(724249387u,724240392u),vec2<u32>(419956744u,724240409u),vec2<u32>(422254873u,724240409u),vec2<u32>(134744107u,724240427u),vec2<u32>(134753067u,724240427u),vec2<u32>(137037867u,724240427u),vec2<u32>(137046792u,724240427u),vec2<u32>(137046827u,724240427u),vec2<u32>(721946667u,724240427u),vec2<u32>(721955592u,724240427u),vec2<u32>(721955627u,724240427u),vec2<u32>(724249352u,724240427u),vec2<u32>(134744089u,724244744u),vec2<u32>(134748424u,724244744u),vec2<u32>(135858184u,724244744u),vec2<u32>(419956744u,724244744u),vec2<u32>(721955609u,724244744u),vec2<u32>(724244744u,724244744u),vec2<u32>(134744072u,724244761u),vec2<u32>(135867161u,724244761u),vec2<u32>(421070873u,724244779u),vec2<u32>(134753067u,724249352u),vec2<u32>(137046792u,724249352u),vec2<u32>(724240427u,724249352u),vec2<u32>(421075208u,724249369u),vec2<u32>(721951019u,724249369u),vec2<u32>(134753032u,724249387u),vec2<u32>(134753067u,724249387u),vec2<u32>(137037832u,724249387u),vec2<u32>(137037867u,724249387u),vec2<u32>(137046792u,724249387u),vec2<u32>(721955592u,724249387u),vec2<u32>(724249387u,724249387u));"],["ops/_shared/iq-grid-iq2s.fa1e8b8eff.wgsl.jinja",`{% set iqSfx = "" %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,1024>=array<vec2<u32>,1024>(
{% include "ops/_shared/iq-grid-iq2s.1d969b24af.wgsl.jinja" %}fn iq_grid{{ iqSfx }}(i: u32) -> vec2<u32> {
return IQ_GRID_0{{ iqSfx }}[i];}`],["ops/_shared/iq-grid-iq3xxs.8f4eff63dd.wgsl.jinja","vec2<u32>(1140868096u,1140868096u),vec2<u32>(1140870400u,1140868096u),vec2<u32>(1140871296u,1140868096u),vec2<u32>(1241532928u,1140868096u),vec2<u32>(1241534208u,1140868096u),vec2<u32>(1241535424u,1140868096u),vec2<u32>(1291863040u,1140868096u),vec2<u32>(1291865344u,1140868096u),vec2<u32>(1325419008u,1140868096u),vec2<u32>(1350585600u,1140868096u),vec2<u32>(1405112064u,1140868096u),vec2<u32>(1405112704u,1140868096u),vec2<u32>(1140869632u,1140869632u),vec2<u32>(1140870912u,1140869632u),vec2<u32>(1241531392u,1140869632u),vec2<u32>(1241533696u,1140869632u),vec2<u32>(1291864576u,1140869632u),vec2<u32>(1291866496u,1140869632u),vec2<u32>(1325417472u,1140869632u),vec2<u32>(1325419776u,1140869632u),vec2<u32>(1350584832u,1140869632u),vec2<u32>(1367363712u,1140869632u),vec2<u32>(1405109248u,1140869632u),vec2<u32>(1140868096u,1140870400u),vec2<u32>(1140870400u,1140870400u),vec2<u32>(1140871296u,1140870400u),vec2<u32>(1241532928u,1140870400u),vec2<u32>(1291863040u,1140870400u),vec2<u32>(1291865344u,1140870400u),vec2<u32>(1325419008u,1140870400u),vec2<u32>(1325420288u,1140870400u),vec2<u32>(1325421504u,1140870400u),vec2<u32>(1367362048u,1140870400u),vec2<u32>(1367364544u,1140870400u),vec2<u32>(1405112704u,1140870400u),vec2<u32>(1140869632u,1140870912u),vec2<u32>(1140872128u,1140870912u),vec2<u32>(1241531392u,1140870912u),vec2<u32>(1241533696u,1140870912u),vec2<u32>(1291866496u,1140870912u),vec2<u32>(1405109248u,1140870912u),vec2<u32>(1241534208u,1140871296u),vec2<u32>(1325421504u,1140871296u),vec2<u32>(1350586496u,1140871296u),vec2<u32>(1367364544u,1140871296u),vec2<u32>(1405112064u,1140871296u),vec2<u32>(1405112704u,1140871296u),vec2<u32>(1140869632u,1140871552u),vec2<u32>(1140872128u,1140871552u),vec2<u32>(1325419776u,1140871552u),vec2<u32>(1367362816u,1140871552u),vec2<u32>(1325420928u,1140871808u),vec2<u32>(1384140928u,1140871808u),vec2<u32>(1241531392u,1140872128u),vec2<u32>(1241534592u,1140872128u),vec2<u32>(1241535104u,1140872128u),vec2<u32>(1350586112u,1140872128u),vec2<u32>(1384139264u,1140872128u),vec2<u32>(1140869632u,1241531392u),vec2<u32>(1140870912u,1241531392u),vec2<u32>(1241531392u,1241531392u),vec2<u32>(1241533696u,1241531392u),vec2<u32>(1291864576u,1241531392u),vec2<u32>(1291865856u,1241531392u),vec2<u32>(1325417472u,1241531392u),vec2<u32>(1325419776u,1241531392u),vec2<u32>(1325420672u,1241531392u),vec2<u32>(1350587328u,1241531392u),vec2<u32>(1367360512u,1241531392u),vec2<u32>(1140868096u,1241532928u),vec2<u32>(1140870400u,1241532928u),vec2<u32>(1241532928u,1241532928u),vec2<u32>(1291863040u,1241532928u),vec2<u32>(1291865344u,1241532928u),vec2<u32>(1140869632u,1241533696u),vec2<u32>(1140870912u,1241533696u),vec2<u32>(1241531392u,1241533696u),vec2<u32>(1241533696u,1241533696u),vec2<u32>(1291864576u,1241533696u),vec2<u32>(1325417472u,1241533696u),vec2<u32>(1405111552u,1241533696u),vec2<u32>(1140868096u,1241534208u),vec2<u32>(1140870400u,1241534208u),vec2<u32>(1291863040u,1241534208u),vec2<u32>(1325419008u,1241534208u),vec2<u32>(1350587008u,1241534208u),vec2<u32>(1384141440u,1241534208u),vec2<u32>(1140869632u,1241534592u),vec2<u32>(1140871552u,1241534592u),vec2<u32>(1367360512u,1241534592u),vec2<u32>(1291863040u,1241534848u),vec2<u32>(1291866240u,1241534848u),vec2<u32>(1350587008u,1241534848u),vec2<u32>(1405110784u,1241534848u),vec2<u32>(1140871552u,1241535104u),vec2<u32>(1291865344u,1241535424u),vec2<u32>(1350583296u,1241535424u),vec2<u32>(1140868096u,1291863040u),vec2<u32>(1140870400u,1291863040u),vec2<u32>(1241532928u,1291863040u),vec2<u32>(1241534208u,1291863040u),vec2<u32>(1291863040u,1291863040u),vec2<u32>(1291865344u,1291863040u),vec2<u32>(1291866752u,1291863040u),vec2<u32>(1325419008u,1291863040u),vec2<u32>(1350585600u,1291863040u),vec2<u32>(1140869632u,1291864576u),vec2<u32>(1140870912u,1291864576u),vec2<u32>(1140871552u,1291864576u),vec2<u32>(1241531392u,1291864576u),vec2<u32>(1241533696u,1291864576u),vec2<u32>(1291864576u,1291864576u),vec2<u32>(1325417472u,1291864576u),vec2<u32>(1384140544u,1291864576u),vec2<u32>(1384141760u,1291864576u),vec2<u32>(1405109248u,1291864576u),vec2<u32>(1140868096u,1291865344u),vec2<u32>(1140870400u,1291865344u),vec2<u32>(1241532928u,1291865344u),vec2<u32>(1241535424u,1291865344u),vec2<u32>(1291863040u,1291865344u),vec2<u32>(1291865344u,1291865344u),vec2<u32>(1325421504u,1291865344u),vec2<u32>(1350583296u,1291865344u),vec2<u32>(1367363968u,1291865344u),vec2<u32>(1140869632u,1291865856u),vec2<u32>(1241531392u,1291865856u),vec2<u32>(1241534592u,1291865856u),vec2<u32>(1405109248u,1291865856u),vec2<u32>(1405112448u,1291865856u),vec2<u32>(1325420928u,1291866240u),vec2<u32>(1367363328u,1291866240u),vec2<u32>(1140870912u,1291866496u),vec2<u32>(1291867072u,1291866496u),vec2<u32>(1350584832u,1291866496u),vec2<u32>(1405112448u,1291866496u),vec2<u32>(1140869632u,1291867072u),vec2<u32>(1140870912u,1291867072u),vec2<u32>(1241535104u,1291867072u),vec2<u32>(1350586752u,1291867072u),vec2<u32>(1140869632u,1325417472u),vec2<u32>(1241531392u,1325417472u),vec2<u32>(1241533696u,1325417472u),vec2<u32>(1291864576u,1325417472u),vec2<u32>(1291865856u,1325417472u),vec2<u32>(1367360512u,1325417472u),vec2<u32>(1384141184u,1325417472u),vec2<u32>(1405111552u,1325417472u),vec2<u32>(1140868096u,1325419008u),vec2<u32>(1140870400u,1325419008u),vec2<u32>(1291863040u,1325419008u),vec2<u32>(1325419008u,1325419008u),vec2<u32>(1350586496u,1325419008u),vec2<u32>(1350587008u,1325419008u),vec2<u32>(1140869632u,1325419776u),vec2<u32>(1140870912u,1325419776u),vec2<u32>(1241531392u,1325419776u),vec2<u32>(1291866496u,1325419776u),vec2<u32>(1367362816u,1325419776u),vec2<u32>(1405111552u,1325419776u),vec2<u32>(1241532928u,1325420288u),vec2<u32>(1325420288u,1325420288u),vec2<u32>(1325417472u,1325420672u),vec2<u32>(1350587328u,1325420672u),vec2<u32>(1405111552u,1325420672u),vec2<u32>(1140868096u,1325420928u),vec2<u32>(1140871808u,1325420928u),vec2<u32>(1291865344u,1325420928u),vec2<u32>(1367363968u,1325420928u),vec2<u32>(1241534592u,1325421184u),vec2<u32>(1325421184u,1325421184u),vec2<u32>(1384140544u,1325421184u),vec2<u32>(1325420288u,1325421504u),vec2<u32>(1384137728u,1325421504u),vec2<u32>(1140871296u,1350583296u),vec2<u32>(1241535424u,1350583296u),vec2<u32>(1325420928u,1350583296u),vec2<u32>(1325421504u,1350583296u),vec2<u32>(1367363328u,1350583296u),vec2<u32>(1367364544u,1350583296u),vec2<u32>(1405112448u,1350584832u),vec2<u32>(1291863040u,1350585600u),vec2<u32>(1325421504u,1350585600u),vec2<u32>(1350583296u,1350585600u),vec2<u32>(1384137728u,1350585600u),vec2<u32>(1384141440u,1350585600u),vec2<u32>(1140872128u,1350586112u),vec2<u32>(1350586752u,1350586112u),vec2<u32>(1140871296u,1350586496u),vec2<u32>(1367362048u,1350586496u),vec2<u32>(1384140928u,1350586496u),vec2<u32>(1291866496u,1350586752u),vec2<u32>(1350586112u,1350586752u),vec2<u32>(1405109248u,1350586752u),vec2<u32>(1140871552u,1350587328u),vec2<u32>(1241531392u,1350587328u),vec2<u32>(1241533696u,1350587328u),vec2<u32>(1325417472u,1350587328u),vec2<u32>(1241533696u,1367360512u),vec2<u32>(1350584832u,1367360512u),vec2<u32>(1405109248u,1367360512u),vec2<u32>(1140868096u,1367362048u),vec2<u32>(1140871808u,1367362048u),vec2<u32>(1291866752u,1367362048u),vec2<u32>(1367363968u,1367362048u),vec2<u32>(1241534592u,1367362816u),vec2<u32>(1325419776u,1367362816u),vec2<u32>(1405111552u,1367362816u),vec2<u32>(1140870400u,1367363328u),vec2<u32>(1367363328u,1367363328u),vec2<u32>(1241531392u,1367363712u),vec2<u32>(1291865856u,1367363712u),vec2<u32>(1291867072u,1367363712u),vec2<u32>(1405111552u,1367363712u),vec2<u32>(1140870400u,1367363968u),vec2<u32>(1325419008u,1367363968u),vec2<u32>(1367360512u,1367364224u),vec2<u32>(1291866240u,1367364544u),vec2<u32>(1350585600u,1367364544u),vec2<u32>(1291866240u,1384137728u),vec2<u32>(1350586496u,1384137728u),vec2<u32>(1350587008u,1384137728u),vec2<u32>(1384140928u,1384137728u),vec2<u32>(1291864576u,1384139264u),vec2<u32>(1384139264u,1384139264u),vec2<u32>(1241535424u,1384140032u),vec2<u32>(1384140928u,1384140032u),vec2<u32>(1325417472u,1384140544u),vec2<u32>(1325421184u,1384140544u),vec2<u32>(1350586496u,1384140928u),vec2<u32>(1140871552u,1384141184u),vec2<u32>(1367362816u,1384141184u),vec2<u32>(1325420288u,1384141440u),vec2<u32>(1140870912u,1384141760u),vec2<u32>(1291864576u,1384141760u),vec2<u32>(1140870912u,1405109248u),vec2<u32>(1140871552u,1405109248u),vec2<u32>(1140872128u,1405109248u),vec2<u32>(1241531392u,1405109248u),vec2<u32>(1325419776u,1405109248u),vec2<u32>(1367362816u,1405109248u),vec2<u32>(1291866752u,1405110784u),vec2<u32>(1350583296u,1405110784u),vec2<u32>(1241533696u,1405111552u),vec2<u32>(1350586752u,1405111552u),vec2<u32>(1367362816u,1405111552u),vec2<u32>(1140868096u,1405112064u),vec2<u32>(1241534848u,1405112064u),vec2<u32>(1325420288u,1405112064u),vec2<u32>(1384137728u,1405112064u),vec2<u32>(1291864576u,1405112448u),vec2<u32>(1350584832u,1405112448u),vec2<u32>(1140868096u,1405112704u),vec2<u32>(1140870400u,1405112704u),vec2<u32>(1291866240u,1405112704u),vec2<u32>(1325417472u,1405112960u));"],["ops/_shared/iq-grid-iq3xxs.295293cefb.wgsl.jinja",`{% set iqSfx = "" %}const IQ_MAG{{ iqSfx }}:array<vec2<u32>,256>=array<vec2<u32>,256>(
{% include "ops/_shared/iq-grid-iq3xxs.8f4eff63dd.wgsl.jinja" %}fn iq3mag{{ iqSfx }}(i: u32) -> vec4<f32> {
let e=IQ_MAG{{ iqSfx }}[i];return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}`],["ops/_shared/iq-grid-iq3s.68075e892b.wgsl.jinja","vec2<u32>(1006648320u,1006648320u),vec2<u32>(1006649856u,1006648320u),vec2<u32>(1006650624u,1006648320u),vec2<u32>(1006651776u,1006648320u),vec2<u32>(1006652288u,1006648320u),vec2<u32>(1107311616u,1006648320u),vec2<u32>(1107313152u,1006648320u),vec2<u32>(1107313920u,1006648320u),vec2<u32>(1107314816u,1006648320u),vec2<u32>(1107315328u,1006648320u),vec2<u32>(1157643264u,1006648320u),vec2<u32>(1157644800u,1006648320u),vec2<u32>(1157646720u,1006648320u),vec2<u32>(1191200512u,1006648320u),vec2<u32>(1216363520u,1006648320u),vec2<u32>(1216365824u,1006648320u),vec2<u32>(1216366976u,1006648320u),vec2<u32>(1216367488u,1006648320u),vec2<u32>(1233142272u,1006648320u),vec2<u32>(1233143552u,1006648320u),vec2<u32>(1249917952u,1006648320u),vec2<u32>(1249920256u,1006648320u),vec2<u32>(1266696704u,1006648320u),vec2<u32>(1266698368u,1006648320u),vec2<u32>(1266699136u,1006648320u),vec2<u32>(1006648320u,1006649856u),vec2<u32>(1006649856u,1006649856u),vec2<u32>(1006650624u,1006649856u),vec2<u32>(1006651520u,1006649856u),vec2<u32>(1107311616u,1006649856u),vec2<u32>(1107313152u,1006649856u),vec2<u32>(1107315072u,1006649856u),vec2<u32>(1157643264u,1006649856u),vec2<u32>(1157646080u,1006649856u),vec2<u32>(1157647232u,1006649856u),vec2<u32>(1191199232u,1006649856u),vec2<u32>(1191201152u,1006649856u),vec2<u32>(1216366720u,1006649856u),vec2<u32>(1249919488u,1006649856u),vec2<u32>(1249921408u,1006649856u),vec2<u32>(1266697472u,1006649856u),vec2<u32>(1006648320u,1006650624u),vec2<u32>(1006649856u,1006650624u),vec2<u32>(1006651776u,1006650624u),vec2<u32>(1006652288u,1006650624u),vec2<u32>(1107311616u,1006650624u),vec2<u32>(1107314432u,1006650624u),vec2<u32>(1107315328u,1006650624u),vec2<u32>(1157644800u,1006650624u),vec2<u32>(1157646720u,1006650624u),vec2<u32>(1191197696u,1006650624u),vec2<u32>(1191200896u,1006650624u),vec2<u32>(1216365824u,1006650624u),vec2<u32>(1216366976u,1006650624u),vec2<u32>(1216367488u,1006650624u),vec2<u32>(1233142272u,1006650624u),vec2<u32>(1233143552u,1006650624u),vec2<u32>(1266695168u,1006650624u),vec2<u32>(1266697984u,1006650624u),vec2<u32>(1006651136u,1006651136u),vec2<u32>(1107313152u,1006651136u),vec2<u32>(1107315072u,1006651136u),vec2<u32>(1157643264u,1006651136u),vec2<u32>(1157645568u,1006651136u),vec2<u32>(1191199232u,1006651136u),vec2<u32>(1191200512u,1006651136u),vec2<u32>(1191201408u,1006651136u),vec2<u32>(1216366720u,1006651136u),vec2<u32>(1233140736u,1006651136u),vec2<u32>(1233143040u,1006651136u),vec2<u32>(1249921920u,1006651136u),vec2<u32>(1266696704u,1006651136u),vec2<u32>(1266698624u,1006651136u),vec2<u32>(1006648320u,1006651520u),vec2<u32>(1107314432u,1006651520u),vec2<u32>(1107315584u,1006651520u),vec2<u32>(1157644800u,1006651520u),vec2<u32>(1157646464u,1006651520u),vec2<u32>(1191200000u,1006651520u),vec2<u32>(1216363520u,1006651520u),vec2<u32>(1216366336u,1006651520u),vec2<u32>(1233142272u,1006651520u),vec2<u32>(1266695168u,1006651520u),vec2<u32>(1006650624u,1006651776u),vec2<u32>(1006651520u,1006651776u),vec2<u32>(1157643264u,1006651776u),vec2<u32>(1157645568u,1006651776u),vec2<u32>(1157646976u,1006651776u),vec2<u32>(1191200512u,1006651776u),vec2<u32>(1216365056u,1006651776u),vec2<u32>(1216366976u,1006651776u),vec2<u32>(1216367488u,1006651776u),vec2<u32>(1249921664u,1006651776u),vec2<u32>(1266697984u,1006651776u),vec2<u32>(1006652032u,1006652032u),vec2<u32>(1107313152u,1006652032u),vec2<u32>(1107314432u,1006652032u),vec2<u32>(1191199232u,1006652032u),vec2<u32>(1233143040u,1006652032u),vec2<u32>(1266696704u,1006652032u),vec2<u32>(1006648320u,1006652288u),vec2<u32>(1006650624u,1006652288u),vec2<u32>(1006651520u,1006652288u),vec2<u32>(1157643264u,1006652288u),vec2<u32>(1157645568u,1006652288u),vec2<u32>(1157646976u,1006652288u),vec2<u32>(1191200512u,1006652288u),vec2<u32>(1233140736u,1006652288u),vec2<u32>(1233143936u,1006652288u),vec2<u32>(1006648320u,1107311616u),vec2<u32>(1006649856u,1107311616u),vec2<u32>(1006650624u,1107311616u),vec2<u32>(1006651520u,1107311616u),vec2<u32>(1107311616u,1107311616u),vec2<u32>(1107313152u,1107311616u),vec2<u32>(1107314432u,1107311616u),vec2<u32>(1107315072u,1107311616u),vec2<u32>(1107315584u,1107311616u),vec2<u32>(1157643264u,1107311616u),vec2<u32>(1157645568u,1107311616u),vec2<u32>(1191199232u,1107311616u),vec2<u32>(1191200896u,1107311616u),vec2<u32>(1191201408u,1107311616u),vec2<u32>(1233143936u,1107311616u),vec2<u32>(1233144448u,1107311616u),vec2<u32>(1249919488u,1107311616u),vec2<u32>(1266697472u,1107311616u),vec2<u32>(1006648320u,1107313152u),vec2<u32>(1006649856u,1107313152u),vec2<u32>(1006651136u,1107313152u),vec2<u32>(1006652032u,1107313152u),vec2<u32>(1107311616u,1107313152u),vec2<u32>(1107314816u,1107313152u),vec2<u32>(1157644800u,1107313152u),vec2<u32>(1191197696u,1107313152u),vec2<u32>(1191200512u,1107313152u),vec2<u32>(1216365056u,1107313152u),vec2<u32>(1233140736u,1107313152u),vec2<u32>(1233143040u,1107313152u),vec2<u32>(1266695168u,1107313152u),vec2<u32>(1266698880u,1107313152u),vec2<u32>(1006648320u,1107313920u),vec2<u32>(1107313920u,1107313920u),vec2<u32>(1107315072u,1107313920u),vec2<u32>(1107315584u,1107313920u),vec2<u32>(1157643264u,1107313920u),vec2<u32>(1157646464u,1107313920u),vec2<u32>(1191200000u,1107313920u),vec2<u32>(1216363520u,1107313920u),vec2<u32>(1216366336u,1107313920u),vec2<u32>(1233144192u,1107313920u),vec2<u32>(1249917952u,1107313920u),vec2<u32>(1266697472u,1107313920u),vec2<u32>(1006649856u,1107314432u),vec2<u32>(1006651520u,1107314432u),vec2<u32>(1006652288u,1107314432u),vec2<u32>(1107311616u,1107314432u),vec2<u32>(1107314432u,1107314432u),vec2<u32>(1157644800u,1107314432u),vec2<u32>(1157647232u,1107314432u),vec2<u32>(1191197696u,1107314432u),vec2<u32>(1191200896u,1107314432u),vec2<u32>(1216365056u,1107314432u),vec2<u32>(1249920256u,1107314432u),vec2<u32>(1266695168u,1107314432u),vec2<u32>(1006651136u,1107314816u),vec2<u32>(1006651776u,1107314816u),vec2<u32>(1107313920u,1107314816u),vec2<u32>(1107314816u,1107314816u),vec2<u32>(1191199232u,1107314816u),vec2<u32>(1191200512u,1107314816u),vec2<u32>(1216365824u,1107314816u),vec2<u32>(1216367232u,1107314816u),vec2<u32>(1233140736u,1107314816u),vec2<u32>(1233143936u,1107314816u),vec2<u32>(1006649856u,1107315072u),vec2<u32>(1107311616u,1107315072u),vec2<u32>(1107314432u,1107315072u),vec2<u32>(1157644800u,1107315072u),vec2<u32>(1191197696u,1107315072u),vec2<u32>(1191200000u,1107315072u),vec2<u32>(1233142272u,1107315072u),vec2<u32>(1157643264u,1107315328u),vec2<u32>(1157646464u,1107315328u),vec2<u32>(1157647232u,1107315328u),vec2<u32>(1216366720u,1107315328u),vec2<u32>(1216367232u,1107315328u),vec2<u32>(1006649856u,1107315584u),vec2<u32>(1006651136u,1107315584u),vec2<u32>(1107311616u,1107315584u),vec2<u32>(1107313920u,1107315584u),vec2<u32>(1157644800u,1107315584u),vec2<u32>(1191201152u,1107315584u),vec2<u32>(1216365056u,1107315584u),vec2<u32>(1249920256u,1107315584u),vec2<u32>(1266695168u,1107315584u),vec2<u32>(1006648320u,1157643264u),vec2<u32>(1006649856u,1157643264u),vec2<u32>(1006651136u,1157643264u),vec2<u32>(1006651776u,1157643264u),vec2<u32>(1006652288u,1157643264u),vec2<u32>(1107311616u,1157643264u),vec2<u32>(1107313920u,1157643264u),vec2<u32>(1107314816u,1157643264u),vec2<u32>(1107315328u,1157643264u),vec2<u32>(1157644800u,1157643264u),vec2<u32>(1157646080u,1157643264u),vec2<u32>(1157647232u,1157643264u),vec2<u32>(1191197696u,1157643264u),vec2<u32>(1191200000u,1157643264u),vec2<u32>(1216365056u,1157643264u),vec2<u32>(1216366336u,1157643264u),vec2<u32>(1216366976u,1157643264u),vec2<u32>(1233140736u,1157643264u),vec2<u32>(1233143040u,1157643264u),vec2<u32>(1249921920u,1157643264u),vec2<u32>(1266695168u,1157643264u),vec2<u32>(1266697984u,1157643264u),vec2<u32>(1266698624u,1157643264u),vec2<u32>(1006648320u,1157644800u),vec2<u32>(1006650624u,1157644800u),vec2<u32>(1107311616u,1157644800u),vec2<u32>(1107314432u,1157644800u),vec2<u32>(1107315584u,1157644800u),vec2<u32>(1157645568u,1157644800u),vec2<u32>(1157646720u,1157644800u),vec2<u32>(1191199232u,1157644800u),vec2<u32>(1191200896u,1157644800u),vec2<u32>(1216365824u,1157644800u),vec2<u32>(1233142272u,1157644800u),vec2<u32>(1006649856u,1157645568u),vec2<u32>(1006651520u,1157645568u),vec2<u32>(1006652288u,1157645568u),vec2<u32>(1157644800u,1157645568u),vec2<u32>(1157646080u,1157645568u),vec2<u32>(1191197696u,1157645568u),vec2<u32>(1191201664u,1157645568u),vec2<u32>(1216365056u,1157645568u),vec2<u32>(1233143552u,1157645568u),vec2<u32>(1233144704u,1157645568u),vec2<u32>(1266696704u,1157645568u),vec2<u32>(1266698368u,1157645568u),vec2<u32>(1006648320u,1157646080u),vec2<u32>(1006650624u,1157646080u),vec2<u32>(1006651776u,1157646080u),vec2<u32>(1107313152u,1157646080u),vec2<u32>(1157645568u,1157646080u),vec2<u32>(1157646464u,1157646080u),vec2<u32>(1191199232u,1157646080u),vec2<u32>(1191200512u,1157646080u),vec2<u32>(1216365824u,1157646080u),vec2<u32>(1233140736u,1157646080u),vec2<u32>(1249921664u,1157646080u),vec2<u32>(1006649856u,1157646464u),vec2<u32>(1006652288u,1157646464u),vec2<u32>(1157643264u,1157646464u),vec2<u32>(1157646080u,1157646464u),vec2<u32>(1191200000u,1157646464u),vec2<u32>(1191201152u,1157646464u),vec2<u32>(1216365056u,1157646464u),vec2<u32>(1266697472u,1157646464u),vec2<u32>(1266698624u,1157646464u),vec2<u32>(1006651520u,1157646720u),vec2<u32>(1107313152u,1157646720u),vec2<u32>(1157645568u,1157646720u),vec2<u32>(1191201664u,1157646720u),vec2<u32>(1216363520u,1157646720u),vec2<u32>(1233143552u,1157646720u),vec2<u32>(1266695168u,1157646720u),vec2<u32>(1006648320u,1157646976u),vec2<u32>(1006650624u,1157646976u),vec2<u32>(1006652288u,1157646976u),vec2<u32>(1157644800u,1157646976u),vec2<u32>(1233144192u,1157646976u),vec2<u32>(1249919488u,1157646976u),vec2<u32>(1006651776u,1157647232u),vec2<u32>(1107313152u,1157647232u),vec2<u32>(1157646976u,1157647232u),vec2<u32>(1191197696u,1157647232u),vec2<u32>(1216366336u,1157647232u),vec2<u32>(1233140736u,1157647232u),vec2<u32>(1006650624u,1191197696u),vec2<u32>(1107313152u,1191197696u),vec2<u32>(1107314432u,1191197696u),vec2<u32>(1107315072u,1191197696u),vec2<u32>(1107315584u,1191197696u),vec2<u32>(1157645568u,1191197696u),vec2<u32>(1191199232u,1191197696u),vec2<u32>(1191200512u,1191197696u),vec2<u32>(1191201152u,1191197696u),vec2<u32>(1216365824u,1191197696u),vec2<u32>(1216366720u,1191197696u),vec2<u32>(1216367488u,1191197696u),vec2<u32>(1233142272u,1191197696u),vec2<u32>(1249920768u,1191197696u),vec2<u32>(1266696704u,1191197696u),vec2<u32>(1006649856u,1191199232u),vec2<u32>(1006651136u,1191199232u),vec2<u32>(1006651776u,1191199232u),vec2<u32>(1107314816u,1191199232u),vec2<u32>(1157644800u,1191199232u),vec2<u32>(1157646080u,1191199232u),vec2<u32>(1216363520u,1191199232u),vec2<u32>(1249917952u,1191199232u),vec2<u32>(1266697472u,1191199232u),vec2<u32>(1266698880u,1191199232u),vec2<u32>(1006648320u,1191200000u),vec2<u32>(1107313920u,1191200000u),vec2<u32>(1157643264u,1191200000u),vec2<u32>(1191200000u,1191200000u),vec2<u32>(1191200896u,1191200000u),vec2<u32>(1233140736u,1191200000u),vec2<u32>(1006649856u,1191200512u),vec2<u32>(1107311616u,1191200512u),vec2<u32>(1107314816u,1191200512u),vec2<u32>(1157644800u,1191200512u),vec2<u32>(1157646080u,1191200512u),vec2<u32>(1157647232u,1191200512u),vec2<u32>(1191197696u,1191200512u),vec2<u32>(1216365056u,1191200512u),vec2<u32>(1216366336u,1191200512u),vec2<u32>(1216367488u,1191200512u),vec2<u32>(1233144192u,1191200512u),vec2<u32>(1266697984u,1191200512u),vec2<u32>(1006651136u,1191200896u),vec2<u32>(1107313152u,1191200896u),vec2<u32>(1107315328u,1191200896u),vec2<u32>(1157645568u,1191200896u),vec2<u32>(1191199232u,1191200896u),vec2<u32>(1233143040u,1191200896u),vec2<u32>(1249917952u,1191200896u),vec2<u32>(1249921152u,1191200896u),vec2<u32>(1006649856u,1191201152u),vec2<u32>(1107311616u,1191201152u),vec2<u32>(1107313920u,1191201152u),vec2<u32>(1157646720u,1191201152u),vec2<u32>(1191200000u,1191201152u),vec2<u32>(1216366720u,1191201152u),vec2<u32>(1233144448u,1191201152u),vec2<u32>(1266697984u,1191201152u),vec2<u32>(1107315328u,1191201408u),vec2<u32>(1216365056u,1191201408u),vec2<u32>(1006649856u,1191201664u),vec2<u32>(1006651136u,1191201664u),vec2<u32>(1157643264u,1191201664u),vec2<u32>(1157645568u,1191201664u),vec2<u32>(1191201152u,1191201664u),vec2<u32>(1006648320u,1216363520u),vec2<u32>(1006651520u,1216363520u),vec2<u32>(1107313920u,1216363520u),vec2<u32>(1157643264u,1216363520u),vec2<u32>(1157646464u,1216363520u),vec2<u32>(1157647232u,1216363520u),vec2<u32>(1191200000u,1216363520u),vec2<u32>(1216365056u,1216363520u),vec2<u32>(1233140736u,1216363520u),vec2<u32>(1266695168u,1216363520u),vec2<u32>(1006650624u,1216365056u),vec2<u32>(1006652288u,1216365056u),vec2<u32>(1107313152u,1216365056u),vec2<u32>(1107314432u,1216365056u),vec2<u32>(1157645568u,1216365056u),vec2<u32>(1191197696u,1216365056u),vec2<u32>(1191201152u,1216365056u),vec2<u32>(1216366336u,1216365056u),vec2<u32>(1233142272u,1216365056u),vec2<u32>(1233144192u,1216365056u),vec2<u32>(1006649856u,1216365824u),vec2<u32>(1006651136u,1216365824u),vec2<u32>(1107311616u,1216365824u),vec2<u32>(1107315072u,1216365824u),vec2<u32>(1157644800u,1216365824u),vec2<u32>(1191200512u,1216365824u),vec2<u32>(1216363520u,1216365824u),vec2<u32>(1233144704u,1216365824u),vec2<u32>(1249920256u,1216365824u),vec2<u32>(1266695168u,1216365824u),vec2<u32>(1006651520u,1216366336u),vec2<u32>(1107313152u,1216366336u),vec2<u32>(1107314432u,1216366336u),vec2<u32>(1157643264u,1216366336u),vec2<u32>(1157645568u,1216366336u),vec2<u32>(1191199232u,1216366336u),vec2<u32>(1191201152u,1216366336u),vec2<u32>(1006648320u,1216366720u),vec2<u32>(1006650624u,1216366720u),vec2<u32>(1157646464u,1216366720u),vec2<u32>(1191201664u,1216366720u),vec2<u32>(1216363520u,1216366720u),vec2<u32>(1266696704u,1216366720u),vec2<u32>(1006651776u,1216366976u),vec2<u32>(1006652288u,1216366976u),vec2<u32>(1157644800u,1216366976u),vec2<u32>(1249920256u,1216366976u),vec2<u32>(1107314432u,1216367232u),vec2<u32>(1191200896u,1216367232u),vec2<u32>(1249917952u,1216367232u),vec2<u32>(1107311616u,1216367488u),vec2<u32>(1107315072u,1216367488u),vec2<u32>(1191197696u,1216367488u),vec2<u32>(1216366336u,1216367488u),vec2<u32>(1233142272u,1216367488u),vec2<u32>(1006650624u,1233140736u),vec2<u32>(1107311616u,1233140736u),vec2<u32>(1107314816u,1233140736u),vec2<u32>(1157645568u,1233140736u),vec2<u32>(1216363520u,1233140736u),vec2<u32>(1216366720u,1233140736u),vec2<u32>(1216367488u,1233140736u),vec2<u32>(1233143040u,1233140736u),vec2<u32>(1249921664u,1233140736u),vec2<u32>(1266698368u,1233140736u),vec2<u32>(1006649856u,1233142272u),vec2<u32>(1006651136u,1233142272u),vec2<u32>(1006651776u,1233142272u),vec2<u32>(1107313920u,1233142272u),vec2<u32>(1157644800u,1233142272u),vec2<u32>(1191200000u,1233142272u),vec2<u32>(1266697472u,1233142272u),vec2<u32>(1006648320u,1233143040u),vec2<u32>(1107313152u,1233143040u),vec2<u32>(1157646080u,1233143040u),vec2<u32>(1191197696u,1233143040u),vec2<u32>(1191201408u,1233143040u),vec2<u32>(1233143552u,1233143040u),vec2<u32>(1006650624u,1233143552u),vec2<u32>(1006652288u,1233143552u),vec2<u32>(1107311616u,1233143552u),vec2<u32>(1157647232u,1233143552u),vec2<u32>(1216366720u,1233143552u),vec2<u32>(1233142272u,1233143552u),vec2<u32>(1249921408u,1233143552u),vec2<u32>(1266697984u,1233143552u),vec2<u32>(1006649856u,1233143936u),vec2<u32>(1006651520u,1233143936u),vec2<u32>(1157643264u,1233143936u),vec2<u32>(1191200000u,1233143936u),vec2<u32>(1216367232u,1233143936u),vec2<u32>(1107313920u,1233144192u),vec2<u32>(1157646976u,1233144192u),vec2<u32>(1233142272u,1233144192u),vec2<u32>(1233143552u,1233144192u),vec2<u32>(1216365824u,1233144448u),vec2<u32>(1006650624u,1233144704u),vec2<u32>(1006651520u,1233144704u),vec2<u32>(1157645568u,1233144704u),vec2<u32>(1107313152u,1249917952u),vec2<u32>(1107314432u,1249917952u),vec2<u32>(1107315072u,1249917952u),vec2<u32>(1191199232u,1249917952u),vec2<u32>(1191200512u,1249917952u),vec2<u32>(1249917952u,1249917952u),vec2<u32>(1006648320u,1249919488u),vec2<u32>(1157643264u,1249919488u),vec2<u32>(1157647232u,1249919488u),vec2<u32>(1249921152u,1249919488u),vec2<u32>(1107313920u,1249920256u),vec2<u32>(1191200896u,1249920256u),vec2<u32>(1216365824u,1249920256u),vec2<u32>(1233144192u,1249920256u),vec2<u32>(1249920256u,1249920256u),vec2<u32>(1266695168u,1249920256u),vec2<u32>(1006648320u,1249920768u),vec2<u32>(1107314816u,1249920768u),vec2<u32>(1157644800u,1249920768u),vec2<u32>(1216363520u,1249920768u),vec2<u32>(1157646720u,1249921152u),vec2<u32>(1216366336u,1249921152u),vec2<u32>(1249920256u,1249921152u),vec2<u32>(1006648320u,1249921408u),vec2<u32>(1006651136u,1249921408u),vec2<u32>(1191200896u,1249921408u),vec2<u32>(1249917952u,1249921408u),vec2<u32>(1006651776u,1249921664u),vec2<u32>(1216363520u,1249921664u),vec2<u32>(1107313152u,1249921920u),vec2<u32>(1107314432u,1249921920u),vec2<u32>(1006648320u,1266695168u),vec2<u32>(1006651520u,1266695168u),vec2<u32>(1006652288u,1266695168u),vec2<u32>(1157643264u,1266695168u),vec2<u32>(1157645568u,1266695168u),vec2<u32>(1191201408u,1266695168u),vec2<u32>(1216363520u,1266695168u),vec2<u32>(1233143936u,1266695168u),vec2<u32>(1249920256u,1266695168u),vec2<u32>(1006650624u,1266696704u),vec2<u32>(1107313152u,1266696704u),vec2<u32>(1157646464u,1266696704u),vec2<u32>(1216366336u,1266696704u),vec2<u32>(1216366976u,1266696704u),vec2<u32>(1006649856u,1266697472u),vec2<u32>(1006651520u,1266697472u),vec2<u32>(1107311616u,1266697472u),vec2<u32>(1107315328u,1266697472u),vec2<u32>(1157644800u,1266697472u),vec2<u32>(1191197696u,1266697472u),vec2<u32>(1233142272u,1266697472u),vec2<u32>(1006650624u,1266697984u),vec2<u32>(1191200000u,1266697984u),vec2<u32>(1191201152u,1266697984u),vec2<u32>(1233143552u,1266697984u),vec2<u32>(1006649856u,1266698368u),vec2<u32>(1006651776u,1266698368u),vec2<u32>(1107314432u,1266698368u),vec2<u32>(1157643264u,1266698368u),vec2<u32>(1233140736u,1266698368u),vec2<u32>(1157645568u,1266698624u),vec2<u32>(1216365824u,1266698624u),vec2<u32>(1006650624u,1266698880u),vec2<u32>(1191199232u,1266698880u),vec2<u32>(1006648320u,1266699136u));"],["ops/_shared/iq-grid-iq3s.0812aa7130.wgsl.jinja",`{% set iqSfx = "" %}const IQ_MAG{{ iqSfx }}:array<vec2<u32>,512>=array<vec2<u32>,512>(
{% include "ops/_shared/iq-grid-iq3s.68075e892b.wgsl.jinja" %}fn iq3mag{{ iqSfx }}(i: u32) -> vec4<f32> {
let e=IQ_MAG{{ iqSfx }}[i];return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}`],["ops/_shared/iq-grid.b22b84ae14.wgsl.jinja",'{% set iqSfx = "" %}{% set iq2xxs = iq2xxs | default(false) %}{% set iq2xs = iq2xs | default(false) %}{% set iq2s = iq2s | default(false) %}{% set iq3s = iq3s | default(false) %}{% set iq3xxs = iq3xxs | default(false) %}{% set iq1m = iq1m is defined and iq1m %}{% if iq1m %}{% include "ops/_shared/iq-grid-iq1s.498a1afa8d.wgsl.jinja" %}{% elif iq2xxs %}{% include "ops/_shared/iq-grid-iq2xxs.cf9c8c7724.wgsl.jinja" %}{% elif iq2xs %}{% include "ops/_shared/iq-grid-iq2xs.fc760188d7.wgsl.jinja" %}{% elif iq2s %}{% include "ops/_shared/iq-grid-iq2s.fa1e8b8eff.wgsl.jinja" %}{% elif iq3xxs %}{% include "ops/_shared/iq-grid-iq3xxs.295293cefb.wgsl.jinja" %}{% elif iq3s %}{% include "ops/_shared/iq-grid-iq3s.0812aa7130.wgsl.jinja" %}{% endif %}'],["ops/_shared/iqx-raw-dequant-helpers.6dec8810cb.wgsl.jinja",`{% set iqSfx = "" %}{% set iqPrimitives = true %}{% set iqInferredPrimitiveMode =
"explicit8"if iq2s|default(false)else"explicit4"if iq3s|default(false)else"none"if iq1m|default(false)else"parity8"
%}{% set iqPrimitiveMode = iqPrimitiveMode | default(iqInferredPrimitiveMode) %}{% if iqPrimitives and iqPrimitiveMode != "none" %}fn sgx(v:vec4<f32>,s:u32,sh:vec4<u32>)->vec4<f32>{return bitcast<vec4<f32>>(bitcast<vec4<u32>>(v)^((vec4<u32>(s)<<sh)&vec4<u32>(0x80000000u)));}const SGX_LO:vec4<u32>=vec4<u32>(31u,30u,29u,28u);
{% if iqPrimitiveMode != "explicit4" %}const SGX_HI:vec4<u32>=vec4<u32>(27u,26u,25u,24u);
{% endif %}{% if iqPrimitiveMode == "parity8" %}fn ksg(sv:u32)->u32{return sv|((countOneBits(sv)&1u)<<7u);}
{% endif %}{% endif %}`],["ops/_shared/lut2-block-scale.5bf39fd59c.wgsl.jinja","{% macro lut2_native_scale(block, metadata, codebook) %}{% if codebook == 9 %}unpack2x16float({{ metadata }}[({{ block }}) >> 3u])[(({{ block }}) >> 2u) & 1u]{% else %}unpack2x16float({{ metadata }}[(({{ block }}) >> 3u) * 3u + ((({{ block }}) & 7u) >> 2u)])[(({{ block }}) >> 1u) & 1u]{% endif %}{% endmacro %}"],["ops/_shared/q4nl-dequant-helpers.dd92a6a387.wgsl.jinja",`{% set lutSfx = "" %}{% set lutTable = "LUT_GRID" %}{% if lutId and lutId == 2 %}{% set gridRow0 = "8.0, 25.0, 43.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "-8.0, -25.0, -43.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 3 %}{% set gridRow0 = "1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0" %}{% set gridRow1 = "-1.0, -3.0, -5.0, -7.0, -9.0, -11.0, -13.0, -15.0" %}{% elif lutId and lutId == 5 %}{% set gridRow0 = "4.0, 12.0, 20.0, 28.0, 36.0, 44.0, 52.0, 62.0" %}{% set gridRow1 = "-4.0, -12.0, -20.0, -28.0, -36.0, -44.0, -52.0, -62.0" %}{% elif lutId and lutId == 4 %}{% set gridRow0 = "-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0" %}{% set gridRow1 = "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 6 %}{% set gridRow0 = "0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0" %}{% set gridRow1 = "0.0, -1.0, -2.0, -3.0, -4.0, -6.0, -8.0, -12.0" %}{% elif lutId and lutId in [8, 9] %}{% set gridRow0 = "-1.0, 0.0, 1.0, 2.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 7 %}{% set gridRow0 = "-9.0, -1.0, 7.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "-7.0, 1.0, 9.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% else %}{% set gridRow0 = "-127.0, -104.0, -83.0, -65.0, -49.0, -35.0, -22.0, -10.0" %}{% set gridRow1 = "1.0, 13.0, 25.0, 38.0, 53.0, 69.0, 89.0, 113.0" %}{% endif %}const LUT_GRID{{ lutSfx }}=array<f32,16>({{ gridRow0 }},{{ gridRow1 }});
{% if emitQuantValueHelpers | default(true) %}fn q4nl{{ lutSfx }}(v:u32)->f32{return {{ lutTable }}[v&15u];}fn q4_lo{{ lutSfx }}(word:u32)->vec4<f32>{return vec4<f32>(q4nl{{ lutSfx }}(word),q4nl{{ lutSfx }}(word>>8u),q4nl{{ lutSfx }}(word>>16u),q4nl{{ lutSfx }}(word>>24u));}
{% endif %}{% if lut2 is defined and lut2 %}fn q2q{{ lutSfx }}(word:u32,sh:u32,db:u32)->vec4<f32>{let v=word>>sh;
{% if lutId in [8, 9] %}let base={{ lutTable }}[db];let step={{ lutTable }}[db+1u]-base;let codes=(vec4<u32>(v)>>vec4<u32>(0u,2u,4u,6u))&vec4<u32>(3u);return(bitcast<vec4<f32>>(codes|vec4<u32>(0x4b000000u))-vec4<f32>(8388608.0))*step+vec4<f32>(base);
{% else %}return vec4<f32>({{ lutTable }}[(v&3u)+db],{{ lutTable }}[((v>>2u)&3u)+db],{{ lutTable }}[((v>>4u)&3u)+db],{{ lutTable }}[((v>>6u)&3u)+db]);
{% endif %}}
{% endif %}{% if lutc is defined and lutc and lutId not in [8, 9] %}fn lutScale{{ lutSfx }}(c:u32)->f32{
{% if lutId == 1 or lutId == 4 %}return f32(c)-32.0;
{% elif lutId == 2 %}return(0.5+f32(c))*0.25;
{% elif lutId == 3 %}return 1.0+2.0*f32(c);
{% elif lutId == 5 %}return(0.5+f32(c))*0.5;
{% elif lutId == 6 and lutc16 %}if(c==0u||c==0x7fu){return 0.0;}let exp=(c>>3u)&0x0fu;let man=c&7u;if(exp==0u){return f32(man)/1024.0;}return bitcast<f32>(((exp+119u)<<23u)|(man<<20u));
{% elif lutId == 6 %}return bitcast<f32>(select((c-1u)<<23u,0x00200000u<<c,c<2u));
{% else %}return f32(c)*0.125;
{% endif %}}
{% endif %}`],["ops/_shared/packed-half-block-f32.baa86c90c3.wgsl.jinja",`{% macro word(i) %}w[({{ i }}) >> 2u][({{ i }}) & 3u]{% endmacro %}{% if packedIqRaw | default(0) %}{% if packedIqRaw == 3 %}let qw_p={{ word("blk * 3u + blk_half") }};let sw_p={{ word("blk * 3u + 2u") }};let d_p=f32(scales[blk>>3u])*(0.5+f32(sw_p>>28u))*0.5;let sa_p=ksg((sw_p>>(14u*blk_half))&127u);let sb_p=ksg((sw_p>>(14u*blk_half+7u))&127u);let hv0=d_p*sgx(iq3mag(qw_p&0xFFu),sa_p,SGX_LO);let hv1=d_p*sgx(iq3mag((qw_p>>8u)&0xFFu),sa_p,SGX_HI);let hv2=d_p*sgx(iq3mag((qw_p>>16u)&0xFFu),sb_p,SGX_LO);let hv3=d_p*sgx(iq3mag(qw_p>>24u),sb_p,SGX_HI);
{% elif packedIqRaw == 4 %}let sbw_p=(blk>>3u)*27u;let k_p=blk&7u;let qw_p={{ word("sbw_p + 3u * k_p + blk_half") }};let sg_p={{ word("sbw_p + 3u * k_p + 2u") }}>>(16u*blk_half);let qh_p=(({{ word("sbw_p + 24u + (k_p >> 2u)") }}>>((k_p&3u)*8u))&0xFFu)>>(4u*blk_half);let d_p=f32(scales[blk>>3u])*(1.0+2.0*f32(({{ word("sbw_p + 26u") }}>>(k_p*4u))&0xFu));
{% for j in range(4) %}let hv{{ j }}=d_p*sgx(iq3mag(((qw_p>>{{ 8 * j }}u)&0xFFu)|(((qh_p>>{{ j }}u)&1u)<<8u)),sg_p>>{{ 4 * j }}u,SGX_LO);
{% endfor %}{% elif packedIqRaw in [2, 5, 6] %}{% if packedIqRaw == 2 %}let qw_p={{ word("blk * 2u") }};let sw_p={{ word("blk * 2u + 1u") }};let d_p=f32(scales[blk>>3u])*(0.5+f32(sw_p>>28u))*0.25;
{% for c in range(2) %}let g{{ c }}_p=iq_grid((qw_p>>(16u*blk_half+{{ 8 * c }}u))&0xFFu);let s{{ c }}_p=ksg((sw_p>>(14u*blk_half+{{ 7 * c }}u))&127u);
{% endfor %}{% elif packedIqRaw == 6 %}let sbw_p=(blk>>3u)*18u;let k_p=blk&7u;let cw_p={{ word("sbw_p + 2u * k_p + blk_half") }};let sc_p=({{ word("sbw_p + 16u + (k_p >> 2u)") }}>>((k_p&3u)*8u))&0xFFu;let d_p=f32(scales[blk>>3u])*(0.5+f32((sc_p>>(4u*blk_half))&0xFu))*0.25;
{% for c in range(2) %}let u{{ c }}_p=(cw_p>>{{ 16 * c }}u)&0xFFFFu;let g{{ c }}_p=iq_grid(u{{ c }}_p&0x1FFu);let s{{ c }}_p=ksg((u{{ c }}_p>>9u)&127u);
{% endfor %}{% else %}let sbw_p=(blk>>3u)*20u;let k_p=blk&7u;let qw_p={{ word("sbw_p + 2u * k_p") }};let sw_p={{ word("sbw_p + 2u * k_p + 1u") }};let qh_p=({{ word("sbw_p + 16u + (k_p >> 2u)") }}>>((k_p&3u)*8u))&0xFFu;let sc_p=({{ word("sbw_p + 18u + (k_p >> 2u)") }}>>((k_p&3u)*8u))&0xFFu;let d_p=f32(scales[blk>>3u])*(0.5+f32((sc_p>>(4u*blk_half))&0xFu))*0.25;
{% for c in range(2) %}let c{{ c }}_p=2u*blk_half+{{ c }}u;let g{{ c }}_p=iq_grid(((qw_p>>(8u*c{{ c }}_p))&0xFFu)|(((qh_p>>(2u*c{{ c }}_p))&3u)<<8u));let s{{ c }}_p=sw_p>>(8u*c{{ c }}_p);
{% endfor %}{% endif %}let hv0=d_p*sgx(vec4<f32>(unpack4xU8(g0_p.x)),s0_p,SGX_LO);let hv1=d_p*sgx(vec4<f32>(unpack4xU8(g0_p.y)),s0_p,SGX_HI);let hv2=d_p*sgx(vec4<f32>(unpack4xU8(g1_p.x)),s1_p,SGX_LO);let hv3=d_p*sgx(vec4<f32>(unpack4xU8(g1_p.y)),s1_p,SGX_HI);
{% else %}{% if packedIqRaw == 7 %}let sbw_p=(blk>>3u)*12u;let k_p=blk&7u;let qw_p={{ word("sbw_p + k_p") }};let mt_p=({{ word("sbw_p + 8u + (k_p >> 1u)") }}>>(16u*(k_p&1u)))&0xFFFFu;let sg_p=((mt_p>>12u)&1u)*0x88u;let mw_p=((mt_p&7u)|(((mt_p>>3u)&7u)<<4u)|sg_p)|((((mt_p>>6u)&7u)|(((mt_p>>9u)&7u)<<4u)|sg_p)<<8u)|((mt_p>>13u)<<16u)|((mt_p>>13u)<<19u);
{% elif packedIqRaw == 8 %}let sbw_p=(blk>>3u)*14u;let k_p=blk&7u;let qw_p={{ word("sbw_p + k_p") }};let mw_p=(({{ word("sbw_p + 8u + (k_p >> 1u)") }}>>(16u*(k_p&1u)))&0xFFFFu)|((({{ word("sbw_p + 12u + (k_p >> 2u)") }}>>(8u*(k_p&3u)))&0x3Fu)<<16u);
{% else %}let qw_p={{ word("blk * 2u") }};let mw_p={{ word("blk * 2u + 1u") }};
{% endif %}let d_p=f32(scales[blk>>3u])*f32(2u*((mw_p>>(16u+3u*blk_half))&7u)+1u);let h_p=(mw_p>>(8u*blk_half))&0xFFu;let g0_p=iq_grid(((qw_p>>(16u*blk_half))&0xFFu)|((h_p&7u)<<8u));let g1_p=iq_grid(((qw_p>>(16u*blk_half+8u))&0xFFu)|(((h_p>>4u)&7u)<<8u));let t0_p=select(0.125,-0.125,(h_p&0x08u)!=0u);let t1_p=select(0.125,-0.125,(h_p&0x80u)!=0u);let hv0=d_p*(vec4<f32>(unpack4xI8(g0_p.x))+t0_p);let hv1=d_p*(vec4<f32>(unpack4xI8(g0_p.y))+t0_p);let hv2=d_p*(vec4<f32>(unpack4xI8(g1_p.x))+t1_p);let hv3=d_p*(vec4<f32>(unpack4xI8(g1_p.y))+t1_p);
{% endif %}{% elif packedFormat == "q4_k_native_compact" %}let sb_p=blk>>3u;let j_p=blk&7u;let pair_p=scales[sb_p*5u+1u+(j_p>>1u)]>>((j_p&1u)*12u);let dm_p=unpack2x16float(scales[sb_p*5u]);let dl_p=dm_p.x*f32(pair_p&63u);let ml_p=dm_p.y*f32((pair_p>>6u)&63u);let qw_p=w[blk]>>vec4<u32>(4u*blk_half);
{% for c in ["x", "y", "z", "w"] %}let p_p{{ loop.index0 }}=dl_p*vec4<f32>(unpack4xU8(qw_p.{{ c }}&0x0F0F0F0Fu));let hv{{ loop.index0 }}=p_p{{ loop.index0 }}-ml_p;
{% endfor %}{% elif packedFormat == "q6_k" %}let sb_p=blk>>3u;let j_p=blk&7u;let g_p=sb_p*13u;let qw_p=w[g_p+j_p]>>vec4<u32>(4u*blk_half);let hi_p=w[g_p+8u+(j_p>>1u)][(j_p&1u)*2u+blk_half];let s_p=f32(scales[sb_p])*f32(unpack4xI8(w[g_p+12u][j_p>>1u])[(j_p&1u)*2u+blk_half]);
{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=s_p*(vec4<f32>(unpack4xU8((qw_p.{{ c }}&0x0F0F0F0Fu)|(((hi_p>>{{ 2 * loop.index0 }}u)&0x03030303u)<<4u)))-32.0);
{% endfor %}{% elif packedFormat == "q5_k" %}let sb_p=blk>>3u;let j_p=blk&7u;let g_p=sb_p*11u;let qw_p=w[g_p+j_p]>>vec4<u32>(4u*blk_half);let qh_p=w[g_p+8u+(j_p>>2u)][j_p&3u]>>(16u*blk_half);let sm_p=w[g_p+10u];let sc_p=f32(scales[sb_p*2u])*f32((sm_p[j_p>>2u]>>((j_p&3u)*8u))&0xFFu);let mn_p=f32(scales[sb_p*2u+1u])*f32((sm_p[2u+(j_p>>2u)]>>((j_p&3u)*8u))&0xFFu);
{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=sc_p*vec4<f32>(unpack4xU8((qw_p.{{ c }}&0x0F0F0F0Fu)|((((qh_p>>{{ 4 * loop.index0 }}u)&15u)*0x02040810u)&0x10101010u)))-mn_p;
{% endfor %}{% elif packedFormat == "q3_k" %}let sb_p=blk>>3u;let j_p=blk&7u;let g_p=sb_p*7u;let cw_p=w[g_p+(j_p>>1u)][(j_p&1u)*2u+blk_half];let hw_p=w[g_p+4u+(j_p>>2u)][j_p&3u]>>(4u*blk_half);let s_p=f32(scales[sb_p])*f32(unpack4xI8(w[g_p+6u][j_p>>1u])[(j_p&1u)*2u+blk_half]);
{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=s_p*(vec4<f32>(unpack4xU8(((cw_p>>{{ 2 * loop.index0 }}u)&0x03030303u)|(((hw_p>>{{ loop.index0 }}u)&0x01010101u)<<2u)))-4.0);
{% endfor %}{% elif packedFormat == "q2_k" %}let sb_p=blk>>3u;let j_p=blk&7u;let g_p=sb_p*5u;let cw_p=w[g_p+(j_p>>1u)][(j_p&1u)*2u+blk_half];let b_p=(w[g_p+4u][j_p>>1u]>>((j_p&1u)*16u+8u*blk_half))&0xFFu;let s_p=f32(scales[sb_p*2u])*f32(b_p&15u);let m_p=f32(scales[sb_p*2u+1u])*f32(b_p>>4u);
{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=s_p*vec4<f32>(unpack4xU8((cw_p>>{{ 2 * loop.index0 }}u)&0x03030303u))-m_p;
{% endfor %}{% elif packedFormat in ["q8_0", "iq"] %}let qw_p=w[blk*2u+blk_half];let s_p=f32(scales[{{ "blk * 2u + blk_half" if packedFormat == "iq" else "blk" }}]);
{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=vec4<f32>(unpack4xI8(qw_p.{{ c }}))*s_p;
{% endfor %}{% elif packedFormat in ["lut2_32", "lut2_16"] %}let wi_p=blk*2u+blk_half;let cw_p=w[wi_p>>2u][wi_p&3u];
{% if lutId in [8, 9] %}let s_p={{ lut2_native_scale("blk", "scales", lutId) }};let dl_p=0u;let dh_p=0u;
{% else %}let m_p=(blk>>3u)*{{ 5 if lutc16 else 3 }}u;
{% if lutc16 %}let c_p=(blk&7u)*2u+blk_half;let b_p=(scales[m_p+(c_p>>2u)]>>((c_p&3u)*8u))&255u;
{% else %}let b_p=(scales[m_p+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u;
{% endif %}let s_p=bitcast<f32>(scales[m_p+{{ 4 if lutc16 else 2 }}u])*lutScale(b_p&15u);let dl_p=((b_p>>4u)&1u)*8u;let dh_p=((b_p>>{{ 5 if lutc16 else 4 }}u)&1u)*8u;
{% endif %}let hv0=s_p*q2q(cw_p,0u,dl_p);let hv1=s_p*q2q(cw_p,8u,dl_p);let hv2=s_p*q2q(cw_p,16u,dh_p);let hv3=s_p*q2q(cw_p,24u,dh_p);
{% elif packedFormat in ["lut32", "lut16", "lut32_compact", "lut16_compact"] %}let qw_p=w[blk]>>vec4<u32>(4u*blk_half);
{% if packedFormat == "lut32_compact" %}let m_p=(blk>>3u)*3u;let s_p=bitcast<f32>(scales[m_p+2u])*lutScale((scales[m_p+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u);
{% elif packedFormat == "lut16_compact" %}let m_p=(blk>>3u)*5u;let s_p=bitcast<f32>(scales[m_p+4u])*lutScale((scales[m_p+((blk&7u)>>1u)]>>((blk&1u)*16u+8u*blk_half))&255u);
{% else %}let s_p=f32(scales[{{ "blk * 2u + blk_half" if packedFormat == "lut16" else "blk" }}]);
{% endif %}{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=s_p*q4_lo(qw_p.{{ c }});
{% endfor %}{% else %}let qw_p=w[blk]>>vec4<u32>(4u*blk_half);
{% if packedFormat == "q4_k" %}let s_p=f32(scales[blk*2u]);let b_p=f32(scales[blk*2u+1u]);
{% else %}let s_p=f32(scales[blk]);
{% endif %}{% for c in ["x", "y", "z", "w"] %}let hv{{ loop.index0 }}=s_p*(vec4<f32>(unpack4xU8(qw_p.{{ c }}&0x0F0F0F0Fu))-8.0){{ " + b_p" if packedFormat == "q4_k" else "" }};
{% endfor %}{% endif %}`],["ops/_shared/subgroup-matrix-dynamic-step.501544938c.wgsl.jinja",`{% set dynamicATiles = aTilesPerSubgroup if aTilesPerSubgroup is defined else 2 %}{% set matrixAccumulator = matrixAccumulator | default("matC") %}{% set matrixStepOffset = matrixStepOffset | default(0) %}{% set matrixStep = "step" if matrixStepOffset == 0 else "(step + " ~ matrixStepOffset ~ "u)" %}{% set directA = directAStorage is defined and directAStorage %}{% set B_BANK = "b_bank_offset + " if doubleBufferedB is defined and doubleBufferedB else "" %}{% if directA %}{% set matrixAStorage = matrixAStorage | default("x") %}{% set matrixAStride = matrixAStride | default("IN_F") %}let matrix_a_offset=(a_global_base+base_A)*{{ matrixAStride }}+kidx+{{ matrixStep }};
{% else %}let matrix_a_offset=subtile_idy*SUB_ROWS*TILE_K+{{ matrixStep }};
{% endif %}{% for m in range(dynamicATiles) %}{% if directA %}var matA{{ m }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&" ~ (matrixAStorage), ("matrix_a_offset" ~ (" + " ~ (m * 8) ~ "u * " ~ matrixAStride if m > 0 else "")), matrixAStride) }};
{% else %}var matA{{ m }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&tile_A", ("matrix_a_offset" ~ (" + " ~ (m * 8) ~ "u * TILE_K" if m > 0 else "")), "TILE_K") }};
{% endif %}{% endfor %}let matrix_b_offset={{ B_BANK }}subtile_idx*SUB_COLS*TILE_K+{{ matrixStep }};
{% for n in range(bTilesPerSubgroup) %}var matB{{ n }}:{{ matrixType(matrixSoftware, "right", operandScalar) }}={{ matrixLoad(matrixSoftware, "right", operandScalar, "col_major", "&tile_B", "matrix_b_offset + " ~ (n * 8) ~ "u * TILE_K", "TILE_K") }};
{% endfor %}{% for m in range(dynamicATiles) %}{% for n in range(bTilesPerSubgroup) %}{{ matrixAccumulator }}{{ m }}{{ n }}={{ matrixMma(matrixSoftware, accScalar, "matA" ~ (m), "matB" ~ (n), (matrixAccumulator) ~ (m) ~ (n)) }};
{% endfor %}{% endfor %}`],["ops/_shared/q4-dequant-helpers.a65f490371.wgsl.jinja",`{% if nibbleHelpers | default(true) %}{% if quantBits == 8 %}fn q8dot(word:u32,xv:vec4<f32>)->f32{return dot(vec4<f32>(unpack4xI8(word)),xv);}
{% else %}fn q4_lo(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}fn q4_hi(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}fn q4wp(word:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q4_lo(word))+dot(hi,q4_hi(word));}
{% endif %}{% endif %}{% if q6k %}fn q6_lo(word:u32,hLo:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|(((hLo>>(2u*w))&0x03030303u)<<4u)))-32.0;}fn q6_hi(word:u32,hHi:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|(((hHi>>(2u*w))&0x03030303u)<<4u)))-32.0;}
{% endif %}`],["ops/_shared/q4-block-dot-flags.df42fc7969.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sScales = "q4_scales" %}{% set sMeta = packMeta %}{% set sQuantBits = quantBits %}{% set sHasMin = 0 %}{% set sQ6k = q6k | default(false) %}{% set sQ5k = 0 %}{% set sQ4knc = q4knCompact | default(false) %}{% set sLutSfx = "" %}'],["ops/_shared/q4-block-dot.014fbf5d2a.wgsl.jinja",`let {{ sPre }}blk={{ sBlk }};
{% if sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ6k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));
{% elif sQuantBits == 8 %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}w0={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}w1={{ sBits }}[{{ sPre }}blk*2u+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3)+q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-decode-row-block-accumulate.98693154e0.wgsl.jinja",`{% set packMeta = packMeta | default("q4_metadata") %}{% set accPrefix = accPrefix | default("acc") %}{% set rowBase = rowBase | default("base0") %}{% include "ops/_shared/q4-block-dot-flags.df42fc7969.wgsl.jinja" %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.014fbf5d2a.wgsl.jinja" %}{% endfor %}`],["ops/_shared/moe-routed-shared-packed-compat.1514a9f3cc.wgsl.jinja",'{% include "ops/_shared/packed-4x8-compat.0c5ce8f6e4.wgsl.jinja" %}'],["ops/_shared/tree-fold-inplace.8e1bf311b5.wgsl.jinja",`{% macro wgsl_tree_fold_stmt(a, op, idx, svar) %}{% if op == "max" or op == "min" %}{{ a }}[{{ idx }}] = {{ op }}({{ a }}[{{ idx }}], {{ a }}[{{ idx }} + {{ svar }}]);{% else %}{{ a }}[{{ idx }}] = {{ a }}[{{ idx }}] {{ "*" if op == "prod" else "+" }} {{ a }}[{{ idx }} + {{ svar }}];{% endif %}{% endmacro %}{% macro wgsl_tree_fold(arrays, op="add", idx="lid", wg="WORKGROUP_SIZE", svar="stride", typed=false, form="tail", barrierFirst=false, reuse=false) %}var {{ svar }}{{ ": u32 " if typed else " " }}={{ wg }}/2u;loop{if({{ svar }}==0u){break;}if({{ idx }}<{{ svar }}){
{% for a in arrays %}{{ wgsl_tree_fold_stmt(a, op, idx, svar) }}
{% endfor %}}workgroupBarrier();{{ svar }}={{ svar }}/2u;
  }{% endmacro %}`],["ops/_shared/q4-dequant-helpers.8aab4e5b5a.wgsl.jinja","fn q8dot(word:u32,xv:vec4<f32>)->f32{return dot(vec4<f32>(unpack4xI8(word)),xv);}"],["ops/_shared/subgroup-local-layout-proof.e89550015d.wgsl.jinja","var<workgroup>subgroupLocalLayoutRejected:atomic<u32>;var<workgroup>subgroupLocalLayoutAcceptedState:u32;fn subgroupLocalLayoutAccepted(localIndex:u32,memberValid:bool)->bool{let valid=subgroupAll(memberValid);if(subgroupElect()&&!valid){atomicStore(&subgroupLocalLayoutRejected,1u);}workgroupBarrier();if(localIndex==0u){subgroupLocalLayoutAcceptedState=select(0u,1u,atomicLoad(&subgroupLocalLayoutRejected)==0u);}return workgroupUniformLoad(&subgroupLocalLayoutAcceptedState)!=0u;}"],["ops/_shared/subgroup-local-fold.5eb84b12bf.wgsl.jinja",`{% include "ops/_shared/subgroup-local-layout-proof.e89550015d.wgsl.jinja" %}var<workgroup> localFoldValues: array<f32, {{ localFoldWorkgroupSize }}>;
fn sumLocalSubgroup(value:f32,tid:u32,nativeTopology:bool)->f32{if(nativeTopology){return subgroupAdd(value);}localFoldValues[tid]=value;workgroupBarrier();let first=(tid/{{ localFoldWidth }}u)*{{ localFoldWidth }}u;var total=0.0;for(var i=0u;i<{{ localFoldWidth }}u;i+=1u){total+=localFoldValues[first+i];}workgroupBarrier();return total;}`],["ops/_shared/reduce-tree.e93bd4c37a.wgsl.jinja",`{% macro wgsl_tree_reduce_f32(name, mode, buffer="partial", wg="WG", trailingBarrier=true, valueType="f32") %}fn {{ name }}(value:{{ valueType }},tid:u32)->{{ valueType }} { {{ buffer }}[tid]=value;workgroupBarrier();var n:u32={{ wg }};loop{let half=(n+1u)/2u;if(tid<n-half){ {{ buffer }}[tid]=max({{ buffer }}[tid],{{ buffer }}[tid+half]);}workgroupBarrier();n=half;if(n==1u){break;} }let reduced={{ buffer }}[0];return reduced;}
{% endmacro %}`],["ops/_shared/reduce-arg-tie-rule.55e239089f.wgsl.jinja",'{% macro arg_merge_wins(v, i, best, bestIdx) %}{{ i }} != SENTINEL_IDX && ({{ bestIdx }} == SENTINEL_IDX || {{ v }} {{ ">" if modeSpec == "max" else "<" }} {{ best }} || ({{ v }} == {{ best }} && {{ i }} {{ ">" if selectLastIndex else "<" }} {{ bestIdx }})){% endmacro %}'],["ops/_shared/pcg-hash.2e09754c20.wgsl.jinja","fn pcg_hash(input:u32)->u32{let state=input*747796405u+2891336453u;let word=((state>>((state>>28u)+4u))^state)*277803737u;return(word>>22u)^word;}"]]),Lm=new Map([["ai.onnx.Add",{manifest:{domain:"ai.onnx",name:"Add",sinceVersion:14,inputs:{a:{onnx:"A",dtype:"T"},b:{onnx:"B",dtype:"T"}},outputs:{c:{onnx:"C",dtype:"T",rank:"max(ranks.a, ranks.b)",shape:"broadcastShape(shapes.a, shapes.b)"}},typeConstraints:{T:["float32","float16","int32","uint32","int8","uint8"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_shape_vec4",priority:20,when:["sameShape(shapes.a, shapes.c)","sameShape(shapes.b, shapes.c)","numel(shapes.c) > 0","numel(shapes.c) % 4 == 0","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Add.Vec4",shader:"ops/_shared/binary-vec4.wgsl.jinja",derive:{op:'"add"',cDtype:"tensorDtypes.c",vec4PerThread:"4 if numel(shapes.c) * dtypeBytes(tensorDtypes.c) <= 16777216 else 1"},bindings:["a","b","c_binary","params"],dispatch:{threads:"ceilDiv(numel(shapes.c) / 4, 4 if numel(shapes.c) * dtypeBytes(tensorDtypes.c) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"broadcast_vec4",priority:10,when:["ranks.a <= ranks.c","ranks.b <= ranks.c","ranks.c >= 1","numel(shapes.c) % 4 == 0","numel(shapes.c) >= 4","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",aElement:'"vec4<" ~ dtypes.T ~ ">" if sameShape(shapes.a, shapes.c) else dtypes.T',bElement:'"vec4<" ~ dtypes.T ~ ">" if sameShape(shapes.b, shapes.c) else dtypes.T',vec4Scalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Add",shader:"ops/_shared/binary-broadcast-vec4.wgsl.jinja",derive:{aShape:"shapes.a",bShape:"shapes.b",cShape:"shapes.c",aRank:"ranks.a",bRank:"ranks.b",cRank:"ranks.c",op:'"add"',cDtype:"tensorDtypes.c",periodicTableCap:"min(floor(tunables.WORKGROUP_SIZE / 2), floor(device.limits.maxComputeWorkgroupStorageSize / 32)) if numel(shapes.c) >= 4 * tunables.WORKGROUP_SIZE else 0"},bindings:["a_element","b_element","c_2_binary","params"],dispatch:{threads:"numel(shapes.c) / 4",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"broadcast",when:["ranks.a <= ranks.c","ranks.b <= ranks.c","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Add",shader:"ops/_shared/binary-broadcast.wgsl.jinja",derive:{aShape:"shapes.a",bShape:"shapes.b",cShape:"shapes.c",aRank:"ranks.a",bRank:"ranks.b",cRank:"ranks.c",op:'"add"',cDtype:"tensorDtypes.c",itemsPerInvocation:4},bindings:["a_scalar","b_scalar","c_scalar","params_count"],dispatch:{threads:"ceilDiv(numel(shapes.c), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{a:{elementType:"$vectorScalar"},b:{elementType:"$vectorScalar"},c_binary:{elementType:"$vectorScalar",name:"c"},params:{struct:[{name:"count",type:"u32",value:"numel(shapes.c) / 4"}]},b_scalar:{name:"b",elementType:"$scalar"},a_scalar:{name:"a",elementType:"$scalar"},a_element:{name:"a",elementType:"$aElement"},b_element:{name:"b",elementType:"$bElement"},c_2_binary:{name:"c",elementType:"$vec4Scalar"},c_scalar:{name:"c",elementType:"$scalar"},params_count:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.c)"}]}}},assets:[["ops/_shared/binary-vec4.wgsl.jinja",`{% set binaryOperator = "+" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}    let av = a[i];
let bv=b[i];
{% if scalar == "i32" or scalar == "u32" %}let r=av {{ binaryOperator }} bv;
{% if cDtype == "int8" %}c[i]=(r<<vec4<u32>(24u))>>vec4<u32>(24u);
{% elif cDtype == "uint8" %}c[i]=r&vec4<u32>(0xFFu);
{% else %}c[i]=r;
{% endif %}{% elif scalar == "f16" %}c[i]=vec4<f16>(vec4<f32>(av){{ binaryOperator }} vec4<f32>(bv));
{% else %}c[i]=av {{ binaryOperator }} bv;
{% endif %}{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/binary-broadcast-vec4.wgsl.jinja",`{% include "ops/_shared/broadcast-offset.9034aa7baf.wgsl.jinja" %}{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{% include "ops/_shared/binary-broadcast-operand.5796cb977c.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set a_numel = namespace(value=1) %}{% for d in aShape %}{% set a_numel.value = a_numel.value * d %}{% endfor %}{% set b_numel = namespace(value=1) %}{% for d in bShape %}{% set b_numel.value = b_numel.value * d %}{% endfor %}{% set c_numel = namespace(value=1) %}{% for d in cShape %}{% set c_numel.value = c_numel.value * d %}{% endfor %}{% set a_same = namespace(value=(aRank == cRank)) %}{% if a_same.value %}{% for axis in range(cRank) %}{% if aShape[axis] != cShape[axis] %}{% set a_same.value = false %}{% endif %}{% endfor %}{% endif %}{% set b_same = namespace(value=(bRank == cRank)) %}{% if b_same.value %}{% for axis in range(cRank) %}{% if bShape[axis] != cShape[axis] %}{% set b_same.value = false %}{% endif %}{% endfor %}{% endif %}{% set a_inner = aShape[aRank - 1] if aRank >= 1 else 1 %}{% set b_inner = bShape[bRank - 1] if bRank >= 1 else 1 %}{% set c_inner = cShape[cRank - 1] %}{% set crosses_inner_rows = c_inner % 4 != 0 %}{% if a_numel.value == 1 %}{% set a_mode = "scalar" %}{% elif a_inner == 1 %}{% set a_mode = "splat" %}{% else %}{% set a_mode = "contig" %}{% endif %}{% if b_numel.value == 1 %}{% set b_mode = "scalar" %}{% elif b_inner == 1 %}{% set b_mode = "splat" %}{% else %}{% set b_mode = "contig" %}{% endif %}{% set a_table = namespace(period=1, matching=true, active=false) %}{% set b_table = namespace(period=1, matching=true, active=false) %}{{ periodic_geometry(a_table, aShape, aRank, a_same.value, a_numel.value, cShape, cRank, crosses_inner_rows, periodicTableCap) }} {{ periodic_geometry(b_table, bShape, bRank, b_same.value, b_numel.value, cShape, cRank, crosses_inner_rows, periodicTableCap) }}
{% if not a_same.value and a_mode != "scalar" and not a_table.active %}{{ offset_fn("a_offset", aShape, aRank, false, a_numel.value, cShape, cRank, c_numel.value) }}
{% endif %}{% if not b_same.value and b_mode != "scalar" and not b_table.active and not (nativeTrailingB | default(false)) %}{{ offset_fn("b_offset", bShape, bRank, false, b_numel.value, cShape, cRank, c_numel.value) }}
{% endif %}{% set is_int = scalar == "i32" or scalar == "u32" %}{% set acc = scalar if is_int else "f32" %}{% if cDtype == "int8" %}fn wrap_dtype(v:vec4<i32>)->vec4<i32>{return(v<<vec4<u32>(24u))>>vec4<u32>(24u);}
{% set wrap = "wrap_dtype" %}{% elif cDtype == "uint8" %}fn wrap_dtype(v:vec4<u32>)->vec4<u32>{return v&vec4<u32>(0xFFu);}
{% set wrap = "wrap_dtype" %}{% else %}{% set wrap = "" %}{% endif %}{{ periodic_declaration("a", a_table, acc) }} {{ periodic_declaration("b", b_table, acc) }}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})
fn main(@builtin(global_invocation_id) gid: vec3<u32>{% if a_table.active or b_table.active %}, @builtin(local_invocation_index) lid: u32{% endif %}) {
{{ periodic_stage("a", a_table, acc) }} {{ periodic_stage("b", b_table, acc) }}
{% if a_table.active or b_table.active %}workgroupBarrier();
{% endif %}{{ flat_index_2d(tunables.WORKGROUP_SIZE, "i4") }}
{% set needsBase = c_numel.value != 0 and ((not a_same.value and a_mode != "scalar" and not a_table.active) or (not b_same.value and b_mode != "scalar" and not b_table.active and not (nativeTrailingB | default(false)))) %}{% if needsBase %}let base=i4*4u;
{% endif %}{{ broadcast_vector_load("a", aShape, a_same.value, a_mode, a_table, acc, crosses_inner_rows, cShape) }} {{ broadcast_vector_load("b", bShape, b_same.value, b_mode, b_table, acc, crosses_inner_rows, cShape) }} c[i4]={{ wrap }}(vec4<{{ scalar }}>(av+bv));}`],["ops/_shared/binary-broadcast.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{% include "ops/_shared/broadcast-offset.25bcd5874d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{{ binary_broadcast_offsets() }} {{ flat_tail_open() }}
{% if scalar == "i32" or scalar == "u32" %}let av=a[{{ broadcast_offset_call("a_offset", aShape, cShape, "i") }}];let bv=b[{{ broadcast_offset_call("b_offset", bShape, cShape, "i") }}];let r=av+bv;
{% if cDtype == "int8" %}c[i]=(r<<24u)>>24u;
{% elif cDtype == "uint8" %}c[i]=r&0xFFu;
{% else %}c[i]=r;
{% endif %}{% else %}let av=f32(a[{{ broadcast_offset_call("a_offset", aShape, cShape, "i") }}]);let bv=f32(b[{{ broadcast_offset_call("b_offset", bShape, cShape, "i") }}]);c[i]={{ scalar }}(av+bv);
{% endif %}{{ flat_tail_close() }} }`]]}],["ai.onnx.Col2Im",{manifest:{domain:"ai.onnx",name:"Col2Im",sinceVersion:18,inputs:{input:{dtype:"T",rank:3},image_shape:{dtype:"I",rank:1,storage:"uint32",narrowing:"checked"},block_shape:{dtype:"I",rank:1,storage:"uint32",narrowing:"checked"}},outputs:{output:{dtype:"T",rank:"dim(shapes.image_shape, 0) + 2"}},attributes:{dilations:{},pads:{},strides:{}},typeConstraints:{T:["float32"],I:["int64"]},tunables:{WORKGROUP_SIZE:{default:128}},derive:{spatialRank:"ranks.output - 2",stridesLengthOk:'not has(attrs, "strides") or (attrs.strides | length) == spatialRank',dilationsLengthOk:'not has(attrs, "dilations") or (attrs.dilations | length) == spatialRank',padsLengthOk:'not has(attrs, "pads") or (attrs.pads | length) == 2 * spatialRank',strideD:'attrs.strides[0] if stridesLengthOk and has(attrs, "strides") and spatialRank == 3 else 1',strideH:'attrs.strides[spatialRank - 2] if stridesLengthOk and has(attrs, "strides") and spatialRank >= 2 else 1',strideW:'attrs.strides[spatialRank - 1] if stridesLengthOk and has(attrs, "strides") and spatialRank >= 1 else 1',dilationD:'attrs.dilations[0] if dilationsLengthOk and has(attrs, "dilations") and spatialRank == 3 else 1',dilationH:'attrs.dilations[spatialRank - 2] if dilationsLengthOk and has(attrs, "dilations") and spatialRank >= 2 else 1',dilationW:'attrs.dilations[spatialRank - 1] if dilationsLengthOk and has(attrs, "dilations") and spatialRank >= 1 else 1',padFront:'attrs.pads[0] if padsLengthOk and has(attrs, "pads") and spatialRank == 3 else 0',padTop:'attrs.pads[spatialRank - 2] if padsLengthOk and has(attrs, "pads") and spatialRank >= 2 else 0',padLeft:'attrs.pads[spatialRank - 1] if padsLengthOk and has(attrs, "pads") and spatialRank >= 1 else 0',padBack:'attrs.pads[spatialRank] if padsLengthOk and has(attrs, "pads") and spatialRank == 3 else 0',padBottom:'attrs.pads[2 * spatialRank - 2] if padsLengthOk and has(attrs, "pads") and spatialRank >= 2 else 0',padRight:'attrs.pads[2 * spatialRank - 1] if padsLengthOk and has(attrs, "pads") and spatialRank >= 1 else 0',spatialAttributesOk:"stridesLengthOk and dilationsLengthOk and padsLengthOk and strideW >= 1 and dilationW >= 1 and padLeft >= 0 and padRight >= 0 and (spatialRank < 2 or (strideH >= 1 and dilationH >= 1 and padTop >= 0 and padBottom >= 0)) and (spatialRank < 3 or (strideD >= 1 and dilationD >= 1 and padFront >= 0 and padBack >= 0))",colBaseContract:"spatialAttributesOk and ranks.input == 3 and ranks.image_shape == 1 and ranks.block_shape == 1 and (ranks.output == 4 or ranks.output == 5) and dim(shapes.image_shape, 0) == spatialRank and dim(shapes.block_shape, 0) == spatialRank and dim(shapes.input, 0) == dim(shapes.output, 0)"},when:["colBaseContract"],bindings:{params:{struct:[{name:"count",type:"u32",value:"numel(shapes.output)"}]},input:{elementType:"f32"},output:{elementType:"f32"},image_shape:{elementType:"u32"},block_shape:{elementType:"u32"}},variants:[{id:"nchw2d",when:["ranks.output == 4"],passes:[{id:"main",name:"Col2Im",shader:"col2im-nd.wgsl.jinja",derive:{spatialRankSpec:2,channels:"dim(shapes.output, 1)",imageD:"dim(shapes.output, 2) if ranks.output == 5 else 1",imageH:"dim(shapes.output, ranks.output - 2)",imageW:"dim(shapes.output, ranks.output - 1)",colChannels:"dim(shapes.input, 1)",blocks:"dim(shapes.input, 2)",strideDSpec:"strideD",strideHSpec:"strideH",strideWSpec:"strideW",padFrontSpec:"padFront",padTopSpec:"padTop",padLeftSpec:"padLeft",padBottomSpec:"padBottom",padRightSpec:"padRight",dilationDSpec:"dilationD",dilationHSpec:"dilationH",dilationWSpec:"dilationW"},bindings:["input","output","image_shape","block_shape","params"],dispatch:{threads:"numel(shapes.output)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["col2im-nd.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const CHANNELS:u32={{ channels }}u;const IMAGE_H:u32={{ imageH }}u;const IMAGE_W:u32={{ imageW }}u;const COL_CHANNELS:u32={{ colChannels }}u;const BLOCKS:u32={{ blocks }}u;const STRIDE_H:u32={{ strideHSpec }}u;const STRIDE_W:u32={{ strideWSpec }}u;const PAD_TOP:i32={{ padTopSpec }};const PAD_LEFT:i32={{ padLeftSpec }};const PAD_BOTTOM:i32={{ padBottomSpec }};const PAD_RIGHT:i32={{ padRightSpec }};const DILATION_H:u32={{ dilationHSpec }}u;const DILATION_W:u32={{ dilationWSpec }}u;fn divisible(value:i32,divisor:u32)->bool{return value>=0&&value%i32(divisor)==0;}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "index") }} let blockD=1u;let blockH=block_shape[0];let blockW=block_shape[1];let imageShapeOk=image_shape[0]==IMAGE_H&&image_shape[1]==IMAGE_W;let blockShapeOk=blockH>0u&&blockW>0u;let availableD=0;let availableH=i32(IMAGE_H)+PAD_TOP+PAD_BOTTOM-i32(DILATION_H*(blockH-1u)+1u);let availableW=i32(IMAGE_W)+PAD_LEFT+PAD_RIGHT-i32(DILATION_W*(blockW-1u)+1u);if(!imageShapeOk||!blockShapeOk||availableD<0||availableH<0||availableW<0){output[index]=0.0;return;}let outD=1u;let outH=u32(availableH)/STRIDE_H+1u;let outW=u32(availableW)/STRIDE_W+1u;let blockVolume=blockD*blockH*blockW;if(COL_CHANNELS!=CHANNELS*blockVolume||BLOCKS!=outD*outH*outW){output[index]=0.0;return;}let iw=index%IMAGE_W;var t=index/IMAGE_W;let ih=t%IMAGE_H;t/=IMAGE_H;let c=t%CHANNELS;let n=t/CHANNELS;var acc=0.0;for(var kh=0u;kh<blockH;kh+=1u){let shiftedH=i32(ih)+PAD_TOP-i32(kh*DILATION_H);if(!divisible(shiftedH,STRIDE_H)){continue;}let oh=u32(shiftedH/i32(STRIDE_H));if(oh>=outH){continue;}for(var kw=0u;kw<blockW;kw+=1u){let shiftedW=i32(iw)+PAD_LEFT-i32(kw*DILATION_W);if(!divisible(shiftedW,STRIDE_W)){continue;}let ow=u32(shiftedW/i32(STRIDE_W));if(ow>=outW){continue;}let colChannel=(c*blockH+kh)*blockW+kw;let blockIndex=oh*outW+ow;let plane=CHANNELS*blockH*blockW;acc+=input[(n*plane+colChannel)*BLOCKS+blockIndex];} }output[index]=acc;}`]],assumes:[["col2im-nd.wgsl.jinja",{padBack:0,imageD:1,strideDSpec:1,padFrontSpec:0,dilationDSpec:1,dilationHSpec:1,dilationWSpec:1}]]}],["ai.onnx.Concat",{manifest:{domain:"ai.onnx",name:"Concat",sinceVersion:13,inputs:{a:{onnx:"A",dtype:"T"},b:{onnx:"B",dtype:"T",optional:!0},d:{onnx:"D",dtype:"T",optional:!0},e:{onnx:"E",dtype:"T",optional:!0},f:{onnx:"F",dtype:"T",optional:!0},g:{onnx:"G",dtype:"T",optional:!0},h:{onnx:"H",dtype:"T",optional:!0},i:{onnx:"I",dtype:"T",optional:!0},j:{onnx:"J",dtype:"T",optional:!0}},outputs:{c:{onnx:"concat_result",dtype:"T",rank:"ranks.a",shape:"prefix(shapes.a, concatAxis) + [concatExtent] + suffix(shapes.a, concatAxis + 1)"}},attributes:{axis:{}},attributeConstraints:{axis:{required:!0}},typeConstraints:{T:["float32","float16","uint32","int32","int16","uint8","int8","bool","int64"]},tunables:{WORKGROUP_SIZE:{default:256}},derive:{concatAxis:"attrs.axis + ranks.a if attrs.axis < 0 else attrs.axis",concatExtent:"dim(shapes.a, concatAxis) + (dim(shapes.b, concatAxis) if present.b else 0) + (dim(shapes.d, concatAxis) if present.d else 0) + (dim(shapes.e, concatAxis) if present.e else 0) + (dim(shapes.f, concatAxis) if present.f else 0) + (dim(shapes.g, concatAxis) if present.g else 0) + (dim(shapes.h, concatAxis) if present.h else 0) + (dim(shapes.i, concatAxis) if present.i else 0) + (dim(shapes.j, concatAxis) if present.j else 0)",baseContract:"ranks.c >= 1 and attrs.axis + ranks.c >= 0 and attrs.axis < ranks.c and f16Ok(dtypes.T)",oneInputContract:"baseContract and not present.b and not present.d and not present.e and not present.f and not present.g and not present.h and not present.i and not present.j and sameShape(shapes.a, shapes.c)",twoInputContract:"baseContract and present.b and not present.d and ranks.a == ranks.c and ranks.b == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b)",threeInputContract:"baseContract and present.b and present.d and not present.e and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d)",fourInputContract:"baseContract and present.d and present.e and not present.f and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e)",fiveInputContract:"baseContract and present.d and present.e and present.f and not present.g and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and ranks.f == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e) + numel(shapes.f)",sixInputContract:"baseContract and present.d and present.e and present.f and present.g and not present.h and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and ranks.f == ranks.c and ranks.g == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e) + numel(shapes.f) + numel(shapes.g)",sevenInputContract:"baseContract and present.d and present.e and present.f and present.g and present.h and not present.i and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and ranks.f == ranks.c and ranks.g == ranks.c and ranks.h == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e) + numel(shapes.f) + numel(shapes.g) + numel(shapes.h)",eightInputContract:"baseContract and present.d and present.e and present.f and present.g and present.h and present.i and not present.j and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and ranks.f == ranks.c and ranks.g == ranks.c and ranks.h == ranks.c and ranks.i == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e) + numel(shapes.f) + numel(shapes.g) + numel(shapes.h) + numel(shapes.i)",nineInputContract:"baseContract and present.d and present.e and present.f and present.g and present.h and present.i and present.j and ranks.a == ranks.c and ranks.b == ranks.c and ranks.d == ranks.c and ranks.e == ranks.c and ranks.f == ranks.c and ranks.g == ranks.c and ranks.h == ranks.c and ranks.i == ranks.c and ranks.j == ranks.c and numel(shapes.c) == numel(shapes.a) + numel(shapes.b) + numel(shapes.d) + numel(shapes.e) + numel(shapes.f) + numel(shapes.g) + numel(shapes.h) + numel(shapes.i) + numel(shapes.j)",nineBlockContract:"nineInputContract and numel(shapes.a) > 0 and numel(shapes.b) > 0 and numel(shapes.d) > 0 and numel(shapes.e) > 0 and numel(shapes.f) > 0 and numel(shapes.g) > 0 and numel(shapes.h) > 0 and numel(shapes.i) > 0 and numel(shapes.j) > 0 and outer(shapes.a, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.b, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.d, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.e, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.f, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.g, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.h, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.i, attrs.axis) == outer(shapes.c, attrs.axis) and outer(shapes.j, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.a, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.b, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.d, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.e, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.f, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.g, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.h, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.i, attrs.axis) == inner(shapes.c, attrs.axis) and inner(shapes.j, attrs.axis) == inner(shapes.c, attrs.axis)",twoBlockContract:"twoInputContract and numel(shapes.c) > 0 and numel(shapes.a) > 0 and numel(shapes.b) > 0 and outer(shapes.a, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.a, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.b, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.b, attrs.axis) == inner(shapes.c, attrs.axis)",threeBlockContract:"threeInputContract and numel(shapes.c) > 0 and numel(shapes.a) > 0 and numel(shapes.b) > 0 and numel(shapes.d) > 0 and outer(shapes.a, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.a, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.b, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.b, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.d, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.d, attrs.axis) == inner(shapes.c, attrs.axis)",fourBlockContract:"fourInputContract and numel(shapes.c) > 0 and numel(shapes.a) > 0 and numel(shapes.b) > 0 and numel(shapes.d) > 0 and numel(shapes.e) > 0 and outer(shapes.a, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.a, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.b, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.b, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.d, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.d, attrs.axis) == inner(shapes.c, attrs.axis) and outer(shapes.e, attrs.axis) == outer(shapes.c, attrs.axis) and inner(shapes.e, attrs.axis) == inner(shapes.c, attrs.axis)",twoLastAxisVec4TailContract:'twoBlockContract and (attrs.axis == -1 or attrs.axis == ranks.c - 1) and dtypes.T == "f32" and dim(shapes.c, attrs.axis) % 4 == 0 and (dim(shapes.a, attrs.axis) % 4 != 0 or dim(shapes.b, attrs.axis) % 4 != 0) and outer(shapes.c, attrs.axis) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(dim(shapes.c, attrs.axis), 4 * tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension',cShape:"shapes.c",rank:"ranks.c",axisSpec:"concatAxis"},bindings:{a_main:{name:"a",elementType:"$ioScalar"},b_main:{name:"b",elementType:"$ioScalar"},c_main:{name:"c",elementType:"$outputElement"}},variants:[{id:"two_input_block_vec4",priority:15,when:['dtypes.T != "vec2<u32>"',"twoBlockContract","(dim(shapes.a, attrs.axis) * inner(shapes.a, attrs.axis)) % 4 == 0","(dim(shapes.b, attrs.axis) * inner(shapes.b, attrs.axis)) % 4 == 0"],derive:{ioScalar:'"vec4<" ~ dtypes.T ~ ">"',outputElement:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Concat.BlockVec4",shader:"datamove-concat-block.wgsl.jinja",derive:{aShape:"shapes.a",bShape:"shapes.b",inputCount:2,count:"numel(shapes.c) / 4"},bindings:["a_main","b_main","c_main"],dispatch:{x:"min(ceilDiv((numel(shapes.c) / 4), (tunables.WORKGROUP_SIZE)), device.limits.maxComputeWorkgroupsPerDimension)",y:1,z:1}}]},{id:"two_input",when:["twoInputContract"],derive:{ioScalar:"dtypes.T",outputElement:"dtypes.T"},passes:[{id:"main",name:"Concat",shader:"concat-n.wgsl.jinja",derive:{inputs:'[ {"name": "a", "shape": shapes.a}, {"name": "b", "shape": shapes.b} ]'},bindings:["a_main","b_main","c_main"],dispatch:{threads:"ceilDiv(numel(shapes.c), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["datamove-concat-block.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% if inputs is defined %}{% set concatInputs = inputs %}{% else %}{% set concatInputs = [{"name": "a", "shape": aShape}, {"name": "b", "shape": bShape}] %}{% endif %}{% set inner = namespace(value=1) %}{% for j in range(axisSpec + 1, rank) %}{% set inner.value = inner.value * cShape[j] %}{% endfor %}{% set inner_c = (((cShape[axisSpec] * inner.value) / 4) | int) %}const COUNT:u32={{ count }}u;const INNER_C:u32={{ inner_c }}u;const WG:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let stride=nwg.x*WG;for(var outIndex=gid.x;outIndex<COUNT;outIndex+=stride){let outer_idx=outIndex/INNER_C;let pos=outIndex%INNER_C;
{% set prefix = namespace(value=0) %}{% for input in concatInputs %}{% set run = (((input.shape[axisSpec] * inner.value) / 4) | int) %}{% set end = prefix.value + run %}{% if prefix.value > 0 and not loop.last %}if(pos>={{ prefix.value }}u&&pos<{{ end }}u){
{% elif prefix.value > 0 %}if(pos>={{ prefix.value }}u){
{% else %}if(pos<{{ end }}u){
{% endif %}c[outIndex]={{ input.name }}[outer_idx*{{ run }}u+(pos-{{ prefix.value }}u)];}
{% set prefix.value = end %}{% endfor %}} }`],["concat-n.wgsl.jinja",`{% set axisBase = 0 %}{% set partialTail = false %}{{ env.wgsl.resourceDeclarations }}
{% set c_count = namespace(value=1) %}{% for d in cShape %}{% set c_count.value = c_count.value * d %}{% endfor %}{% for input in inputs %}{% set prefix = namespace(value=axisBase) %}{% for prev in inputs[:loop.index0] %}{% set prefix.value = prefix.value + prev.shape[axisSpec] %}{% endfor %}fn {{ input.name }}_offset({% if c_count.value > 0 %}out_index: u32{% endif %}) -> u32 {
{% if c_count.value == 0 %}return 0u;
{% else %}var rem=out_index;var offset=0u;
{% for axis in range(rank) %}{% set c_stride = namespace(value=1) %}{% for j in range(axis + 1, rank) %}{% set c_stride.value = c_stride.value * cShape[j] %}{% endfor %}{% set in_stride = namespace(value=1) %}{% for j in range(axis + 1, rank) %}{% set in_stride.value = in_stride.value * input.shape[j] %}{% endfor %}var coord{{ axis }}=rem/{{ c_stride.value }}u;rem=rem% {{ c_stride.value }}u;
{% if axis == axisSpec and prefix.value > 0 %}coord{{ axis }}=coord{{ axis }}-{{ prefix.value }}u;
{% endif %}offset=offset+coord{{ axis }}*{{ in_stride.value }}u;
{% endfor %}return offset;
{% endif %}}
{% endfor %}fn axis_coord({% if c_count.value > 0 %}out_index: u32{% endif %}) -> u32 {
{% if c_count.value == 0 %}return 0u;
{% else %}{% set axis_stride = namespace(value=1) %}{% for j in range(axisSpec + 1, rank) %}{% set axis_stride.value = axis_stride.value * cShape[j] %}{% endfor %}return(out_index/{{ axis_stride.value }}u)% {{ cShape[axisSpec] }}u;
{% endif %}}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let begin=(gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ tunables.WORKGROUP_SIZE }}u)*4u;let end=min(begin+4u,{{ c_count.value }}u);for(var out_idx=begin;out_idx<end;out_idx=out_idx+1u){
    let coord = axis_coord({% if c_count.value > 0 %}out_idx{% endif %});
{% set cumulative = namespace(value=axisBase) %}{% for input in inputs %}{% set lower = cumulative.value %}{% set cumulative.value = cumulative.value + input.shape[axisSpec] %}{% set bounded = (not loop.last) or partialTail %}{% if lower > 0 and bounded %}if(coord>={{ lower }}u&&coord<{{ cumulative.value }}u){
{% elif lower > 0 %}if(coord>={{ lower }}u){
{% elif bounded %}if(coord<{{ cumulative.value }}u){
{% else %}{
{% endif %}      c[out_idx] = {{ input.name }}[{{ input.name }}_offset({% if c_count.value > 0 %}out_idx{% endif %})];
}
{% endfor %}} }`]]}],["ai.onnx.Conv",{manifest:{domain:"ai.onnx",name:"Conv",sinceVersion:11,inputs:{x:{onnx:"X",dtype:"T"},w:{onnx:"W",dtype:"T"},bias:{onnx:"B",dtype:"T",rank:1,optional:!0}},outputs:{y:{onnx:"Y",dtype:"T",rank:"ranks.x",shape:"[dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputWidth] if ranks.x == 3 else ([dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputHeight, expectedOutputWidth] if ranks.x == 4 else [dim(shapes.x, 0), dim(shapes.w, 0), expectedOutputDepth, expectedOutputHeight, expectedOutputWidth])"}},attributes:{auto_pad:{default:"NOTSET"},group:{default:1},dilations:{},kernel_shape:{},pads:{},strides:{}},attributeConstraints:{auto_pad:{values:["NOTSET","SAME_UPPER","SAME_LOWER","VALID"]}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256},DIRECT_MAX_KERNEL_SIZE:{default:11},GROUPED_MIN_KERNEL_SIZE:{default:3},GROUPED_MAX_KERNEL_SIZE:{default:11},GROUPED_SCALAR_OC_MAX_KERNEL_SIZE:{default:9},GROUPED_WIDE_WORKGROUP_SIZE:{default:64},GROUPED_MIN_OUTPUT_ELEMENTS:{default:256},GROUPED_MAX_REGISTER_SPAN:{default:64},REG_MIN_TILE_COUNT:{default:128},REG_MIN_SPATIAL:{default:1024},REG_MIN_KERNEL_ROWS:{default:64},CONV3D_REG_MIN_M:{default:64},CONV1D_WG_X:{default:16},CONV1D_WG_Y:{default:16},CONV1D_K_TILE:{default:16},CONV1D_TILE_M:{default:4},CONV1D_TILE_N:{default:4},CONV1D_REG_MIN_TILE_COUNT:{default:32},CONV1D_REG_MIN_KERNEL_ROWS:{default:32},SGMAT_MIN_ALIGNED_K:{default:32},SGMAT_MIN_PADDED_K:{default:16},SGMAT_MIN_ALIGNED_N:{default:64},SGMAT_MIN_PADDED_N:{default:32},SGMAT_MIN_M:{default:32},IMPLICIT_MAX_M_TILES:{default:9},IMPLICIT_MIN_OUTPUT_SPATIAL:{default:1024},SGMAT_SPLIT_K_TARGET_WORKGROUPS:{default:512},SGMAT_SPLIT_K_MAX_BASE_WORKGROUPS:{default:64},SGMAT_SPLIT_K_MIN_TILES_PER_SLICE:{default:2},SGMAT_SPLIT_K_MAX_PARTIAL_BYTES:{default:16777216},CHANNEL_REDUCE_MAX_OUT_CHANNELS:{default:16},CHANNEL_REDUCE_MIN_SPATIAL:{default:1024},CHANNEL_REDUCE_WORKGROUP_SIZE:{default:256},IM2COL_ITEMS_PER_THREAD:{default:8},IM2COL_MIN_THREADS:{default:16384},IMPLICIT_TILED_MAX_M_TILES:{default:4},IMPLICIT_TILED_MIN_TILE_COUNT:{default:48},TILED_SPLIT_K_MODE:{default:1},TILED_SPLIT_K_TARGET_WORKGROUPS:{default:512},TILED_SPLIT_K_MAX_BASE_WORKGROUPS:{default:64},TILED_SPLIT_K_MIN_TILES_PER_SLICE:{default:2},TILED_SPLIT_K_MAX_PARTIAL_BYTES:{default:16777216},GROUPED_DILATED_LANES:{default:4},GROUPED_DILATED_MIN_PLAIN_SPAN:{default:0},DIRECT_IM2COL_MIN_M_TILES:{default:8},MATERIALIZED_REUSE_MIN_ELEMENTS:{default:4194304},WINOGRAD_WIDE_OUTPUT_MIN_CHANNELS:{default:128},WINOGRAD_MIN_WORKGROUPS:{default:16},WINOGRAD_MAX_SCRATCH_BYTES:{default:268435456}},derive:{inputDepth:"dim(shapes.x, 2) if ranks.x == 5 else 1",inputHeight:"dim(shapes.x, 3) if ranks.x == 5 else (dim(shapes.x, 2) if ranks.x == 4 else 1)",inputWidth:"dim(shapes.x, ranks.x - 1) if ranks.x >= 3 else 1",outputDepth:"dim(shapes.y, 2) if ranks.y == 5 else 1",outputHeight:"dim(shapes.y, 3) if ranks.y == 5 else (dim(shapes.y, 2) if ranks.y == 4 else 1)",outputWidth:"dim(shapes.y, ranks.y - 1) if ranks.y >= 3 else 1",kernelDepth:"dim(shapes.w, 2) if ranks.w == 5 else 1",kernelHeight:"dim(shapes.w, 3) if ranks.w == 5 else (dim(shapes.w, 2) if ranks.w == 4 else 1)",kernelWidth:"dim(shapes.w, ranks.w - 1) if ranks.w >= 3 else 1",spatialRank:"ranks.w - 2",kernelShapeLengthOk:'not has(attrs, "kernel_shape") or (attrs.kernel_shape | length) == spatialRank',stridesLengthOk:'not has(attrs, "strides") or (attrs.strides | length) == spatialRank',dilationsLengthOk:'not has(attrs, "dilations") or (attrs.dilations | length) == spatialRank',padsLengthOk:'not has(attrs, "pads") or (attrs.pads | length) == 2 * spatialRank',kernelD:'attrs.kernel_shape[0] if kernelShapeLengthOk and has(attrs, "kernel_shape") and spatialRank == 3 else 1',kernelH:'attrs.kernel_shape[spatialRank - 2] if kernelShapeLengthOk and has(attrs, "kernel_shape") and spatialRank >= 2 else 1',kernelW:'attrs.kernel_shape[spatialRank - 1] if kernelShapeLengthOk and has(attrs, "kernel_shape") and spatialRank >= 1 else 1',strideD:'attrs.strides[0] if stridesLengthOk and has(attrs, "strides") and spatialRank == 3 else 1',strideH:'attrs.strides[spatialRank - 2] if stridesLengthOk and has(attrs, "strides") and spatialRank >= 2 else 1',strideW:'attrs.strides[spatialRank - 1] if stridesLengthOk and has(attrs, "strides") and spatialRank >= 1 else 1',dilationD:'attrs.dilations[0] if dilationsLengthOk and has(attrs, "dilations") and spatialRank == 3 else 1',dilationH:'attrs.dilations[spatialRank - 2] if dilationsLengthOk and has(attrs, "dilations") and spatialRank >= 2 else 1',dilationW:'attrs.dilations[spatialRank - 1] if dilationsLengthOk and has(attrs, "dilations") and spatialRank >= 1 else 1',padFront:'attrs.pads[0] if padsLengthOk and has(attrs, "pads") and spatialRank == 3 else 0',padTop:'attrs.pads[spatialRank - 2] if padsLengthOk and has(attrs, "pads") and spatialRank >= 2 else 0',padLeft:'attrs.pads[spatialRank - 1] if padsLengthOk and has(attrs, "pads") and spatialRank >= 1 else 0',padBack:'attrs.pads[spatialRank] if padsLengthOk and has(attrs, "pads") and spatialRank == 3 else 0',padBottom:'attrs.pads[2 * spatialRank - 2] if padsLengthOk and has(attrs, "pads") and spatialRank >= 2 else 0',padRight:'attrs.pads[2 * spatialRank - 1] if padsLengthOk and has(attrs, "pads") and spatialRank >= 1 else 0',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',autoPadSame:'attrs.auto_pad == "SAME_UPPER" or attrs.auto_pad == "SAME_LOWER"',autoPadValid:'attrs.auto_pad == "VALID"',samePadDepth:"max(0, (outputDepth - 1) * strideD + (kernelDepth - 1) * dilationD + 1 - inputDepth)",samePadHeight:"max(0, (outputHeight - 1) * strideH + (kernelHeight - 1) * dilationH + 1 - inputHeight)",samePadWidth:"max(0, (outputWidth - 1) * strideW + (kernelWidth - 1) * dilationW + 1 - inputWidth)",samePadFront:'floor(samePadDepth / 2) if attrs.auto_pad == "SAME_UPPER" else samePadDepth - floor(samePadDepth / 2)',samePadTop:'floor(samePadHeight / 2) if attrs.auto_pad == "SAME_UPPER" else samePadHeight - floor(samePadHeight / 2)',samePadLeft:'floor(samePadWidth / 2) if attrs.auto_pad == "SAME_UPPER" else samePadWidth - floor(samePadWidth / 2)',effectivePadFront:"samePadFront if autoPadSame else (0 if autoPadValid else padFront)",effectivePadTop:"samePadTop if autoPadSame else (0 if autoPadValid else padTop)",effectivePadLeft:"samePadLeft if autoPadSame else (0 if autoPadValid else padLeft)",expectedOutputDepth:"ceil(inputDepth / strideD) if autoPadSame else floor((inputDepth + (0 if autoPadValid else padFront + padBack) - ((kernelDepth - 1) * dilationD + 1)) / strideD) + 1",expectedOutputHeight:"ceil(inputHeight / strideH) if autoPadSame else floor((inputHeight + (0 if autoPadValid else padTop + padBottom) - ((kernelHeight - 1) * dilationH + 1)) / strideH) + 1",expectedOutputWidth:"ceil(inputWidth / strideW) if autoPadSame else floor((inputWidth + (0 if autoPadValid else padLeft + padRight) - ((kernelWidth - 1) * dilationW + 1)) / strideW) + 1",spatialAttributeLengthsOk:"kernelShapeLengthOk and stridesLengthOk and dilationsLengthOk and padsLengthOk",kernelShapeMatchesWeights:'not has(attrs, "kernel_shape") or (kernelW == kernelWidth and (spatialRank < 2 or kernelH == kernelHeight) and (spatialRank < 3 or kernelD == kernelDepth))',kernelExtentsOk:"kernelWidth >= 1 and (spatialRank < 2 or kernelHeight >= 1) and (spatialRank < 3 or kernelDepth >= 1)",stridesValuesOk:'not has(attrs, "strides") or (strideD >= 1 and floor(strideD) == strideD and strideH >= 1 and floor(strideH) == strideH and strideW >= 1 and floor(strideW) == strideW)',dilationsValuesOk:'not has(attrs, "dilations") or (dilationD >= 1 and floor(dilationD) == dilationD and dilationH >= 1 and floor(dilationH) == dilationH and dilationW >= 1 and floor(dilationW) == dilationW)',padsValuesOk:"padFront >= 0 and floor(padFront) == padFront and padTop >= 0 and floor(padTop) == padTop and padLeft >= 0 and floor(padLeft) == padLeft and padBack >= 0 and floor(padBack) == padBack and padBottom >= 0 and floor(padBottom) == padBottom and padRight >= 0 and floor(padRight) == padRight",explicitPadsOk:'attrs.auto_pad == "NOTSET" or not has(attrs, "pads")',spatialAttributesOk:"spatialRank >= 1 and spatialRank <= 3 and spatialAttributeLengthsOk and kernelShapeMatchesWeights and kernelExtentsOk and stridesValuesOk and dilationsValuesOk and padsValuesOk and explicitPadsOk",convWorkgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",configuredWorkgroupOk:"tunables.WORKGROUP_SIZE <= deviceWorkgroupCap",conv1dRanksOk:"spatialAttributesOk and ranks.x == 3 and ranks.w == 3 and ranks.y == 3",nchwRanksOk:"spatialAttributesOk and ranks.x == 4 and ranks.w == 4 and ranks.y == 4",conv3dRanksOk:"spatialAttributesOk and ranks.x == 5 and ranks.w == 5 and ranks.y == 5",batchSize:"dim(shapes.x, 0) if ranks.x >= 1 else 0",inChannels:"dim(shapes.x, 1) if ranks.x >= 2 else 0",outChannels:"dim(shapes.w, 0) if ranks.w >= 1 else 0",weightInChannels:"dim(shapes.w, 1) if ranks.w >= 2 else 0",inChannelsPerGroup:"inChannels / attrs.group if attrs.group > 0 else 0",outChannelsPerGroup:"outChannels / attrs.group if attrs.group > 0 else 0",inputSpatial:"inputHeight * inputWidth",outputSpatial:"outputHeight * outputWidth",kernelRows:"weightInChannels * kernelHeight * kernelWidth",conv1dBlockN:"tunables.CONV1D_WG_X * tunables.CONV1D_TILE_N",outChannelsOk:"ranks.y >= 2 and ranks.x >= 1 and ranks.w >= 1 and dim(shapes.y, 0) == batchSize and dim(shapes.y, 1) == outChannels",biasOk:"present.bias and ranks.bias == 1 and dim(shapes.bias, 0) == outChannels",noBiasContract:"not present.bias",biasContract:"present.bias and biasOk",hasBias:"present.bias",paddedKernelRows:"ceilDiv(kernelRows, 32) * 32",paddedOutputSpatial:"ceilDiv(outputSpatial, 64) * 64",fScalar:'"f16" if dtypes.T == "f16" else "f32"',subgroupCols:2,M:"outChannels",usesF16:'dtypes.T == "f16"',implicitTiledWorthIt:"(strideH < kernelHeight or strideW < kernelWidth) and ceil(outChannels / 64) <= tunables.IMPLICIT_TILED_MAX_M_TILES",groupedDilatedQuads:"ceilDiv(ceilDiv(outputWidth, dilationW), tunables.GROUPED_DILATED_LANES)",groupedDilatedLanesOk:"dilationW > 1 and strideW == 1 and outputWidth >= tunables.GROUPED_DILATED_LANES * dilationW",subgroupMatrix8F16Available:'subgroupMatrixConfigAvailable("f16", "f32", 8, 8, 8)',subgroupMatrix8F32Available:'subgroupMatrixConfigAvailable("f32", "f32", 8, 8, 8)',subgroupMatrixTopologyBytes:"3 * 16",convFlops:"2 * batchSize * outChannels * outputDepth * outputHeight * outputWidth * inChannelsPerGroup * kernelDepth * kernelHeight * kernelWidth",outputVolume:"outputDepth * outputHeight * outputWidth",kernelVolumeRows:"weightInChannels * kernelDepth * kernelHeight * kernelWidth",conv1dOutputOk:"conv1dRanksOk and outputWidth == expectedOutputWidth",conv1dBlockM:"tunables.CONV1D_WG_Y * tunables.CONV1D_TILE_M",outHeightOk:"nchwRanksOk and outputHeight == expectedOutputHeight",outWidthOk:"nchwRanksOk and outputWidth == expectedOutputWidth",im2colRanksOk:"conv1dRanksOk or nchwRanksOk",conv3dOutputOk:"conv3dRanksOk and outputDepth == expectedOutputDepth and outputHeight == expectedOutputHeight and outputWidth == expectedOutputWidth",groupOk:"ranks.w >= 2 and ranks.x >= 2 and attrs.group >= 1 and outChannels % attrs.group == 0 and weightInChannels * attrs.group == inChannels",im2colElements:"batchSize * kernelRows * outputSpatial",paddedIm2colElements:"batchSize * paddedKernelRows * paddedOutputSpatial",sgmatF32WideSharedBytes:'((64 * 32 + 64 * 32 + (256 / 32) * 4 * 64) * dtypeBytes("float32")) + subgroupMatrixTopologyBytes',sgmatKTiles:"ceilDiv(kernelRows, 32)",sgmatReduceWorkgroupSize:"min(256, deviceWorkgroupCap)",pointwiseUnitStride:"kernelHeight == 1 and kernelWidth == 1 and strideH == 1 and strideW == 1 and dilationH == 1 and dilationW == 1 and effectivePadTop == 0 and effectivePadLeft == 0 and outputHeight == inputHeight and outputWidth == inputWidth",channelReduceWorkgroupSize:"min(tunables.CHANNEL_REDUCE_WORKGROUP_SIZE, deviceWorkgroupCap)",halfRegTileRows:"min(64, 32 * ceilDiv(outChannels, 32))",halfTiledHasEnoughSpatialWork:"batchSize * outputSpatial >= 256",groupedRollKernelRows:"dim(shapes.w, 1) > 1",winogradEpilogueOk:!0,hasActivation:!1,hasZ:!1,winogradTilesX:"ceilDiv(outputWidth, 2)",winogradWg:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",narrowWideRowsPerThread:8,packedMaterializedN:"ceilDiv(outputSpatial, 4) * 4",shortImplicitK:"device.adapterInfo.subgroupMaxSize",shortImplicitGrid:"batchSize * ceilDiv(outChannels, 64) * ceilDiv(outputSpatial, 64)",coalescedImplicitGather:'variableSubgroup16To32 and dtypes.T == "f32"',wideImplicitM:"device.adapterInfo.subgroupMaxSize * 4 if variableSubgroup16To32 else 64",wideImplicitMicroM:"device.adapterInfo.subgroupMinSize / 2 if variableSubgroup16To32 else 8",tiledKTile:"32 if kernelRows % 32 == 0 else 16",tiledBaseWorkgroups:"ceilDiv(outputSpatial, 64) * ceilDiv(outChannels, 64) * batchSize",compactHalfSplit:"variableSubgroup16To32 and usesF16 and nchwRanksOk and halfRegTileRows > 0 and halfRegTileRows < 64 and kernelHeight * kernelWidth > 1 and kernelRows >= tiledKTile * 16 and (halfRegTileRows + 64) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and 64 <= deviceWorkgroupCap and 16 <= device.limits.maxComputeWorkgroupSizeX and halfRegTileRows / 8 <= device.limits.maxComputeWorkgroupSizeY",tiledMPadded:"ceilDiv(outChannels, halfRegTileRows) * halfRegTileRows if compactHalfSplit else (ceilDiv(outChannels, 64) * 64)",tiledNPadded:"ceilDiv(outputSpatial, 64) * 64",tiledReduceWorkgroupSize:"min(256, deviceWorkgroupCap)",conv1dTiledGeometryOk:"tunables.CONV1D_WG_X >= 1 and tunables.CONV1D_WG_Y >= 1 and tunables.CONV1D_K_TILE >= 1 and tunables.CONV1D_TILE_M >= 1 and tunables.CONV1D_TILE_N >= 1 and tunables.CONV1D_WG_X * tunables.CONV1D_WG_Y <= device.limits.maxComputeInvocationsPerWorkgroup and tunables.CONV1D_WG_X <= device.limits.maxComputeWorkgroupSizeX and tunables.CONV1D_WG_Y <= device.limits.maxComputeWorkgroupSizeY and (conv1dBlockM + conv1dBlockN) * tunables.CONV1D_K_TILE * 4 <= device.limits.maxComputeWorkgroupStorageSize",conv1dTiledDispatchOk:"batchSize >= 1 and batchSize <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(outputWidth, conv1dBlockN) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(outChannels, conv1dBlockM) <= device.limits.maxComputeWorkgroupsPerDimension",conv1dTiledFit:"ceilDiv(outChannels, conv1dBlockM) * ceilDiv(outputWidth, conv1dBlockN) >= tunables.CONV1D_REG_MIN_TILE_COUNT and kernelRows >= tunables.CONV1D_REG_MIN_KERNEL_ROWS",im2colOutputOk:"conv1dOutputOk or (outHeightOk and outWidthOk)",conv2dOutputOk:"outHeightOk and outWidthOk",im2colBytes:'im2colElements * dtypeBytes("float32")',paddedIm2colBytes:'paddedIm2colElements * dtypeBytes("float32")',im2col3dElements:"batchSize * kernelVolumeRows * outputVolume",sgmatUseF32WideTile:'dtypes.T == "f32" and outChannels % 64 == 0 and wave32Effective and 256 <= deviceWorkgroupCap and sgmatF32WideSharedBytes <= device.limits.maxComputeWorkgroupStorageSize',implicitIm2col3dOk:"conv3dRanksOk and attrs.group == 1 and weightInChannels == inChannels and conv3dOutputOk",paddedKernelVolumeRows:"ceilDiv(kernelVolumeRows, 32) * 32",halfIm2colFits:'im2colElements * dtypeBytes("float16") <= device.limits.maxStorageBufferBindingSize and im2colElements * dtypeBytes("float16") <= device.limits.maxBufferSize',halfRegKCapacity:'max(1, floor(device.limits.maxComputeWorkgroupStorageSize / (2 * (halfRegTileRows + 64) * dtypeBytes("float16"))))',winogradTiles:"ceilDiv(outputHeight, 2) * winogradTilesX",narrowWideColumnCapacity:"max(1, min(floor(device.limits.maxComputeWorkgroupStorageSize / (tiledKTile * 4)) - halfRegTileRows, 4 * min(device.limits.maxComputeWorkgroupSizeX, floor(device.limits.maxComputeInvocationsPerWorkgroup / max(1, halfRegTileRows / narrowWideRowsPerThread)))))",packedMaterializedBytes:"batchSize * kernelRows * packedMaterializedN * 2",unsplitReuseFit:'variableSubgroup16To32 and dtypes.T == "f32" and tiledBaseWorkgroups >= ceilDiv(tunables.REG_MIN_TILE_COUNT, 4) and (64 + 64) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and 16 <= device.limits.maxComputeWorkgroupSizeX and 8 <= device.limits.maxComputeWorkgroupSizeY and 128 <= deviceWorkgroupCap and pointwiseUnitStride',wideImplicitMOk:"variableSubgroup16To32 and usesF16 and outChannels % wideImplicitM == 0 and batchSize * ceilDiv(outChannels, wideImplicitM) * ceilDiv(outputSpatial, 64) * wideImplicitM >= tunables.IMPLICIT_TILED_MIN_TILE_COUNT * 64 and (wideImplicitM + 64) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and device.adapterInfo.subgroupMinSize <= device.limits.maxComputeWorkgroupSizeX and wideImplicitM / wideImplicitMicroM <= device.limits.maxComputeWorkgroupSizeY and device.adapterInfo.subgroupMinSize * wideImplicitM / wideImplicitMicroM <= deviceWorkgroupCap",tiledKTiles:"ceilDiv(kernelRows, tiledKTile)",tiledSplitKStarved:"tiledBaseWorkgroups >= 1 and tiledBaseWorkgroups <= tunables.TILED_SPLIT_K_MAX_BASE_WORKGROUPS",halfSplitReuse:"variableSubgroup16To32 and usesF16 and kernelHeight * kernelWidth > 1 and tiledSplitKStarved and outChannels >= 64 and (64 + 64) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and 64 <= deviceWorkgroupCap and 16 <= device.limits.maxComputeWorkgroupSizeX and 4 <= device.limits.maxComputeWorkgroupSizeY",im2colScratchFits:"im2colBytes <= device.limits.maxStorageBufferBindingSize and im2colBytes <= device.limits.maxBufferSize",paddedIm2colScratchFits:"paddedIm2colBytes <= device.limits.maxStorageBufferBindingSize and paddedIm2colBytes <= device.limits.maxBufferSize",im2col3dBytes:'im2col3dElements * dtypeBytes("float32")',sgmatTileRows:"64 if sgmatUseF32WideTile else 32",sgmatWorkgroupThreads:"256 if sgmatUseF32WideTile else 128",sgmatSubgroupRows:"4 if sgmatUseF32WideTile else 2",implicitIm2colOk:'attrs.group == 1 and weightInChannels == inChannels and ((nchwRanksOk and conv2dOutputOk) or (conv1dOutputOk and ceilDiv(outChannels, 64) * ceilDiv(outputSpatial, 64) >= tunables.REG_MIN_TILE_COUNT and outputSpatial >= tunables.REG_MIN_SPATIAL and kernelRows >= tunables.REG_MIN_KERNEL_ROWS and (pinSubgroupSize32 or not device.features.has("shader-f16") or (wave32Adapter and kernelRows >= 8 * tunables.REG_MIN_KERNEL_ROWS and kernelRows <= 4 * outChannels))))',channelReduceOk:"nchwRanksOk and attrs.group == 1 and weightInChannels == inChannels and pointwiseUnitStride and conv2dOutputOk and outChannelsOk and inChannels >= 1 and outChannels <= tunables.CHANNEL_REDUCE_MAX_OUT_CHANNELS and outputSpatial >= tunables.CHANNEL_REDUCE_MIN_SPATIAL",halfRegKLimit:"min(64, pow2ceil(halfRegKCapacity) / (2 if pow2ceil(halfRegKCapacity) > halfRegKCapacity else 1))",winogradFullBytes:"16 * (outChannels * inChannels + batchSize * (inChannels + outChannels) * winogradTiles) * 4",winogradStripedBytes:"4 * (outChannels * inChannels + batchSize * (inChannels + 2 * outChannels) * winogradTiles) * 4",winogradEligible:'usesF16 and variableSubgroup16To32 and not device.features.has("chromium-experimental-subgroup-matrix") and nchwRanksOk and configuredWorkgroupOk and outChannelsOk and conv2dOutputOk and attrs.group == 1 and weightInChannels == inChannels and inChannels >= 32 and outChannels >= 32 and (ceilDiv(outChannels, 64) == 1 or inChannels >= tunables.WINOGRAD_WIDE_OUTPUT_MIN_CHANNELS) and batchSize * ceilDiv(outChannels, 64) * ceilDiv(winogradTiles, 64) >= tunables.WINOGRAD_MIN_WORKGROUPS and winogradEpilogueOk',narrowWideColumnCeiling:"pow2ceil(narrowWideColumnCapacity)",packedMaterializedFit:'variableSubgroup16To32 and dtypes.T == "f16" and outputSpatial % 4 != 0 and ceilDiv(outChannels, 64) > tunables.IMPLICIT_TILED_MAX_M_TILES and packedMaterializedBytes <= device.limits.maxStorageBufferBindingSize and packedMaterializedBytes <= device.limits.maxBufferSize',tiledSplitKCandidate:"16 if (tiledBaseWorkgroups * 16 <= tunables.TILED_SPLIT_K_TARGET_WORKGROUPS and tiledKTiles >= 16 * tunables.TILED_SPLIT_K_MIN_TILES_PER_SLICE) else 8 if (tiledBaseWorkgroups * 8 <= tunables.TILED_SPLIT_K_TARGET_WORKGROUPS and tiledKTiles >= 8 * tunables.TILED_SPLIT_K_MIN_TILES_PER_SLICE) else 4 if (tiledBaseWorkgroups * 4 <= tunables.TILED_SPLIT_K_TARGET_WORKGROUPS and tiledKTiles >= 4 * tunables.TILED_SPLIT_K_MIN_TILES_PER_SLICE) else 2 if (tiledBaseWorkgroups * 2 <= tunables.TILED_SPLIT_K_TARGET_WORKGROUPS and tiledKTiles >= 2 * tunables.TILED_SPLIT_K_MIN_TILES_PER_SLICE) else 1",shortNarrowHalfUnsplit:"variableSubgroup16To32 and usesF16 and nchwRanksOk and kernelHeight * kernelWidth > 1 and attrs.group == 1 and tiledBaseWorkgroups >= tunables.IMPLICIT_TILED_MIN_TILE_COUNT and halfRegTileRows < 64 and tiledSplitKStarved and tiledSplitKCandidate == 2 and tiledKTiles <= 4 * tunables.TILED_SPLIT_K_MIN_TILES_PER_SLICE and batchSize * ceilDiv(outChannels, halfRegTileRows) * ceilDiv(outputSpatial, 64) >= ceilDiv(tunables.REG_MIN_TILE_COUNT, 4) and (halfRegTileRows + 64) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize",im2col3dScratchFits:"im2col3dBytes <= device.limits.maxStorageBufferBindingSize and im2col3dBytes <= device.limits.maxBufferSize",sgmatDirectMatrixStore:'dtypes.T == "f32" and outChannels % sgmatTileRows == 0',tileRows:"sgmatTileRows",workgroupThreads:"sgmatWorkgroupThreads",subgroupRows:"sgmatSubgroupRows",sgmatYWorkgroups:"ceilDiv(outChannels, sgmatTileRows)",halfRegKTile:"halfRegKLimit if halfRegKLimit >= 32 and kernelRows % halfRegKLimit == 0 else tiledKTile",narrowWideColumns:"narrowWideColumnCeiling / (2 if narrowWideColumnCeiling > narrowWideColumnCapacity else 1)",tiledSplitK:'max(1, min(tiledSplitKCandidate, max(2, pow2ceil(ceilDiv(tunables.REG_MIN_TILE_COUNT, max(1, tiledBaseWorkgroups)))))) if (variableSubgroup16To32 and dtypes.T == "f32" and kernelHeight * kernelWidth > 1 and tiledSplitKStarved) or halfSplitReuse else 1 if shortNarrowHalfUnsplit else tiledSplitKCandidate if tiledSplitKStarved else 1',directMatrixStore:"sgmatDirectMatrixStore",implicitIm2colWorthIt:"sgmatYWorkgroups <= tunables.IMPLICIT_MAX_M_TILES and outputSpatial >= tunables.IMPLICIT_MIN_OUTPUT_SPATIAL",sgmatMPadded:"sgmatYWorkgroups * sgmatTileRows",sgmatBaseWorkgroups:"ceilDiv(outputSpatial, 64) * sgmatYWorkgroups * batchSize",implicitIm2col3dWorthIt:"sgmatYWorkgroups <= tunables.IMPLICIT_MAX_M_TILES and outputVolume >= tunables.IMPLICIT_MIN_OUTPUT_SPATIAL",halfRegWorkgroupFits:'128 <= device.limits.maxComputeInvocationsPerWorkgroup and 16 <= device.limits.maxComputeWorkgroupSizeX and 8 <= device.limits.maxComputeWorkgroupSizeY and (halfRegTileRows + 64) * halfRegKTile * dtypeBytes("float16") <= device.limits.maxComputeWorkgroupStorageSize',narrowWideF32:'variableSubgroup16To32 and dtypes.T == "f32" and halfRegTileRows > 0 and halfRegTileRows < 64 and narrowWideColumns > 64 and outputSpatial >= 4 * narrowWideColumns and kernelRows >= tiledKTile * 16 and (halfRegTileRows + narrowWideColumns) * tiledKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and (narrowWideColumns / 4) * (halfRegTileRows / narrowWideRowsPerThread) <= device.limits.maxComputeInvocationsPerWorkgroup and narrowWideColumns / 4 <= device.limits.maxComputeWorkgroupSizeX and halfRegTileRows / narrowWideRowsPerThread <= device.limits.maxComputeWorkgroupSizeY and batchSize * ceilDiv(outChannels, halfRegTileRows) * ceilDiv(outputSpatial, narrowWideColumns) >= tunables.REG_MIN_TILE_COUNT and not device.features.has("chromium-experimental-subgroup-matrix")',tiledSplitKChunk:"ceilDiv(tiledKTiles, tiledSplitK) * tiledKTile",sgmatSplitKStarved:"sgmatBaseWorkgroups >= 1 and sgmatBaseWorkgroups <= tunables.SGMAT_SPLIT_K_MAX_BASE_WORKGROUPS",sgmatSplitKCandidate:"16 if (sgmatBaseWorkgroups * 16 <= tunables.SGMAT_SPLIT_K_TARGET_WORKGROUPS and sgmatKTiles >= 16 * tunables.SGMAT_SPLIT_K_MIN_TILES_PER_SLICE) else 8 if (sgmatBaseWorkgroups * 8 <= tunables.SGMAT_SPLIT_K_TARGET_WORKGROUPS and sgmatKTiles >= 8 * tunables.SGMAT_SPLIT_K_MIN_TILES_PER_SLICE) else 4 if (sgmatBaseWorkgroups * 4 <= tunables.SGMAT_SPLIT_K_TARGET_WORKGROUPS and sgmatKTiles >= 4 * tunables.SGMAT_SPLIT_K_MIN_TILES_PER_SLICE) else 2 if (sgmatBaseWorkgroups * 2 <= tunables.SGMAT_SPLIT_K_TARGET_WORKGROUPS and sgmatKTiles >= 2 * tunables.SGMAT_SPLIT_K_MIN_TILES_PER_SLICE) else 1",compactMaterializedColumns:"variableSubgroup16To32 and usesF16 and kernelHeight * kernelWidth > 1 and tiledSplitK >= 2 and halfRegTileRows == 64 and tiledBaseWorkgroups < ceilDiv(tunables.REG_MIN_TILE_COUNT, 2) and im2colElements * 2 <= tiledSplitK * batchSize * tiledMPadded * tiledNPadded * 4",preferHalfColumnsForReuse:'dtypes.T == "f16" and device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and ceilDiv(outChannels, 64) > 1 and halfIm2colFits and halfRegWorkgroupFits and configuredWorkgroupOk and tiledBaseWorkgroups >= ceilDiv(tunables.REG_MIN_TILE_COUNT, 4 if compactMaterializedColumns else 2) and ceilDiv(outputSpatial, tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension and batchSize * kernelRows * outputSpatial * (ceilDiv(outChannels, 64) - 1) >= ceilDiv(tunables.MATERIALIZED_REUSE_MIN_ELEMENTS, 4 if compactMaterializedColumns else 1)',implicitIm2colPreferredOverDirect:"implicitIm2colOk and implicitIm2colWorthIt and sgmatYWorkgroups < tunables.DIRECT_IM2COL_MIN_M_TILES",shortImplicitProbe:"variableSubgroup16To32 and usesF16 and outputSpatial < tunables.REG_MIN_SPATIAL and shortImplicitGrid >= tunables.IMPLICIT_TILED_MIN_TILE_COUNT and kernelRows >= tunables.REG_MIN_KERNEL_ROWS and (halfRegTileRows + 64) * shortImplicitK * 4 <= device.limits.maxComputeWorkgroupStorageSize and halfRegWorkgroupFits and shortImplicitGrid <= max(tunables.IMPLICIT_TILED_MIN_TILE_COUNT, ceilDiv(tunables.REG_MIN_TILE_COUNT, 2))",tiledSplitKRequested:"tiledSplitK >= 2",tiledSplitKDispatchZ:"batchSize * tiledSplitK",tiledSplitKPartialElements:"tiledSplitK * batchSize * tiledMPadded * tiledNPadded",sgmatSplitK:"sgmatSplitKCandidate if sgmatSplitKStarved else 1",halfColumnVectorWidth:"4 if variableSubgroup16To32 and preferHalfColumnsForReuse and outputSpatial % 4 == 0 else 1",tiledSplitKPartialBytes:'tiledSplitKPartialElements * dtypeBytes("float32")',tiledSplitKDispatchFits:"tiledSplitKDispatchZ <= device.limits.maxComputeWorkgroupsPerDimension",sgmatSplitKRequested:"sgmatSplitK >= 2",sgmatSplitKChunk:"ceilDiv(sgmatKTiles, sgmatSplitK) * 32",sgmatSplitKDispatchZ:"batchSize * sgmatSplitK",sgmatSplitKPartialElements:"sgmatSplitK * batchSize * sgmatMPadded * paddedOutputSpatial",tiledSplitKPartialFits:"tiledSplitKPartialBytes <= tunables.TILED_SPLIT_K_MAX_PARTIAL_BYTES and tiledSplitKPartialBytes <= device.limits.maxStorageBufferBindingSize and tiledSplitKPartialBytes <= device.limits.maxBufferSize",sgmatSplitKPartialBytes:'sgmatSplitKPartialElements * dtypeBytes("float32")',sgmatSplitKDispatchFits:"sgmatSplitKDispatchZ <= device.limits.maxComputeWorkgroupsPerDimension",sgmatSplitKPartialFits:"sgmatSplitKPartialBytes <= tunables.SGMAT_SPLIT_K_MAX_PARTIAL_BYTES and sgmatSplitKPartialBytes <= device.limits.maxStorageBufferBindingSize and sgmatSplitKPartialBytes <= device.limits.maxBufferSize",paddedSplitKPreferred:"sgmatSplitKRequested and sgmatSplitKPartialFits and sgmatSplitKDispatchFits and paddedIm2colScratchFits",wideColumnRowsPerThread:"device.adapterInfo.subgroupMinSize / 2 if variableSubgroup16To32 and 64 / (device.adapterInfo.subgroupMinSize / 2) <= device.limits.maxComputeWorkgroupSizeY and 32 * 64 / (device.adapterInfo.subgroupMinSize / 2) <= deviceWorkgroupCap else 16"},when:["f16Ok(dtypes.T)"],bindings:{x:{elementType:"$T"},w:{elementType:"$T"},y:{elementType:"$T"},bias:{elementType:"$T"},x_main:{name:"x",elementType:"$scalar"},w_main:{name:"w",elementType:"$scalar"},y_main:{name:"y",elementType:"$outputElement"},xm:{arg:"x",elementType:"$T"},cols:{buffer:"storage",elementType:"f32"},params_im2col:{name:"params",struct:[{name:"outCount",type:"u32",value:"outputSpatial"},{name:"outW",type:"u32",value:"outputWidth"},{name:"inH",type:"u32",value:"inputHeight"},{name:"inW",type:"u32",value:"inputWidth"},{name:"inChannels",type:"u32",value:"inChannels"},{name:"kRows",type:"u32",value:"kernelRows"}]},xm_f32:{scratch:"cols",name:"xm",buffer:"read-only-storage",elementType:"f32"},y_f32:{scratch:"partial",name:"y",elementType:"f32"},partial:{buffer:"read-only-storage",elementType:"f32"},params_m_k_n:{name:"params",struct:[{name:"M",type:"u32",value:"outChannels"},{name:"K",type:"u32",value:"kernelRows"},{name:"N",type:"u32",value:"outputSpatial"}]},params_partial:{name:"params",struct:[{name:"M",type:"u32",value:"outChannels"},{name:"K",type:"u32",value:"inChannels"},{name:"N",type:"u32",value:"inputSpatial"}]},params_direct_unrolled:{name:"params",struct:[{name:"inChannels",type:"u32",value:"inChannels"},{name:"inH",type:"u32",value:"inputHeight"},{name:"inW",type:"u32",value:"inputWidth"},{name:"outChannels",type:"u32",value:"outChannels"},{name:"weightInChannels",type:"u32",value:"weightInChannels"},{name:"inChannelsPerGroup",type:"u32",value:"inChannelsPerGroup"},{name:"outChannelsPerGroup",type:"u32",value:"outChannelsPerGroup"},{name:"outH",type:"u32",value:"outputHeight"},{name:"outW",type:"u32",value:"outputWidth"},{name:"count",type:"u32",value:"numel(shapes.y)"}]}},variants:[{id:"grouped_large_kernel_w4_bias",priority:70,when:["biasContract","nchwRanksOk","groupOk","outChannelsOk","conv2dOutputOk","attrs.group > 1","numel(shapes.y) >= tunables.GROUPED_MIN_OUTPUT_ELEMENTS","kernelHeight >= tunables.GROUPED_MIN_KERNEL_SIZE","kernelHeight <= tunables.GROUPED_MAX_KERNEL_SIZE","kernelWidth >= tunables.GROUPED_MIN_KERNEL_SIZE","kernelWidth <= tunables.GROUPED_MAX_KERNEL_SIZE","outputWidth > 0","outputWidth % 4 == 0"],derive:{ocTile:"1 if variableSubgroup16To32 and groupedRollKernelRows and inChannelsPerGroup < outChannelsPerGroup and outChannelsPerGroup % 4 == 0 else (4 if (outChannels / attrs.group) % 4 == 0 else (2 if (outChannels / attrs.group) % 2 == 0 else 1))",tileWorkgroupSize:"min(tunables.GROUPED_WIDE_WORKGROUP_SIZE if outChannelsPerGroup % 2 == 0 else convWorkgroupSize, convWorkgroupSize)",scalar:"dtypes.T",outputElement:'"vec4<" ~ dtypes.T ~ ">"',vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Conv.GroupedLargeKernelW4Bias",shader:"ops/_shared/conv2d-grouped-large-w4.wgsl.jinja",derive:{countTiles:"numel(shapes.y) / (4 * ocTile)",span:"(kernelWidth - 1) * dilationW + 1 + 3 * strideW",outW4:"outputWidth / 4",tailOutput:!1,spanCap:"tunables.GROUPED_MAX_REGISTER_SPAN",workgroupSizeSpec:"tileWorkgroupSize",outW:"outputWidth",outH:"outputHeight",outC:"outChannels",outCPerGroup:"outChannels / attrs.group",inC:"inChannels",inCPerGroup:"inChannels / attrs.group",inH:"inputHeight",inW:"inputWidth",rollKernelRows:"groupedRollKernelRows",kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",dilatedLanes:!1,lanes:4,quads:1},bindings:["x_main","w_main","bias","y_main"],dispatch:{threads:"numel(shapes.y) / (4 * ocTile)",workgroupSize:"tileWorkgroupSize"},profile:{flops:"convFlops"}}],demoteWhen:["not (wave32Adapter or (kernelHeight <= tunables.GROUPED_SCALAR_OC_MAX_KERNEL_SIZE and kernelWidth <= tunables.GROUPED_SCALAR_OC_MAX_KERNEL_SIZE)) and not groupedRollKernelRows","false"]},{id:"grouped_large_kernel_w4_tail_bias",priority:70,when:["biasContract","nchwRanksOk","groupOk","outChannelsOk","conv2dOutputOk","attrs.group > 1","numel(shapes.y) >= tunables.GROUPED_MIN_OUTPUT_ELEMENTS","kernelHeight >= tunables.GROUPED_MIN_KERNEL_SIZE","kernelHeight <= tunables.GROUPED_MAX_KERNEL_SIZE","kernelWidth >= tunables.GROUPED_MIN_KERNEL_SIZE","kernelWidth <= tunables.GROUPED_MAX_KERNEL_SIZE","outputWidth > 0","outputWidth % 4 != 0"],derive:{ocTile:"1 if variableSubgroup16To32 and groupedRollKernelRows and inChannelsPerGroup < outChannelsPerGroup and outChannelsPerGroup % 4 == 0 else (4 if (outChannels / attrs.group) % 4 == 0 else (2 if (outChannels / attrs.group) % 2 == 0 else 1))",tileWorkgroupSize:"min(tunables.GROUPED_WIDE_WORKGROUP_SIZE if outChannelsPerGroup % 2 == 0 else convWorkgroupSize, convWorkgroupSize)",scalar:"dtypes.T",outputElement:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Conv.GroupedLargeKernelW4TailBias",shader:"ops/_shared/conv2d-grouped-large-w4.wgsl.jinja",derive:{countTiles:"dim(shapes.y, 0) * outputHeight * ceilDiv(outputWidth, 4) * (outChannels / ocTile)",span:"(kernelWidth - 1) * dilationW + 1 + 3 * strideW",outW4:"ceilDiv(outputWidth, 4)",tailOutput:!0,spanCap:"tunables.GROUPED_MAX_REGISTER_SPAN",workgroupSizeSpec:"tileWorkgroupSize",outW:"outputWidth",outH:"outputHeight",outC:"outChannels",outCPerGroup:"outChannels / attrs.group",inC:"inChannels",inCPerGroup:"inChannels / attrs.group",inH:"inputHeight",inW:"inputWidth",rollKernelRows:"groupedRollKernelRows",kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",dilatedLanes:!1,lanes:4,quads:1},bindings:["x_main","w_main","bias","y_main"],dispatch:{threads:"dim(shapes.y, 0) * outputHeight * ceilDiv(outputWidth, 4) * (outChannels / ocTile)",workgroupSize:"tileWorkgroupSize"},profile:{flops:"convFlops"}}],demoteWhen:["not (wave32Adapter or (kernelHeight <= tunables.GROUPED_SCALAR_OC_MAX_KERNEL_SIZE and kernelWidth <= tunables.GROUPED_SCALAR_OC_MAX_KERNEL_SIZE)) and not groupedRollKernelRows","false"]},{id:"gemm_1x1_subgroup_matrix_bias",priority:165,when:["wave32Effective","biasContract","nchwRanksOk","kernelHeight == 1","kernelWidth == 1","attrs.group == 1","weightInChannels == inChannels","strideH == 1","strideW == 1","dilationH == 1","dilationW == 1","padTop == 0 and padBottom == 0","padLeft == 0 and padRight == 0","outChannelsOk","outputHeight == inputHeight","outputWidth == inputWidth","inChannels >= tunables.SGMAT_MIN_ALIGNED_K","inChannels % 32 == 0","inputSpatial >= tunables.SGMAT_MIN_ALIGNED_N","inputSpatial % 64 == 0","outChannels >= tunables.SGMAT_MIN_M","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(inputSpatial / 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension",'subgroupMatrix8F16Available if dtypes.T == "f16" else subgroupMatrix8F32Available','(0 if outChannels % sgmatTileRows == 0 else (sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + (0 if sgmatDirectMatrixStore else (sgmatWorkgroupThreads / 32) * 4 * 64 * 4) + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{directMatrixInputs:"outChannels % sgmatTileRows == 0",K:"inChannels",N:"inputSpatial"},passes:[{id:"main",name:"Conv.Gemm1x1SubgroupMatrixBias",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm","bias","y"],dispatch:{x:"ceil(inputSpatial / 64)",y:"sgmatYWorkgroups",z:"batchSize"},profile:{flops:"convFlops",gemmK:"inChannels"},subgroupCollectivesWidth:"portable"}]},{id:"im2col_gemm_subgroup_matrix_bias",priority:151,when:["wave32Effective","biasContract","im2colRanksOk","attrs.group == 1","weightInChannels == inChannels","outChannelsOk","im2colOutputOk","kernelRows >= tunables.SGMAT_MIN_ALIGNED_K","kernelRows % 32 == 0","(ceilDiv(kernelRows, 32) * 32) <= device.limits.maxComputeWorkgroupsPerDimension","outputSpatial >= tunables.SGMAT_MIN_ALIGNED_N","outputSpatial % 64 == 0","outChannels >= tunables.SGMAT_MIN_M","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","im2colScratchFits","configuredWorkgroupOk",'(true) and (subgroupMatrix8F16Available if dtypes.T == "f16" else subgroupMatrix8F32Available)','(0 if false else (sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + (0 if sgmatDirectMatrixStore else (sgmatWorkgroupThreads / 32) * 4 * 64 * 4) + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{K:"kernelRows",N:"outputSpatial",directMatrixInputs:!1},intermediates:[{id:"cols",dtype:"float32",shape:"[im2colElements]"}],passes:[{id:"im2col",name:"Conv.Im2col",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",colsScalar:'"f32"'},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil(max(min(outputSpatial, tunables.IM2COL_MIN_THREADS), ceil(outputSpatial / tunables.IM2COL_ITEMS_PER_THREAD)) / tunables.WORKGROUP_SIZE)",y:"kernelRows",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmSubgroupMatrixBias",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm_f32","bias","y"],dispatch:{x:"ceil(outputSpatial / 64)",y:"sgmatYWorkgroups",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelRows"},subgroupCollectivesWidth:"portable"}]},{id:"im2col_direct_f32_subgroup_matrix_bias",priority:163,when:["wave32Effective","biasContract","im2colRanksOk","attrs.group == 1","weightInChannels == inChannels","outChannelsOk","im2colOutputOk","kernelRows >= tunables.SGMAT_MIN_ALIGNED_K","kernelRows % 32 == 0","(ceilDiv(kernelRows, 32) * 32) <= device.limits.maxComputeWorkgroupsPerDimension","outputSpatial >= tunables.SGMAT_MIN_ALIGNED_N","outputSpatial % 64 == 0","outChannels >= tunables.SGMAT_MIN_M","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","im2colScratchFits","configuredWorkgroupOk",'dtypes.T == "f32" and outChannels % sgmatTileRows == 0','(0 if true else (sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + (0 if sgmatDirectMatrixStore else (sgmatWorkgroupThreads / 32) * 4 * 64 * 4) + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",M:8,N:8,K:8,resultComponentType:"f32"}]},derive:{K:"kernelRows",N:"outputSpatial",directMatrixInputs:!0},intermediates:[{id:"cols",dtype:"float32",shape:"[im2colElements]"}],passes:[{id:"im2col",name:"Conv.Im2col",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",colsScalar:'"f32"'},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil(max(min(outputSpatial, tunables.IM2COL_MIN_THREADS), ceil(outputSpatial / tunables.IM2COL_ITEMS_PER_THREAD)) / tunables.WORKGROUP_SIZE)",y:"kernelRows",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmSubgroupMatrixBias",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm_f32","bias","y"],dispatch:{x:"ceil(outputSpatial / 64)",y:"sgmatYWorkgroups",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelRows"},subgroupCollectivesWidth:"portable"}],demoteWhen:["paddedSplitKPreferred","implicitIm2colPreferredOverDirect"]},{id:"implicit_im2col_subgroup_matrix_bias",priority:161,when:["wave32Effective","biasContract","implicitIm2colOk or implicitIm2col3dOk","implicitIm2col3dWorthIt","outChannelsOk","outChannels >= tunables.SGMAT_MIN_M","outputVolume >= tunables.SGMAT_MIN_ALIGNED_N","kernelVolumeRows >= tunables.SGMAT_MIN_PADDED_K","paddedKernelVolumeRows <= device.limits.maxComputeWorkgroupsPerDimension","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputVolume / 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension",'subgroupMatrix8F16Available if dtypes.T == "f16" else subgroupMatrix8F32Available','((sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + (sgmatWorkgroupThreads / 32) * 4 * 64 * 4 + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{K:"kernelVolumeRows",N:"outputVolume",padded:!0,kPadded:"paddedKernelVolumeRows",implicitIm2col:!0,convKernelH:"kernelHeight",convKernelW:"kernelWidth",convStrideH:"strideH",convStrideW:"strideW",convDilationH:"dilationH",convDilationW:"dilationW",convPadTop:"effectivePadTop",convPadLeft:"effectivePadLeft",convInH:"inputHeight",convInW:"inputWidth",convOutW:"outputWidth",convInChannels:"inChannels",convSpatialDims:"spatialRank",convKernelD:"kernelDepth",convStrideD:"strideD",convDilationD:"dilationD",convPadFront:"effectivePadFront",convInD:"inputDepth",convOutH:"outputHeight",directMatrixInputs:!1},passes:[{id:"main",name:"Conv.ImplicitIm2colSubgroupMatrixBias",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm","bias","y"],dispatch:{x:"ceil(outputVolume / 64)",y:"sgmatYWorkgroups",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelVolumeRows"},subgroupCollectivesWidth:"portable"}]},{id:"im2col_gemm_subgroup_matrix_padded_bias",priority:146,when:["wave32Effective","biasContract","im2colRanksOk","attrs.group == 1","weightInChannels == inChannels","outChannelsOk","im2colOutputOk","kernelRows >= tunables.SGMAT_MIN_PADDED_K","(ceilDiv(kernelRows, 32) * 32) <= device.limits.maxComputeWorkgroupsPerDimension","outputSpatial >= tunables.SGMAT_MIN_PADDED_N","outChannels >= tunables.SGMAT_MIN_M","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(outputSpatial, 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension","ceil((ceilDiv(outputSpatial, 64) * 64) / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","paddedIm2colScratchFits","configuredWorkgroupOk",'subgroupMatrix8F16Available if dtypes.T == "f16" else subgroupMatrix8F32Available','((sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + (sgmatWorkgroupThreads / 32) * 4 * 64 * 4 + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{K:"kernelRows",N:"outputSpatial",padded:!0,kPadded:"(ceilDiv(kernelRows, 32) * 32)",nPadded:"(ceilDiv(outputSpatial, 64) * 64)",directMatrixInputs:!1},intermediates:[{id:"cols",dtype:"float32",shape:"[paddedIm2colElements]"}],passes:[{id:"im2col",name:"Conv.Im2colPadded",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft"},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil((ceilDiv(outputSpatial, 64) * 64) / tunables.WORKGROUP_SIZE)",y:"(ceilDiv(kernelRows, 32) * 32)",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmSubgroupMatrixBiasPadded",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm_f32","bias","y"],dispatch:{x:"ceil((ceilDiv(outputSpatial, 64) * 64) / 64)",y:"sgmatYWorkgroups",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelRows"},subgroupCollectivesWidth:"portable"}]},{id:"im2col_gemm_subgroup_matrix_padded_bias_splitk",priority:153,when:["wave32Effective","biasContract","im2colRanksOk","attrs.group == 1","weightInChannels == inChannels","outChannelsOk","im2colOutputOk","kernelRows >= tunables.SGMAT_MIN_PADDED_K","outputSpatial >= tunables.SGMAT_MIN_PADDED_N","outChannels >= tunables.SGMAT_MIN_M","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(outputSpatial, 64) <= device.limits.maxComputeWorkgroupsPerDimension","sgmatYWorkgroups <= device.limits.maxComputeWorkgroupsPerDimension","ceil((ceilDiv(outputSpatial, 64) * 64) / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","configuredWorkgroupOk","sgmatSplitKRequested","sgmatSplitKPartialFits","sgmatSplitKDispatchFits","paddedIm2colScratchFits","paddedKernelRows <= device.limits.maxComputeWorkgroupsPerDimension",'subgroupMatrix8F16Available if dtypes.T == "f16" else subgroupMatrix8F32Available','((sgmatTileRows * 32 + 64 * 32) * (2 if dtypes.T == "f16" else 4)) + 0 + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize'],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{K:"kernelRows",N:"outputSpatial",padded:!0,kPadded:"paddedKernelRows",nPadded:"(ceilDiv(outputSpatial, 64) * 64)",splitK:"sgmatSplitK",mPadded:"sgmatMPadded",batchCount:"batchSize",reduceWorkgroupSize:"sgmatReduceWorkgroupSize",kChunk:"sgmatSplitKChunk",directMatrixInputs:!1},intermediates:[{id:"cols",dtype:"float32",shape:"[paddedIm2colElements]"},{id:"partial",dtype:"float32",shape:"[sgmatSplitKPartialElements]"}],passes:[{id:"im2col",name:"Conv.Im2colPadded",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft"},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil((ceilDiv(outputSpatial, 64) * 64) / tunables.WORKGROUP_SIZE)",y:"paddedKernelRows",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmSubgroupMatrixPaddedSplitK",shader:"ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",bindings:["w","xm_f32","y_f32"],dispatch:{x:"ceil((ceilDiv(outputSpatial, 64) * 64) / 64)",y:"sgmatYWorkgroups",z:"sgmatSplitKDispatchZ"},profile:{flops:"convFlops",gemmK:"sgmatSplitKChunk"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"Conv.SplitKReduceBias",shader:"ops/_shared/conv-splitk-reduce.wgsl.jinja",bindings:["partial","bias","y"],dispatch:{threads:"batchSize * outChannels * outputSpatial",workgroupSize:"sgmatReduceWorkgroupSize"}}]},{id:"im2col_gemm_tiled_bias_reg",priority:133,when:["biasContract","implicitIm2colOk","outChannelsOk","ceil(outChannels / 64) * ceil(outputSpatial / 64) >= tunables.REG_MIN_TILE_COUNT","outputSpatial >= tunables.REG_MIN_SPATIAL","kernelRows >= tunables.REG_MIN_KERNEL_ROWS","kernelRows <= device.limits.maxComputeWorkgroupsPerDimension","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outChannels / 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","im2colScratchFits","configuredWorkgroupOk","true"],derive:{gemmMTile:64,gemmThreadRows:"16 if variableSubgroup16To32 and not usesF16 else ceilDiv(gemmMTile, 8)",materializedK32Active:"spatialRank == 1 and (variableSubgroup16To32 and not usesF16 and (gemmMTile + 64) * 32 * 4 <= device.limits.maxComputeWorkgroupStorageSize)",gemmKTile:"32 if materializedK32Active else (32 if kernelRows % 32 == 0 else 16)",materializedColumnCapacity:"min(device.limits.maxComputeWorkgroupSizeX * 4, floor(deviceWorkgroupCap / ceilDiv(gemmMTile, gemmThreadRows)) * 4, floor(device.limits.maxComputeWorkgroupStorageSize / (gemmKTile * 4)) - gemmMTile)",materializedColumnPower:"pow2ceil(max(64, materializedColumnCapacity))",materializedColumnTile:"max(64, floor(materializedColumnPower / 2) if materializedColumnPower > materializedColumnCapacity else materializedColumnPower)",materializedWideActive:"materializedK32Active and materializedColumnTile >= 128 and batchSize * ceilDiv(outChannels, gemmMTile) * ceilDiv(outputSpatial, materializedColumnTile) >= tunables.REG_MIN_TILE_COUNT",gemmNTile:"materializedColumnTile if materializedWideActive else 64",gemmWorkgroupX:"gemmNTile / 4"},intermediates:[{id:"cols",dtype:"float32",shape:"[im2colElements]"}],passes:[{id:"im2col",name:"Conv.Im2col",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",colsScalar:'"f32"',colsVectorWidth:1,columnElement:'"vec4<f16>" if colsVectorWidth == 4 else colsScalar'},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil(max(min(outputSpatial / colsVectorWidth, tunables.IM2COL_MIN_THREADS), ceil(outputSpatial / (colsVectorWidth * tunables.IM2COL_ITEMS_PER_THREAD))) / tunables.WORKGROUP_SIZE)",y:"kernelRows",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmTiledBiasReg",shader:"ops/_shared/conv-1x1-gemm-tiled-reg.wgsl.jinja",bindings:["w","xm_f32","bias","y","params_m_k_n"],dispatch:{x:"ceilDiv(outputSpatial, gemmNTile)",y:"ceil(outChannels / 64)",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelRows"}}]},{id:"gemm_1x1_tiled_bias_reg",priority:143,when:["biasContract","nchwRanksOk","kernelHeight == 1","kernelWidth == 1","attrs.group == 1","weightInChannels == inChannels","strideH == 1","strideW == 1","dilationH == 1","dilationW == 1","padTop == 0 and padBottom == 0","padLeft == 0 and padRight == 0","outChannelsOk","outputHeight == inputHeight","outputWidth == inputWidth","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(inputSpatial / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outChannels / 32) <= device.limits.maxComputeWorkgroupsPerDimension","(ceil(outChannels / 64) * ceil(inputSpatial / 64) >= tunables.REG_MIN_TILE_COUNT) or unsplitReuseFit","inputSpatial >= tunables.REG_MIN_SPATIAL","inChannels >= tunables.REG_MIN_KERNEL_ROWS"],derive:{gemmKTile:"32 if inChannels % 32 == 0 else 16",gemmMTile:"32 if unsplitReuseFit and ceilDiv(outChannels, 64) * 64 > ceilDiv(outChannels, 32) * 32 else 64",gemmTileScalar:'"f32" if variableSubgroup16To32 and usesF16 and (64 + 64) * gemmKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize else ("f16" if usesF16 else "f32")',pointwiseWideActive:"variableSubgroup16To32 and usesF16 and (64 + 128) * gemmKTile * 4 <= device.limits.maxComputeWorkgroupStorageSize and 32 <= device.limits.maxComputeWorkgroupSizeX and 4 <= device.limits.maxComputeWorkgroupSizeY and 128 <= deviceWorkgroupCap and batchSize * ceilDiv(outChannels, 64) * ceilDiv(inputSpatial, 128) >= tunables.REG_MIN_TILE_COUNT",gemmNTile:"128 if pointwiseWideActive else 64",gemmThreadRows:"16 if pointwiseWideActive else 8",gemmWorkgroupX:"gemmNTile / 4",gemmWorkgroupY:"gemmMTile / gemmThreadRows"},passes:[{id:"main",name:"Conv.Gemm1x1TiledRegBias",shader:"ops/_shared/conv-1x1-gemm-tiled-reg.wgsl.jinja",bindings:["w","xm","bias","y","params_partial"],dispatch:{x:"ceilDiv(inputSpatial, gemmNTile)",y:"ceilDiv(outChannels, gemmMTile)",z:"batchSize"},profile:{flops:"convFlops",gemmK:"inChannels"}}]},{id:"gemm_1x1_tiled_bias",priority:141,when:["biasContract","nchwRanksOk","kernelHeight == 1","kernelWidth == 1","attrs.group == 1","weightInChannels == inChannels","strideH == 1","strideW == 1","dilationH == 1","dilationW == 1","padTop == 0 and padBottom == 0","padLeft == 0 and padRight == 0","outChannelsOk","outputHeight == inputHeight","outputWidth == inputWidth","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(inputSpatial / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outChannels / 32) <= device.limits.maxComputeWorkgroupsPerDimension"],derive:{},passes:[{id:"main",name:"Conv.Gemm1x1TiledBias",shader:"ops/_shared/conv-1x1-gemm-tiled.wgsl.jinja",bindings:["w","xm","bias","y","params_partial"],dispatch:{x:"ceil(inputSpatial / 32)",y:"ceil(outChannels / 32)",z:"batchSize"},profile:{flops:"convFlops",gemmK:"inChannels"}}]},{id:"gemm_1x1_tiled_bias_reg_splitk",priority:145,when:["tunables.TILED_SPLIT_K_MODE == 1","biasContract","nchwRanksOk","kernelHeight == 1","kernelWidth == 1","attrs.group == 1","weightInChannels == inChannels","strideH == 1","strideW == 1","dilationH == 1","dilationW == 1","padTop == 0 and padBottom == 0","padLeft == 0 and padRight == 0","outChannelsOk","outputHeight == inputHeight","outputWidth == inputWidth","inChannels >= tunables.REG_MIN_KERNEL_ROWS","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(inputSpatial, 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(outChannels, 64) <= device.limits.maxComputeWorkgroupsPerDimension","tiledSplitKRequested","tiledSplitKPartialFits","tiledSplitKDispatchFits"],derive:{hasBias:!1,gemmKTile:"tiledKTile",N:"inputSpatial",splitK:"tiledSplitK",kTiles:"tiledKTiles",mPadded:"tiledMPadded",nPadded:"tiledNPadded",batchCount:"batchSize",reduceWorkgroupSize:"tiledReduceWorkgroupSize"},intermediates:[{id:"partial",dtype:"float32",shape:"[tiledSplitKPartialElements]"}],passes:[{id:"partial",name:"Conv.Gemm1x1TiledRegSplitKBias",shader:"ops/_shared/conv-1x1-gemm-tiled-reg.wgsl.jinja",bindings:["w","xm","y_f32","params_partial"],dispatch:{x:"ceilDiv(inputSpatial, 64)",y:"ceilDiv(outChannels, 64)",z:"tiledSplitKDispatchZ"},profile:{flops:"convFlops",gemmK:"tiledSplitKChunk"}},{id:"reduce",name:"Conv.Gemm1x1TiledSplitKReduceBias",shader:"ops/_shared/conv-splitk-reduce.wgsl.jinja",derive:{hasBias:!0},bindings:["partial","bias","y"],dispatch:{threads:"batchSize * outChannels * inputSpatial",workgroupSize:"tiledReduceWorkgroupSize"}}],demoteWhen:["unsplitReuseFit"]},{id:"direct_unrolled_bias",priority:61,when:["biasContract","nchwRanksOk","groupOk","outChannelsOk","conv2dOutputOk","kernelHeight >= 1","kernelHeight <= tunables.DIRECT_MAX_KERNEL_SIZE","kernelWidth >= 1","kernelWidth <= tunables.DIRECT_MAX_KERNEL_SIZE","configuredWorkgroupOk"],supersededBy:["im2col_gemm_tiled_bias_reg","im2col_gemm_tiled_bias"],derive:{},passes:[{id:"main",name:"Conv.DirectUnrolledBias",shader:"ops/_shared/conv-direct-unrolled.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft"},bindings:["x","w","bias","y","params_direct_unrolled"],dispatch:{threads:"numel(shapes.y)",workgroupSize:"tunables.WORKGROUP_SIZE"},profile:{flops:"convFlops"}}]},{id:"im2col_gemm_tiled_bias",priority:131,when:["biasContract","implicitIm2colOk","outChannelsOk","kernelRows <= device.limits.maxComputeWorkgroupsPerDimension","batchSize >= 1","batchSize <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outChannels / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(outputSpatial / tunables.WORKGROUP_SIZE) <= device.limits.maxComputeWorkgroupsPerDimension","im2colScratchFits","configuredWorkgroupOk","true"],derive:{},intermediates:[{id:"cols",dtype:"float32",shape:"[im2colElements]"}],passes:[{id:"im2col",name:"Conv.Im2col",shader:"ops/_shared/conv-im2col-nchw.wgsl.jinja",derive:{kernelHSpec:"kernelHeight",kernelWSpec:"kernelWidth",strideHSpec:"strideH",strideWSpec:"strideW",dilationHSpec:"dilationH",dilationWSpec:"dilationW",padTopSpec:"effectivePadTop",padLeftSpec:"effectivePadLeft",colsScalar:'"f32"',colsVectorWidth:1,columnElement:'"vec4<f16>" if colsVectorWidth == 4 else colsScalar'},bindings:["x","cols","params_im2col"],dispatch:{x:"ceil(max(min(outputSpatial / colsVectorWidth, tunables.IM2COL_MIN_THREADS), ceil(outputSpatial / (colsVectorWidth * tunables.IM2COL_ITEMS_PER_THREAD))) / tunables.WORKGROUP_SIZE)",y:"kernelRows",z:"batchSize"}},{id:"gemm",name:"Conv.Im2colGemmTiledBias",shader:"ops/_shared/conv-1x1-gemm-tiled.wgsl.jinja",bindings:["w","xm_f32","bias","y","params_m_k_n"],dispatch:{x:"ceil(outputSpatial / 32)",y:"ceil(outChannels / 32)",z:"batchSize"},profile:{flops:"convFlops",gemmK:"kernelRows"}}]}]},assets:[["ops/_shared/conv2d-grouped-large-w4.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set laneStep = dilationWSpec if dilatedLanes else strideWSpec %}{% set comps = ["x", "y", "z", "w"] %}{% macro acc_ref(oct, lane) %}acc{{ oct }}.{{ comps[lane] }}{% endmacro %}{% set usedCount = namespace(value=0) %}{% for s in range(span) %}{% set used = namespace(value=false) %}{% for kw in range(kernelWSpec) %}{% for lane in range(lanes) %}{% if s == kw * dilationWSpec + lane * laneStep %}{% set used.value = true %}{% endif %}{% endfor %}{% endfor %}{% if used.value %}{% set usedCount.value = usedCount.value + 1 %}{% endif %}{% endfor %}{% set registerForm = (usedCount.value <= spanCap) if dilatedLanes else (span <= spanCap) %}const COUNT_TILES:u32={{ countTiles }}u;
{% if tailOutput %}const OUT_W: u32 = max(1u, {{ outW }}u);
{% endif %}const OUT_W4:u32=max(1u,{{ outW4 }}u);const OUT_H:u32=max(1u,{{ outH }}u);const OUT_C:u32=max(1u,{{ outC }}u);const OC_TILE:u32={{ ocTile }}u;const OUT_C_TILES:u32=OUT_C/OC_TILE;const OUT_CPG:u32={{ outCPerGroup }}u;const IN_C:u32={{ inC }}u;const IN_CPG:u32={{ inCPerGroup }}u;const IN_H:i32={{ inH }};const IN_W:i32={{ inW }};const IN_H_U:u32={{ inH }}u;const IN_W_U:u32={{ inW }}u;const IN_PLANE:u32=IN_H_U*IN_W_U;
{% if not tailOutput and not dilatedLanes %}const OUT_PLANE4: u32 = OUT_H * OUT_W4;
{% endif %}const KAREA:u32={{ kernelHSpec * kernelWSpec }}u;const STRIDE_H:u32={{ strideHSpec }}u;const STRIDE_W:i32={{ strideWSpec }};const PAD_TOP:i32={{ padTopSpec }};const PAD_LEFT:i32={{ padLeftSpec }};const WG:u32={{ workgroupSizeSpec }}u;@compute@workgroup_size({{ workgroupSizeSpec }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WG", "q", "COUNT_TILES") }} let ow4=q%OUT_W4;var t=q/OUT_W4;let oh=t%OUT_H;t/=OUT_H;let ocBase=(t%OUT_C_TILES)*OC_TILE;let batch=t/OUT_C_TILES;let group=ocBase/OUT_CPG;let ihBase=i32(oh*STRIDE_H)-PAD_TOP;let iwBase=i32(ow4*4u)*STRIDE_W-PAD_LEFT;var xChannelBase=(batch*IN_C+group*IN_CPG)*IN_PLANE;
{% for oct in range(ocTile) %}var wBase{{ oct }}=(ocBase+{{ oct }}u)*IN_CPG*KAREA;let bias{{ oct }}=f32(bias[ocBase+{{ oct }}u]);var acc{{ oct }}=vec4<f32>(bias{{ oct }});
{% endfor %}for(var ic=0u;ic<IN_CPG;ic++){
{% if rollKernelRows %}  for (var kernelRow = 0u; kernelRow < {{ kernelHSpec }}u; kernelRow++) {
{% endif %}{% for kh in range(1 if rollKernelRows else kernelHSpec) %}{
      let ih = ihBase + {% if rollKernelRows %}i32(kernelRow) * {{ dilationHSpec }}{% else %}{{ kh * dilationHSpec }}{% endif %};
if(ih>=0&&ih<IN_H){let xRow=xChannelBase+u32(ih)*IN_W_U;
{% if registerForm %}{% for s in range(span) %}{% set used = namespace(value=false) %}{% for kw in range(kernelWSpec) %}{% for lane in range(lanes) %}{% if s == kw * dilationWSpec + lane * laneStep %}{% set used.value = true %}{% endif %}{% endfor %}{% endfor %}{% if used.value %}var xw{{ s }}=0.0;
{% endif %}{% endfor %}{% for s in range(span) %}{% set used = namespace(value=false) %}{% for kw in range(kernelWSpec) %}{% for lane in range(lanes) %}{% if s == kw * dilationWSpec + lane * laneStep %}{% set used.value = true %}{% endif %}{% endfor %}{% endfor %}{% if used.value %}{let iw=iwBase+{{ s }};if(iw>=0&&iw<IN_W){xw{{ s }}=f32(x[xRow+u32(iw)]);} }
{% endif %}{% endfor %}{% for kw in range(kernelWSpec) %}{
{% for oct in range(ocTile) %}          let weight{{ oct }} = f32(w[wBase{{ oct }} + {% if rollKernelRows %}kernelRow * {{ kernelWSpec }}u + {{ kw }}u{% else %}{{ kh * kernelWSpec + kw }}u{% endif %}]);
{% endfor %}{% for lane in range(lanes) %}{% for oct in range(ocTile) %}{{ acc_ref(oct, lane) }}+=xw{{ kw * dilationWSpec + lane * laneStep }}*weight{{ oct }};
{% endfor %}{% endfor %}}
{% endfor %}{% else %}{% for kw in range(kernelWSpec) %}{
{% for oct in range(ocTile) %}          let weight{{ oct }} = f32(w[wBase{{ oct }} + {% if rollKernelRows %}kernelRow * {{ kernelWSpec }}u + {{ kw }}u{% else %}{{ kh * kernelWSpec + kw }}u{% endif %}]);
{% endfor %}let iwK=iwBase+{{ kw * dilationWSpec }};
{% for lane in range(lanes) %}let iwLane{{ lane }}=iwK+{{ lane * laneStep }};if(iwLane{{ lane }}>=0&&iwLane{{ lane }}<IN_W){let xValue=f32(x[xRow+u32(iwLane{{ lane }})]);
{% for oct in range(ocTile) %}{{ acc_ref(oct, lane) }}+=xValue*weight{{ oct }};
{% endfor %}}
{% endfor %}}
{% endfor %}{% endif %}} }
{% endfor %}{% if rollKernelRows %}  }
{% endif %}    xChannelBase += IN_PLANE;
{% for oct in range(ocTile) %}wBase{{ oct }}+=KAREA;
{% endfor %}}
{% if tailOutput %}let outBase=((batch*OUT_C+ocBase)*OUT_H+oh)*OUT_W+ow4*4u;
{% for oct in range(ocTile) %}let outChannelBase{{ oct }}=outBase+{{ oct }}u*OUT_H*OUT_W;
{% for lane in range(4) %}if(ow4*4u+{{ lane }}u<OUT_W){y[outChannelBase{{ oct }}+{{ lane }}u]={{ scalar }}(acc{{ oct }}.{{ ["x", "y", "z", "w"][lane] }});}
{% endfor %}{% endfor %}{% else %}let outBase=((batch*OUT_C+ocBase)*OUT_H+oh)*OUT_W4+ow4;
{% for oct in range(ocTile) %}y[outBase+{{ oct }}u*OUT_PLANE4]={{ vectorScalar }}(acc{{ oct }});
{% endfor %}{% endif %}}`],["ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",`{% set fusedNarrowProjection = false %}{% set polyphase = false %}{% set implicit = implicitIm2col is defined and implicitIm2col %}{% set implicit3d = implicit and (convSpatialDims is defined and convSpatialDims == 3) %}{% set splitKValue = splitK if splitK is defined else 1 %}{% set splitKPartial = splitKValue > 1 %}{% set kLoopVar = "K_LOOP" if padded else "K" %}{% set nColsVar = "N_COLS" if padded else "N" %}{% set convKernelH = convKernelH | default(0) %}{% set convKernelW = convKernelW | default(0) %}{% set convStrideH = convStrideH | default(0) %}{% set convStrideW = convStrideW | default(0) %}{% set convDilationH = convDilationH | default(0) %}{% set convDilationW = convDilationW | default(0) %}{% set convPadTop = convPadTop | default(0) %}{% set convPadLeft = convPadLeft | default(0) %}{% set convInH = convInH | default(0) %}{% set convInW = convInW | default(0) %}{% set convOutW = convOutW | default(0) %}{% set convInChannels = convInChannels | default(0) %}{% set convKernelD = convKernelD | default(0) %}{% set convStrideD = convStrideD | default(0) %}{% set convDilationD = convDilationD | default(0) %}{% set convPadFront = convPadFront | default(0) %}{% set convInD = convInD | default(0) %}{% set convOutH = convOutH | default(0) %}{% set nPadded = nPadded | default(0) %}{% set mPadded = mPadded | default(0) %}{% set kChunk = kChunk | default(0) %}{% set batchCount = batchCount | default(0) %}{% include "ops/_shared/subgroup-matrix-enable.de53490305.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set operandScalar = "f32" %}{% set operandZero = "0.0" %}{% set accScalar = "f32" %}{% set useDirectMatrixStore = directMatrixStore is defined and directMatrixStore %}{% set needsScratchStore = (not useDirectMatrixStore or padded) and not splitKPartial and not fusedNarrowProjection %}{% set tileRowsValue = tileRows if tileRows is defined else 32 %}{% set tileColsValue = 64 %}{% set workgroupThreadsValue = workgroupThreads if workgroupThreads is defined else 128 %}{% set subgroupRowsValue = subgroupRows if subgroupRows is defined else 2 %}{% set subRowsValue = (tileRowsValue / subgroupRowsValue)|int %}{% set subColsValue = 32 %}{% set bLoadWidth = (tileColsValue * 32 / workgroupThreadsValue)|int %}{% set bKChunks = (32 / bLoadWidth)|int %}const M:u32={{ M }}u;const K:u32={{ K }}u;
{% if not (splitKPartial and padded) or implicit %}const N:u32={{ N }}u;
{% endif %}{% if padded %}const K_LOOP:u32={{ kPadded }}u;
{% endif %}{% if implicit %}const CONV_KERNEL_H:u32={{ convKernelH }}u;const CONV_KERNEL_W:u32={{ convKernelW }}u;
{% if implicit3d %}const CONV_KERNEL_D:u32={{ convKernelD }}u;const CONV_KSIZE_HW:u32=CONV_KERNEL_H*CONV_KERNEL_W;const CONV_KSIZE:u32=CONV_KERNEL_D*CONV_KSIZE_HW;const CONV_STRIDE_D:u32={{ convStrideD }}u;const CONV_DILATION_D:u32={{ convDilationD }}u;const CONV_PAD_FRONT:i32={{ convPadFront }};const CONV_IN_D:i32={{ convInD }};const CONV_OUT_H:u32={{ convOutH }}u;
{% else %}const CONV_KSIZE:u32=CONV_KERNEL_H*CONV_KERNEL_W;
{% endif %}const CONV_STRIDE_H:u32={{ convStrideH }}u;const CONV_STRIDE_W:u32={{ convStrideW }}u;const CONV_DILATION_H:u32={{ convDilationH }}u;const CONV_DILATION_W:u32={{ convDilationW }}u;const CONV_PAD_TOP:i32={{ convPadTop }};const CONV_PAD_LEFT:i32={{ convPadLeft }};const CONV_IN_H:i32={{ convInH }};const CONV_IN_W:i32={{ convInW }};const CONV_OUT_W:u32={{ convOutW }}u;
{% if implicit3d %}const B_BATCH_STRIDE:u32={{ convInChannels }}u*u32(CONV_IN_D)*u32(CONV_IN_H)*u32(CONV_IN_W);
{% else %}const B_BATCH_STRIDE:u32={{ convInChannels }}u*u32(CONV_IN_H)*u32(CONV_IN_W);
{% endif %}{% else %}{% if padded %}const N_COLS:u32={{ nPadded }}u;
{% endif %}const B_BATCH_STRIDE:u32={{ kLoopVar }}*{{ nColsVar }};
{% endif %}{% if not splitKPartial and not polyphase and not fusedNarrowProjection %}const C_BATCH_STRIDE:u32=M*N;
{% endif %}{% if splitKPartial %}const SPLIT_K:u32={{ splitKValue }}u;const K_CHUNK:u32={{ kChunk }}u;const PARTIAL_ROWS:u32={{ mPadded }}u;const PARTIAL_COLS:u32={{ nPadded }}u;const PARTIAL_SLICE_STRIDE:u32={{ batchCount }}u*PARTIAL_ROWS*PARTIAL_COLS;
{% endif %}const TILE_COLS:u32={{ tileColsValue }}u;const TILE_ROWS:u32={{ tileRowsValue }}u;const TILE_K:u32=32u;const SUB_COLS:u32={{ subColsValue }}u;const SUB_ROWS:u32={{ subRowsValue }}u;
{% if not directMatrixInputs %}var<workgroup>tile_A:array<{{ operandScalar }},{{ tileRowsValue }}*32>;var<workgroup>tile_B:array<{{ operandScalar }},{{ tileColsValue }}*32>;
{% endif %}{% if needsScratchStore %}var<workgroup>scratch:array<array<array<{{ accScalar }},64>,4>,{{ (workgroupThreadsValue / 32)|int }}>;
{% endif %}{% if not directMatrixInputs %}fn loadSHMA(tile_base:u32,k_idx:u32,row:u32,c_idx:u32){let a_global=tile_base+row;let col=c_idx*8u;for(var col_offset=0u;col_offset<8u;col_offset=col_offset+1u){let k=k_idx+col+col_offset;
    if (a_global < M{% if padded %} && k < K{% endif %}) {
{% set A_PHASE = "" %}tile_A[row*TILE_K+col+col_offset]={{ operandScalar }}(w[{{ A_PHASE }}a_global*K+k]);}else{tile_A[row*TILE_K+col+col_offset]={{ operandZero }};} } }fn loadSHMB(b_base:u32,tile_base:u32,k_idx:u32,row:u32,c_idx:u32){let b_col=tile_base+row;let col=c_idx*{{ bLoadWidth }}u;
{% if implicit %}{% if implicit3d %}let id0=i32((b_col/(CONV_OUT_W*CONV_OUT_H))*CONV_STRIDE_D)-CONV_PAD_FRONT;let ih0=i32(((b_col/CONV_OUT_W)%CONV_OUT_H)*CONV_STRIDE_H)-CONV_PAD_TOP;
{% else %}let ih0=i32((b_col/CONV_OUT_W)*CONV_STRIDE_H)-CONV_PAD_TOP;
{% endif %}let iw0=i32((b_col%CONV_OUT_W)*CONV_STRIDE_W)-CONV_PAD_LEFT;let in_column=b_col<N;var k=k_idx+col;var ic=k/CONV_KSIZE;var kq=k%CONV_KSIZE;
{% if implicit3d %}var kd=kq/CONV_KSIZE_HW;var khw=kq%CONV_KSIZE_HW;var kh=khw/CONV_KERNEL_W;var kw=khw%CONV_KERNEL_W;var id=id0+i32(kd*CONV_DILATION_D);
{% else %}var kh=kq/CONV_KERNEL_W;var kw=kq%CONV_KERNEL_W;
{% endif %}var ih=ih0+i32(kh*CONV_DILATION_H);var iw=iw0+i32(kw*CONV_DILATION_W);let interior=in_column&&k_idx+col+{{ bLoadWidth }}u<=K
{% if implicit3d %}&&id0>=0&&id0+i32((CONV_KERNEL_D-1u)*CONV_DILATION_D)<CONV_IN_D
{% endif %}&&ih0>=0&&ih0+i32((CONV_KERNEL_H-1u)*CONV_DILATION_H)<CONV_IN_H&&iw0>=0&&iw0+i32((CONV_KERNEL_W-1u)*CONV_DILATION_W)<CONV_IN_W;if(interior){let colStep=i32(CONV_DILATION_W);let rowStep=i32(CONV_DILATION_H)*CONV_IN_W;
{% if implicit3d %}let depthStep=i32(CONV_DILATION_D)*CONV_IN_H*CONV_IN_W;let planeStep=CONV_IN_D*CONV_IN_H*CONV_IN_W;
{% else %}let planeStep=CONV_IN_H*CONV_IN_W;
{% endif %}let kwWrap=colStep*i32(CONV_KERNEL_W)-rowStep;
{% if implicit3d %}let khWrap=rowStep*i32(CONV_KERNEL_H)-depthStep;let kdWrap=depthStep*i32(CONV_KERNEL_D)-planeStep;var addr=i32(b_base)+i32(ic)*planeStep+(id*CONV_IN_H+ih)*CONV_IN_W+iw;
{% else %}let khWrap=rowStep*i32(CONV_KERNEL_H)-planeStep;var addr=i32(b_base)+i32(ic)*planeStep+ih*CONV_IN_W+iw;
{% endif %}for(var i=0u;i<{{ bLoadWidth }}u;i=i+1u){tile_B[row*TILE_K+col+i]={{ operandScalar }}(xm[u32(addr)]);addr=addr+colStep;kw=kw+1u;if(kw==CONV_KERNEL_W){kw=0u;addr=addr-kwWrap;kh=kh+1u;if(kh==CONV_KERNEL_H){kh=0u;addr=addr-khWrap;
{% if implicit3d %}kd=kd+1u;if(kd==CONV_KERNEL_D){kd=0u;addr=addr-kdWrap;}
{% endif %}} } }return;}for(var i=0u;i<{{ bLoadWidth }}u;i=i+1u){var value={{ operandScalar }}(0.0);
    if (in_column && k < K{% if implicit3d %} && id >= 0 && id < CONV_IN_D{% endif %} && ih >= 0 && ih < CONV_IN_H && iw >= 0 && iw < CONV_IN_W) {
{% if implicit3d %}value={{ operandScalar }}(xm[b_base+((ic*u32(CONV_IN_D)+u32(id))*u32(CONV_IN_H)+u32(ih))*u32(CONV_IN_W)+u32(iw)]);
{% else %}value={{ operandScalar }}(xm[b_base+(ic*u32(CONV_IN_H)+u32(ih))*u32(CONV_IN_W)+u32(iw)]);
{% endif %}}tile_B[row*TILE_K+col+i]=value;k=k+1u;kw=kw+1u;iw=iw+i32(CONV_DILATION_W);if(kw==CONV_KERNEL_W){kw=0u;iw=iw0;kh=kh+1u;ih=ih+i32(CONV_DILATION_H);if(kh==CONV_KERNEL_H){kh=0u;ih=ih0;
{% if implicit3d %}kd=kd+1u;id=id+i32(CONV_DILATION_D);if(kd==CONV_KERNEL_D){kd=0u;id=id0;ic=ic+1u;}
{% else %}ic=ic+1u;
{% endif %}} } }
{% else %}for(var i=0u;i<{{ bLoadWidth }}u;i=i+1u){let k=k_idx+col+i;tile_B[row*TILE_K+col+i]={{ operandScalar }}(xm[b_base+k*{{ nColsVar }}+b_col]);}
{% endif %}}
{% endif %}{% set biasAdd = " + bv" %}{% macro store_val(valExpr, idxExpr) %}{{ T }}({{ valExpr }}){% endmacro %}{% if needsScratchStore %}fn storeOutput(offset: u32, row_base: u32, row: u32, col: u32, src_slot: u32, row_limit: i32{% if padded %}, col_anchor: u32{% endif %}) {
if(row_limit>0&&row<u32(row_limit)){let col2=col+1u;let bv={{ accScalar }}(bias[row_base+row]);
{% if padded %}if(col_anchor+col+0u<N){y[offset+row*N+col]={{ store_val("scratch[src_slot][0][row * 8u + col]" ~ biasAdd, "offset + row * N + col") }};}if(col_anchor+col+8u<N){y[offset+row*N+col+8u]={{ store_val("scratch[src_slot][1][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 8u") }};}if(col_anchor+col+16u<N){y[offset+row*N+col+16u]={{ store_val("scratch[src_slot][2][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 16u") }};}if(col_anchor+col+24u<N){y[offset+row*N+col+24u]={{ store_val("scratch[src_slot][3][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 24u") }};}if(col_anchor+col2+0u<N){y[offset+row*N+col2]={{ store_val("scratch[src_slot][0][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2") }};}if(col_anchor+col2+8u<N){y[offset+row*N+col2+8u]={{ store_val("scratch[src_slot][1][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 8u") }};}if(col_anchor+col2+16u<N){y[offset+row*N+col2+16u]={{ store_val("scratch[src_slot][2][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 16u") }};}if(col_anchor+col2+24u<N){y[offset+row*N+col2+24u]={{ store_val("scratch[src_slot][3][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 24u") }};}
{% else %}y[offset+row*N+col]={{ store_val("scratch[src_slot][0][row * 8u + col]" ~ biasAdd, "offset + row * N + col") }};y[offset+row*N+col+8u]={{ store_val("scratch[src_slot][1][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 8u") }};y[offset+row*N+col+16u]={{ store_val("scratch[src_slot][2][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 16u") }};y[offset+row*N+col+24u]={{ store_val("scratch[src_slot][3][row * 8u + col]" ~ biasAdd, "offset + row * N + col + 24u") }};y[offset+row*N+col2]={{ store_val("scratch[src_slot][0][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2") }};y[offset+row*N+col2+8u]={{ store_val("scratch[src_slot][1][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 8u") }};y[offset+row*N+col2+16u]={{ store_val("scratch[src_slot][2][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 16u") }};y[offset+row*N+col2+24u]={{ store_val("scratch[src_slot][3][row * 8u + col2]" ~ biasAdd, "offset + row * N + col2 + 24u") }};
{% endif %}} }
{% endif %}{% include "ops/_shared/subgroup-matrix-topology.5c216ddcce.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.cedda9952e.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}{% if splitKPartial %}let slice=workgroup_id.z%SPLIT_K;let batch=workgroup_id.z/SPLIT_K;let b_base=batch*B_BATCH_STRIDE;let c_base=slice*PARTIAL_SLICE_STRIDE+batch*PARTIAL_ROWS*PARTIAL_COLS;
{% else %}let batch=workgroup_id.z;let b_base=batch*B_BATCH_STRIDE;let c_base=batch*C_BATCH_STRIDE;
{% endif %}let a_global_base=workgroup_id.y*TILE_ROWS;let b_global_base=workgroup_id.x*TILE_COLS;let subtile_id=matrixRole;let subtile_idx=subtile_id/{{ subgroupRowsValue }}u;let subtile_idy=subtile_id% {{ subgroupRowsValue }}u;let base_A=subtile_idy*SUB_ROWS;let base_B=subtile_idx*SUB_COLS;var matC00:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC01:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC02:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC03:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC10:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC11:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC12:{{ matrixType(matrixSoftware, "result", accScalar) }};var matC13:{{ matrixType(matrixSoftware, "result", accScalar) }};
{% if splitKPartial %}let k_begin=slice*K_CHUNK;let k_end=min(k_begin+K_CHUNK,{{ kLoopVar }});for(var kidx=k_begin;kidx<k_end;kidx=kidx+TILE_K){
{% else %}for(var kidx=0u;kidx<{{ kLoopVar }};kidx=kidx+TILE_K){
{% endif %}{% if not directMatrixInputs %}loadSHMA(a_global_base,kidx,local_idx/4u,local_idx%4u);loadSHMB(b_base,b_global_base,kidx,local_idx/{{ bKChunks }}u,local_idx% {{ bKChunks }}u);workgroupBarrier();
{% endif %}for(var step=0u;step<TILE_K;step=step+8u){
{% include "ops/_shared/subgroup-matrix-8mma-step.fac6af68e9.wgsl.jinja" %}    }
{% if not directMatrixInputs %}workgroupBarrier();
{% endif %}}
{% if splitKPartial %}let partial_offset=c_base+(a_global_base+base_A)*PARTIAL_COLS+b_global_base+base_B;
{% for r in range(2) %}{% for c in range(4) %}{{ matrixStore(matrixSoftware, "row_major", "&y", "partial_offset + " ~ (r * 8) ~ "u * PARTIAL_COLS + " ~ (c * 8) ~ "u", "matC" ~ r ~ c, "PARTIAL_COLS") | indent(2, true) }}
{% endfor %}{% endfor %}{% elif useDirectMatrixStore %}{% if padded %}if(b_global_base+TILE_COLS<=N){
{% endif %}let matrix_c_offset=c_base+(a_global_base+base_A)*N+b_global_base+base_B;
{% for r in range(2) %}{% for c in range(4) %}{{ matrixStore(matrixSoftware, "row_major", "&y", "matrix_c_offset + " ~ (r * 8) ~ "u * N + " ~ (c * 8) ~ "u", "matC" ~ r ~ c, "N") | indent(2, true) }}
{% endfor %}{% endfor %}storageBarrier();for(var tile_idx=local_idx;tile_idx<TILE_ROWS*TILE_COLS;tile_idx+={{ workgroupThreadsValue }}u){let local_row=tile_idx/TILE_COLS;let local_col=tile_idx%TILE_COLS;let global_row=a_global_base+local_row;let global_col=b_global_base+local_col;let y_index=c_base+global_row*N+global_col;var raw=f32(y[y_index]);raw+=f32(bias[global_row]);y[y_index]={{ T }}(raw);}
{% if padded %}return;}
{% endif %}{% endif %}{% if needsScratchStore %}{% for c in range(4) %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[subtile_id][" ~ c ~ "]", "0u", "matC0" ~ c, "8u") | indent(2, true) }}
{% endfor %}workgroupBarrier();let row=sg_id/4u;let col=(sg_id%4u)*2u;let row_base=a_global_base+base_A;var matrix_c_offset=c_base+row_base*N+b_global_base+base_B;var row_limit=i32(M)-i32(row_base);
  storeOutput(matrix_c_offset, row_base, row, col, subtile_id, row_limit{% if padded %}, b_global_base + base_B{% endif %});
workgroupBarrier();
{% for c in range(4) %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[subtile_id][" ~ c ~ "]", "0u", "matC1" ~ c, "8u") | indent(2, true) }}
{% endfor %}workgroupBarrier();matrix_c_offset=matrix_c_offset+8u*N;row_limit=i32(M)-i32(row_base+8u);
  storeOutput(matrix_c_offset, row_base + 8u, row, col, subtile_id, row_limit{% if padded %}, b_global_base + base_B{% endif %});
{% endif %}{% endmacro %}@compute@workgroup_size({{ workgroupThreadsValue }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)workgroup_id:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,
  @builtin(subgroup_size) matrixPhysicalWidth: u32{% if needsScratchStore %},
  @builtin(subgroup_invocation_id) matrixPhysicalLane: u32{% endif %}){let matrixRole=claimMatrixSubgroupRole(matrixPhysicalWidth,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,{{ workgroupThreadsValue }}u/32u);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){
{% if needsScratchStore %}let sg_id=matrixPhysicalLane;
{% endif %}{{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;
{% if needsScratchStore %}let sg_id=local_idx%32u;
{% endif %}{{ matrixKernelBody(true) }} } }`],["ops/_shared/conv-im2col-nchw.wgsl.jinja",`{% set colsScalar = colsScalar | default("f32") %}{% set colsVectorWidth = colsVectorWidth | default(1) %}{{ env.wgsl.resourceDeclarations }} const KERNEL_H:u32={{ kernelHSpec }}u;const KERNEL_W:u32={{ kernelWSpec }}u;const KSIZE:u32=KERNEL_H*KERNEL_W;const STRIDE_H:u32={{ strideHSpec }}u;const STRIDE_W:u32={{ strideWSpec }}u;const DILATION_H:u32={{ dilationHSpec }}u;const DILATION_W:u32={{ dilationWSpec }}u;const PAD_TOP:i32={{ padTopSpec }};const PAD_LEFT:i32={{ padLeftSpec }};
{% if padded %}const COLS_STRIDE:u32={{ nPadded }}u;const K_PADDED:u32={{ kPadded }}u;
{% endif %}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let k=gid.y;let batch=gid.z;let ic=k/KSIZE;let kq=k%KSIZE;let kh=kq/KERNEL_W;let kw=kq%KERNEL_W;let planeBase=(batch*params.inChannels+ic)*params.inH;
{% if colsVectorWidth == 4 %}let limit=params.outCount/4u;
{% elif padded %}let limit=COLS_STRIDE;
{% else %}let limit=params.outCount;
{% endif %}let step=nwg.x*{{ tunables.WORKGROUP_SIZE }}u;var pos=gid.x;loop{if(pos>=limit){break;}
{% if colsVectorWidth == 4 %}let scalarPos=pos*4u;
{% if strideWSpec == 1 %}let oh=scalarPos/params.outW;let ow=scalarPos%params.outW;let ih=i32(oh*STRIDE_H+kh*DILATION_H)-PAD_TOP;let iw=i32(ow*STRIDE_W+kw*DILATION_W)-PAD_LEFT;
{% endif %}var values=vec4<f16>(0.0h);
{% if strideWSpec == 1 %}if(ow+3u<params.outW&&ih>=0&&ih<i32(params.inH)&&iw>=0&&iw+3<i32(params.inW)){let offset=(planeBase+u32(ih))*params.inW+u32(iw);values=vec4<f16>(x[offset],x[offset+1u],x[offset+2u],x[offset+3u]);}else{
{% endif %}for(var component=0u;component<4u;component++){let componentPos=scalarPos+component;let componentH=i32((componentPos/params.outW)*STRIDE_H+kh*DILATION_H)-PAD_TOP;let componentW=i32((componentPos%params.outW)*STRIDE_W+kw*DILATION_W)-PAD_LEFT;if(componentH>=0&&componentH<i32(params.inH)&&componentW>=0&&componentW<i32(params.inW)){values[component]=x[(planeBase+u32(componentH))*params.inW+u32(componentW)];} }
{% if strideWSpec == 1 %}}
{% endif %}cols[(batch*params.kRows+k)*limit+pos]=values;
{% else %}var value=0.0;
{% if padded %}if(pos<params.outCount&&k<params.kRows){
{% endif %}let oh=pos/params.outW;let ow=pos%params.outW;let ih=i32(oh*STRIDE_H+kh*DILATION_H)-PAD_TOP;let iw=i32(ow*STRIDE_W+kw*DILATION_W)-PAD_LEFT;if(ih>=0&&ih<i32(params.inH)&&iw>=0&&iw<i32(params.inW)){value=f32(x[(planeBase+u32(ih))*params.inW+u32(iw)]);}
{% if padded %}}
{% endif %}{% set columnRows = "K_PADDED" if padded else "params.kRows" %}{% set columnStride = "COLS_STRIDE" if padded else "params.outCount" %}    cols[(batch * {{ columnRows }} + k) * {{ columnStride }} + pos] = {% if colsScalar == "f16" %}f16(value){% else %}value{% endif %};
{% endif %}pos=pos+step;} }`],["ops/_shared/conv-splitk-reduce.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const M:u32={{ M }}u;const N:u32={{ N }}u;const SPLIT_K:u32={{ splitK }}u;const PARTIAL_ROWS:u32={{ mPadded }}u;const PARTIAL_COLS:u32={{ nPadded }}u;const PARTIAL_SLICE_STRIDE:u32={{ batchCount }}u*PARTIAL_ROWS*PARTIAL_COLS;const COUNT:u32={{ batchCount }}u*M*N;const WORKGROUP_SIZE:u32={{ reduceWorkgroupSize }}u;@compute@workgroup_size({{ reduceWorkgroupSize }},1,1)fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WORKGROUP_SIZE", "idx", "COUNT") }} let col=idx%N;let row=(idx/N)%M;let batch=idx/(M*N);var src=(batch*PARTIAL_ROWS+row)*PARTIAL_COLS+col;var acc=0.0;for(var z=0u;z<SPLIT_K;z=z+1u){acc=acc+partial[src];src=src+PARTIAL_SLICE_STRIDE;}acc=acc+f32(bias[row]);y[idx]={{ T }}(acc);}`],["ops/_shared/conv-1x1-gemm-tiled-reg.wgsl.jinja",`{% set convKernelH = convKernelH | default(1) %}{% set convKernelW = convKernelW | default(1) %}{% set convStrideH = convStrideH | default(1) %}{% set convStrideW = convStrideW | default(1) %}{% set convDilationH = convDilationH | default(1) %}{% set convDilationW = convDilationW | default(1) %}{% set convPadTop = convPadTop | default(0) %}{% set convPadLeft = convPadLeft | default(0) %}{% set convInH = convInH | default(0) %}{% set convInW = convInW | default(0) %}{% set convOutW = convOutW | default(0) %}{% set convInChannels = convInChannels | default(0) %}{{ env.wgsl.resourceDeclarations }}
{% set tileT = gemmTileScalar if gemmTileScalar is defined else ("f16" if usesF16 else "f32") %}{% set GEMM_BK = gemmKTile if gemmKTile is defined else 16 %}{% set GEMM_TM = gemmThreadRows if gemmThreadRows is defined else 8 %}{% set GEMM_WG_X = gemmWorkgroupX if gemmWorkgroupX is defined else 16 %}{% set GEMM_BM = gemmMTile if gemmMTile is defined else 64 %}{% set GEMM_BN = gemmNTile if gemmNTile is defined else GEMM_WG_X * 4 %}{% set GEMM_TN = gemmThreadColumns if gemmThreadColumns is defined else (GEMM_BN / GEMM_WG_X)|int %}{% set GEMM_WG_Y = gemmWorkgroupY if gemmWorkgroupY is defined else (GEMM_BM / GEMM_TM)|int %}{% set components = ["x", "y", "z", "w"] %}const BK:u32={{ GEMM_BK }}u;const BM:u32={{ GEMM_BM }}u;const BN:u32={{ GEMM_BN }}u;const TM:u32={{ GEMM_TM }}u;const TN:u32={{ GEMM_TN }}u;const WG_X:u32={{ GEMM_WG_X }}u;const WG_SIZE:u32={{ GEMM_WG_X * GEMM_WG_Y }}u;const AK_VECS:u32=BK/4u;const BN_VECS:u32=BN/4u;
{% set implicitIm2col = implicitIm2col is defined and implicitIm2col %}{% set splitKValue = splitK if splitK is defined else 1 %}{% set splitKPartial = splitKValue > 1 %}{% set kTiles = kTiles | default(0) %}{% set mPadded = mPadded | default(0) %}{% set nPadded = nPadded | default(0) %}{% set batchCount = batchCount | default(0) %}{% set gemmKLimit = "kEnd" if splitKPartial else "K" %}{% set implicitGatherMode = 0 if coalescedImplicitGather | default(false) else (tunables.IMPLICIT_TILED_GATHER_MODE if tunables.IMPLICIT_TILED_GATHER_MODE is defined else 0) %}{% set implicitGatherWgSize = GEMM_WG_X * GEMM_WG_Y %}{% set implicitGatherColumnVecs = (GEMM_BN / 4)|int %}{% set implicitGatherKChunks = (implicitGatherWgSize / implicitGatherColumnVecs)|int %}{% set implicitGatherDivides = implicitGatherWgSize % implicitGatherColumnVecs == 0 and implicitGatherKChunks > 0 and GEMM_BK % implicitGatherKChunks == 0 %}{% set useCarriedImplicitGather = implicitIm2col and implicitGatherMode == 1 and implicitGatherDivides %}{% if useCarriedImplicitGather %}{% set implicitGatherLoadWidth = (GEMM_BK / implicitGatherKChunks)|int %}{% endif %}{% if implicitIm2col %}const CONV_KERNEL_H:u32={{ convKernelH }}u;const CONV_KERNEL_W:u32={{ convKernelW }}u;const CONV_KSIZE:u32=CONV_KERNEL_H*CONV_KERNEL_W;const CONV_STRIDE_H:u32={{ convStrideH }}u;const CONV_STRIDE_W:u32={{ convStrideW }}u;const CONV_DILATION_H:u32={{ convDilationH }}u;const CONV_DILATION_W:u32={{ convDilationW }}u;const CONV_PAD_TOP:i32={{ convPadTop }};const CONV_PAD_LEFT:i32={{ convPadLeft }};const CONV_IN_H:u32={{ convInH }}u;const CONV_IN_W:u32={{ convInW }}u;const CONV_OUT_W:u32={{ convOutW }}u;const CONV_IN_CHANNELS:u32={{ convInChannels }}u;
{% endif %}{% if splitKPartial %}const SPLIT_K:u32={{ splitKValue }}u;const K_TILES:u32={{ kTiles }}u;const PARTIAL_ROWS:u32={{ mPadded }}u;const PARTIAL_COLS:u32={{ nPadded }}u;const PARTIAL_BATCH_STRIDE:u32=PARTIAL_ROWS*PARTIAL_COLS;const PARTIAL_SLICE_STRIDE:u32={{ batchCount }}u*PARTIAL_BATCH_STRIDE;
{% endif %}{% set emitConvStoreOut = not splitKPartial and not (packedOutput | default(false)) %}{% include "ops/_shared/conv-gemm-store-out.6930a83d54.wgsl.jinja" %}var<workgroup>tileA:array<array<vec4<{{ tileT }}>,AK_VECS>,BM>;var<workgroup>tileB:array<array<vec4<{{ tileT }}>,BN_VECS>,BK>;
{% macro load_a_vec4(rowBase, columnBase, rowLimit, columnLimit) %}for(var linear=li;linear<BM*AK_VECS;linear+=WG_SIZE){let ar=linear/AK_VECS;let ac4=linear%AK_VECS;let am={{ rowBase }}+ar;let ak={{ columnBase }}+ac4*4u;var av=vec4<{{ tileT }}>({{ tileT }}(0.0));if(am<{{ rowLimit }}){let aBase=am*K+ak;if(ak+3u<{{ columnLimit }}){av=vec4<{{ tileT }}>({{ tileT }}(w[aBase]),{{ tileT }}(w[aBase+1u]),{{ tileT }}(w[aBase+2u]),{{ tileT }}(w[aBase+3u]));}else{for(var t=0u;t<4u;t=t+1u){if(ak+t<{{ columnLimit }}){av[t]={{ tileT }}(w[aBase+t]);} } } }tileA[ar][ac4]=av;
    }{% endmacro %}{% macro load_b_vec4(rowBase, columnBase, rowLimit, columnLimit) %}for(var linear=li;linear<BK*BN_VECS;linear+=WG_SIZE){let br=linear/BN_VECS;let bc4=linear%BN_VECS;let bk={{ rowBase }}+br;let bn={{ columnBase }}+bc4*4u;var bvec=vec4<{{ tileT }}>({{ tileT }}(0.0));if(bk<{{ rowLimit }}){let bBase=xBatchBase+bk*N+bn;if(bn+3u<{{ columnLimit }}){bvec=vec4<{{ tileT }}>({{ tileT }}(xm[bBase]),{{ tileT }}(xm[bBase+1u]),{{ tileT }}(xm[bBase+2u]),{{ tileT }}(xm[bBase+3u]));}else{for(var t=0u;t<4u;t=t+1u){if(bn+t<{{ columnLimit }}){bvec[t]={{ tileT }}(xm[bBase+t]);} } } }tileB[br][bc4]=bvec;
    }{% endmacro %}{% macro load_b_implicit_vec4(rowBase, columnBase, rowLimit, columnLimit) %}for(var linear=li;linear<BK*BN_VECS;linear+=WG_SIZE){let br=linear/BN_VECS;let bc4=linear%BN_VECS;let bk={{ rowBase }}+br;var bvec=vec4<{{ tileT }}>({{ tileT }}(0.0));if(bk<{{ rowLimit }}){let ic=bk/CONV_KSIZE;let kq=bk%CONV_KSIZE;let kh=kq/CONV_KERNEL_W;let kw=kq%CONV_KERNEL_W;let planeBase=(batch*CONV_IN_CHANNELS+ic)*CONV_IN_H;for(var t=0u;t<4u;t=t+1u){let bn={{ columnBase }}+bc4*4u+t;if(bn<{{ columnLimit }}){let oh=bn/CONV_OUT_W;let ow=bn%CONV_OUT_W;let ih=i32(oh*CONV_STRIDE_H+kh*CONV_DILATION_H)-CONV_PAD_TOP;let iw=i32(ow*CONV_STRIDE_W+kw*CONV_DILATION_W)-CONV_PAD_LEFT;if(ih>=0&&ih<i32(CONV_IN_H)&&iw>=0&&iw<i32(CONV_IN_W)){bvec[t]={{ tileT }}(xm[(planeBase+u32(ih))*CONV_IN_W+u32(iw)]);} } } }tileB[br][bc4]=bvec;
    }{% endmacro %}{% macro load_b_implicit_carried(rowBase, columnBase, rowLimit, columnLimit) %}let bColVec=li/{{ implicitGatherKChunks }}u;let bChunk=li% {{ implicitGatherKChunks }}u;let bLocalK=bChunk*{{ implicitGatherLoadWidth }}u;let bStartK={{ rowBase }}+bLocalK;let bCol0={{ columnBase }}+bColVec*4u;let bCols=vec4<u32>(bCol0,bCol0+1u,bCol0+2u,bCol0+3u);let inColumn=bCols<vec4<u32>({{ columnLimit }});let ih0=vec4<i32>((bCols/vec4<u32>(CONV_OUT_W))*CONV_STRIDE_H)-vec4<i32>(CONV_PAD_TOP);let iw0=vec4<i32>((bCols%vec4<u32>(CONV_OUT_W))*CONV_STRIDE_W)-vec4<i32>(CONV_PAD_LEFT);var carriedK=bStartK;var carriedIc=carriedK/CONV_KSIZE;let carriedKq=carriedK%CONV_KSIZE;var carriedKh=carriedKq/CONV_KERNEL_W;var carriedKw=carriedKq%CONV_KERNEL_W;var carriedIh=ih0+vec4<i32>(i32(carriedKh*CONV_DILATION_H));var carriedIw=iw0+vec4<i32>(i32(carriedKw*CONV_DILATION_W));let interior=all(inColumn)&&bStartK+{{ implicitGatherLoadWidth }}u<={{ rowLimit }}&&all(ih0>=vec4<i32>(0))&&all(ih0+vec4<i32>(i32((CONV_KERNEL_H-1u)*CONV_DILATION_H))<vec4<i32>(i32(CONV_IN_H)))&&all(iw0>=vec4<i32>(0))&&all(iw0+vec4<i32>(i32((CONV_KERNEL_W-1u)*CONV_DILATION_W))<vec4<i32>(i32(CONV_IN_W)));if(interior){let batchBase=batch*CONV_IN_CHANNELS*CONV_IN_H*CONV_IN_W;let colStep=i32(CONV_DILATION_W);let rowStep=i32(CONV_DILATION_H*CONV_IN_W);let planeStep=i32(CONV_IN_H*CONV_IN_W);let kwWrap=colStep*i32(CONV_KERNEL_W)-rowStep;let khWrap=rowStep*i32(CONV_KERNEL_H)-planeStep;var carriedAddress=vec4<i32>(i32(carriedIc)*planeStep)+carriedIh*vec4<i32>(i32(CONV_IN_W))+carriedIw;for(var i=0u;i<{{ implicitGatherLoadWidth }}u;i=i+1u){tileB[bLocalK+i][bColVec]=vec4<{{ tileT }}>(
{% for component in components %}          {{ tileT }}(xm[batchBase + u32(carriedAddress.{{ component }})]){% if not loop.last %},{% endif %}{% endfor %});carriedAddress=carriedAddress+vec4<i32>(colStep);carriedKw=carriedKw+1u;if(carriedKw==CONV_KERNEL_W){carriedKw=0u;carriedAddress=carriedAddress-vec4<i32>(kwWrap);carriedKh=carriedKh+1u;if(carriedKh==CONV_KERNEL_H){carriedKh=0u;carriedAddress=carriedAddress-vec4<i32>(khWrap);} } } }else{for(var i=0u;i<{{ implicitGatherLoadWidth }}u;i=i+1u){var carriedValue=vec4<{{ tileT }}>({{ tileT }}(0.0));let carriedRow=(batch*CONV_IN_CHANNELS+carriedIc)*CONV_IN_H;let carriedOk=inColumn&(carriedIh>=vec4<i32>(0))&(carriedIh<vec4<i32>(i32(CONV_IN_H)))&(carriedIw>=vec4<i32>(0))&(carriedIw<vec4<i32>(i32(CONV_IN_W)));if(carriedK<{{ rowLimit }}){
{% for component in components %}if(carriedOk.{{ component }}){carriedValue.{{ component }}={{ tileT }}(xm[(carriedRow+u32(carriedIh.{{ component }}))*CONV_IN_W+u32(carriedIw.{{ component }})]);}
{% endfor %}}tileB[bLocalK+i][bColVec]=carriedValue;carriedK=carriedK+1u;carriedKw=carriedKw+1u;carriedIw=carriedIw+vec4<i32>(i32(CONV_DILATION_W));if(carriedKw==CONV_KERNEL_W){carriedKw=0u;carriedIw=iw0;carriedKh=carriedKh+1u;carriedIh=carriedIh+vec4<i32>(i32(CONV_DILATION_H));if(carriedKh==CONV_KERNEL_H){carriedKh=0u;carriedIh=ih0;carriedIc=carriedIc+1u;} } }
    }{% endmacro %}@compute@workgroup_size({{ GEMM_WG_X }},{{ GEMM_WG_Y }},1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let M=params.M;let K=params.K;let N=params.N;let mBase=wg.y*BM;let nBase=wg.x*BN;
{% if splitKPartial %}let slice=wg.z%SPLIT_K;let batch=wg.z/SPLIT_K;
{% else %}let batch=wg.z;
{% endif %}{% if not implicitIm2col %}let xBatchBase=batch*K*N;
{% endif %}let li=lid.y*WG_X+lid.x;
{% for row in range(GEMM_TM) %}var acc{{ row }}=vec{{ GEMM_TN }}<f32>(0.0);
{% endfor %}{% if splitKPartial %}let kBegin=min((slice*K_TILES/SPLIT_K)*BK,K);let kEnd=min(((slice+1u)*K_TILES/SPLIT_K)*BK,K);let kCount=kEnd-kBegin;let numTiles=(kCount+BK-1u)/BK;
{% else %}let numTiles=(K+BK-1u)/BK;
{% endif %}for(var kt:u32=0u;kt<numTiles;kt=kt+1u){
{% if splitKPartial %}let kBase=kBegin+kt*BK;
{% else %}let kBase=kt*BK;
{% endif %}{{ load_a_vec4("mBase", "kBase", "M", gemmKLimit) }}
{% if useCarriedImplicitGather %}{{ load_b_implicit_carried("kBase", "nBase", gemmKLimit, "N") }}
{% elif implicitIm2col %}{{ load_b_implicit_vec4("kBase", "nBase", gemmKLimit, "N") }}
{% else %}{{ load_b_vec4("kBase", "nBase", gemmKLimit, "N") }}
{% endif %}workgroupBarrier();let aRow=lid.y*TM;for(var kv:u32=0u;kv<AK_VECS;kv=kv+1u){
{% for row in range(GEMM_TM) %}let av{{ row }}=vec4<f32>(tileA[aRow+{{ row }}u][kv]);
{% endfor %}{% for sub in range(4) %}let bv{{ sub }}=vec{{ GEMM_TN }}<f32>(tileB[kv*4u+{{ sub }}u][lid.x]);
{% for row in range(GEMM_TM) %}acc{{ row }}+=vec{{ GEMM_TN }}<f32>(av{{ row }}.{{ components[sub] }})*bv{{ sub }};
{% endfor %}{% endfor %}}workgroupBarrier();}
{% if splitKPartial %}let m0=mBase+lid.y*TM;let n0=nBase+lid.x*TN;let partialBase=slice*PARTIAL_SLICE_STRIDE+batch*PARTIAL_BATCH_STRIDE;
{% else %}let yBatchBase=batch*M*N;let m0=mBase+lid.y*TM;let n0=nBase+lid.x*TN;
{% endif %}{% for row in range(GEMM_TM) %}{% for column in range(GEMM_TN) %}{% if splitKPartial %}y[partialBase+(m0+{{ row }}u)*PARTIAL_COLS+n0+{{ column }}u]=acc{{ row }}.{{ components[column] }};
{% else %}store_out(m0+{{ row }}u,n0+{{ column }}u,yBatchBase,acc{{ row }}.{{ components[column] }});
{% endif %}{% endfor %}{% endfor %}}`],["ops/_shared/conv-1x1-gemm-tiled.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% set accumulatorPrefix = "acc" %}{% set tileT = "f32" %}{% set wgX = 16 %}{% set wgY = 16 %}const BK:u32=16u;const BM:u32=32u;const BN:u32=32u;
{% include "ops/_shared/conv-gemm-store-out.b342b63455.wgsl.jinja" %}var<workgroup>tileA:array<array<{{ tileT }},BK>,BM>;var<workgroup>tileB:array<array<{{ tileT }},BN>,BK>;@compute@workgroup_size({{ wgX }},{{ wgY }},1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let M=params.M;let K=params.K;let N=params.N;let mBase=wg.y*BM;let nBase=wg.x*BN;let batch=wg.z;let xBatchBase=batch*K*N;let li=lid.y*{{ wgX }}u+lid.x;var acc00:f32=0.0;var acc01:f32=0.0;var acc10:f32=0.0;var acc11:f32=0.0;let numTiles=(K+BK-1u)/BK;for(var kt:u32=0u;kt<numTiles;kt=kt+1u){let kBase=kt*BK;for(var e:u32=0u;e<(BM*BK)/{{ wgX * wgY }}u;e=e+1u){let idx=li+e*{{ wgX * wgY }}u;let ar=idx/BK;let ac=idx%BK;let am=mBase+ar;let ak=kBase+ac;if(am<M&&ak<K){tileA[ar][ac]={{ tileT }}(w[am*K+ak]);}else{tileA[ar][ac]={{ tileT }}(0.0);}let br=idx/BN;let bc=idx%BN;let bk=kBase+br;let bn=nBase+bc;if(bk<K&&bn<N){tileB[br][bc]={{ tileT }}(xm[xBatchBase+bk*N+bn]);}else{tileB[br][bc]={{ tileT }}(0.0);} }workgroupBarrier();for(var kk:u32=0u;kk<BK;kk=kk+1u){let a0=f32(tileA[lid.y*2u][kk]);let a1=f32(tileA[lid.y*2u+1u][kk]);let b0=f32(tileB[kk][lid.x*2u]);let b1=f32(tileB[kk][lid.x*2u+1u]);
{% for i in range(2) %}{% for j in range(2) %}{{ accumulatorPrefix }}{{ i }}{{ j }}={{ accumulatorPrefix }}{{ i }}{{ j }}+a{{ i }}*b{{ j }};
{% endfor %}{% endfor %}}workgroupBarrier();}let m0=mBase+lid.y*2u;let m1=m0+1u;let n0=nBase+lid.x*2u;let n1=n0+1u;let yBatchBase=batch*M*N;store_out(m0,n0,yBatchBase,acc00);store_out(m0,n1,yBatchBase,acc01);store_out(m1,n0,yBatchBase,acc10);store_out(m1,n1,yBatchBase,acc11);}`],["ops/_shared/conv-direct-unrolled.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const KERNEL_AREA:u32={{ kernelHSpec * kernelWSpec }}u;const STRIDE_H:u32={{ strideHSpec }}u;const STRIDE_W:u32={{ strideWSpec }}u;const PAD_TOP:i32={{ padTopSpec }};const PAD_LEFT:i32={{ padLeftSpec }};@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "index") }} let ow=index%params.outW;var t=index/params.outW;let oh=t%params.outH;t=t/params.outH;let oc=t%params.outChannels;let batch=t/params.outChannels;let group=oc/params.outChannelsPerGroup;let ihBase=i32(oh*STRIDE_H)-PAD_TOP;let iwBase=i32(ow*STRIDE_W)-PAD_LEFT;let xChannelStride=params.inH*params.inW;var xBase=(batch*params.inChannels+group*params.inChannelsPerGroup)*xChannelStride;var wBase=oc*params.weightInChannels*KERNEL_AREA;var acc=0.0;acc=f32(bias[oc]);for(var ic=0u;ic<params.weightInChannels;ic=ic+1u){
{% for kh in range(kernelHSpec) %}{let ih=ihBase+{{ kh * dilationHSpec }}i;if(ih>=0&&ih<i32(params.inH)){let xRow=xBase+u32(ih)*params.inW;
{% for kw in range(kernelWSpec) %}{let iw=iwBase+{{ kw * dilationWSpec }}i;if(iw>=0&&iw<i32(params.inW)){acc=acc+f32(x[xRow+u32(iw)])*f32(w[wBase+{{ kh * kernelWSpec + kw }}u]);} }
{% endfor %}} }
{% endfor %}xBase=xBase+xChannelStride;wBase=wBase+KERNEL_AREA;}y[index]={{ T }}(acc);}`]],assumes:[["ops/_shared/conv2d-grouped-large-w4.wgsl.jinja",{hasBias:!0,scalar:"f32",vectorScalar:"vec4<f32>",spanCap:64,dilationHSpec:1,dilationWSpec:1}],["ops/_shared/conv-1x1-subgroup-matrix.wgsl.jinja",{T:"f32",hasBias:!0,fScalar:"f32"}],["ops/_shared/conv-im2col-nchw.wgsl.jinja",{dilationHSpec:1,dilationWSpec:1}],["ops/_shared/conv-splitk-reduce.wgsl.jinja",{T:"f32",hasBias:!0}],["ops/_shared/conv-1x1-gemm-tiled-reg.wgsl.jinja",{T:"f32",usesF16:!1}],["ops/_shared/conv-1x1-gemm-tiled.wgsl.jinja",{T:"f32",hasBias:!0,usesF16:!1}],["ops/_shared/conv-direct-unrolled.wgsl.jinja",{T:"f32",hasBias:!0,dilationHSpec:1,dilationWSpec:1}]]}],["ai.onnx.Cos",{manifest:{domain:"ai.onnx",name:"Cos",sinceVersion:7,inputs:{x:{onnx:"input",dtype:"T"}},outputs:{y:{onnx:"output",dtype:"T",rank:"ranks.x",shape:"shapes.x"}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_layout_vec4",priority:20,when:["numel(shapes.x) > 0","numel(shapes.x) % 4 == 0","numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Cos.Vec4",shader:"ops/_shared/unary-vec4.wgsl.jinja",derive:{op:'"cos"',vec4PerThread:"4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1"},bindings:["x","y","params_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y) / 4, 4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"elementwise",when:["numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Cos",shader:"ops/_shared/unary-scalar.wgsl.jinja",derive:{op:'"cos"',itemsPerInvocation:4},bindings:["x_scalar","y_scalar","params_2_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{x:{buffer:"read-only-storage",elementType:"$vectorScalar"},y:{buffer:"storage",elementType:"$vectorScalar"},params_unary:{struct:[{name:"count",type:"u32",value:"numel(shapes.y) / 4"}],name:"params"},x_scalar:{buffer:"read-only-storage",name:"x",elementType:"$scalar"},y_scalar:{buffer:"storage",name:"y",elementType:"$scalar"},params_2_unary:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}}},assets:[["ops/_shared/unary-vec4.wgsl.jinja",`{% macro unary_vec4_map(fn, value="fv") -%}vec4<f32>({{ fn }}({{ value }}.x),{{ fn }}({{ value }}.y),{{ fn }}({{ value }}.z),{{ fn }}({{ value }}.w))
{%- endmacro %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.5eb0cb613d.wgsl.jinja" %}{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}  let xv = x[i];
let fv=vec4<f32>(xv);y[i]={{ vectorScalar }}({{ unary_vec4_map("cos_accurate") }});
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/unary-scalar.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.5eb0cb613d.wgsl.jinja" %}{{ flat_tail_open() }}
y[i]={{ scalar }}(cos_accurate(f32(x[i])));{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/unary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>"}],["ops/_shared/unary-scalar.wgsl.jinja",{scalar:"f32"}]]}],["ai.onnx.DFT",{manifest:{domain:"ai.onnx",name:"DFT",sinceVersion:20,inputs:{input:{dtype:"T"},dft_length:{dtype:"L",rank:0,optional:!0},axis:{dtype:"I",rank:0,optional:!0,storage:"int32",narrowing:"checked"}},outputs:{output:{dtype:"T",rank:"ranks.input"}},attributes:{inverse:{default:0},onesided:{default:0}},attributeConstraints:{inverse:{values:[0,1]},onesided:{values:[0,1]}},typeConstraints:{T:["float32","float16"],L:["int32"],I:["int64"]},when:["f16Ok(dtypes.T)","ranks.input >= 2","dim(shapes.input, ranks.input - 1) == 1 or dim(shapes.input, ranks.input - 1) == 2","attrs.onesided == 0 or dim(shapes.input, ranks.input - 1) == (2 if attrs.inverse != 0 else 1)","dim(shapes.output, ranks.output - 1) == (1 if attrs.inverse != 0 and attrs.onesided != 0 else 2)","numel(shapes.output) == 0 or present.axis or present.dft_length or dim(shapes.input, ranks.input - 2) > (1 if attrs.inverse != 0 and attrs.onesided != 0 else 0)","present.axis or present.dft_length or attrs.inverse == 0 or attrs.onesided == 0 or dim(shapes.output, ranks.output - 2) == 2 * (dim(shapes.input, ranks.input - 2) - 1)"],tunables:{WORKGROUP_SIZE:{default:128},TILED_REAL_WORKGROUP_SIZE:{default:256},TILED_REAL_MIN_LENGTH:{default:512},RUNTIME_AXIS_FFT_MIN_LENGTH:{default:64}},derive:{variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',ioScalar:"dtypes.T",dftAxisAttr:"ranks.input - 2",dftAttrOuter:"outer(shapes.input, dftAxisAttr)",dftAttrInputInner:"inner(shapes.input, dftAxisAttr) / dim(shapes.input, ranks.input - 1)",dftAttrOutputOuter:"outer(shapes.output, dftAxisAttr)",dftAttrOutputInner:"inner(shapes.output, dftAxisAttr) / dim(shapes.output, ranks.output - 1)",outputRows:"numel(shapes.output) / dim(shapes.output, ranks.output - 1)",rank3BatchDispatchFits:"dim(shapes.input, 0) <= device.limits.maxComputeWorkgroupsPerDimension",rank3Axis1SharedFftFits:"rank3BatchDispatchFits and pow2ceil(dim(shapes.input, 1)) == dim(shapes.input, 1) and dim(shapes.input, 1) >= 4 and dim(shapes.input, 1) * 16 <= device.limits.maxComputeWorkgroupStorageSize",rank3Axis1StorageFftFits:"rank3BatchDispatchFits and pow2ceil(dim(shapes.input, 1)) == dim(shapes.input, 1) and dim(shapes.input, 1) >= 4 and dim(shapes.input, 1) * 16 > device.limits.maxComputeWorkgroupStorageSize",rank3Axis1FftLog2:"log2ceil(dim(shapes.input, 1))",fftAnyAxisLen0:"dim(shapes.input, 0) if ranks.input - 1 > 0 else 0",fftAnyAxisLen1:"dim(shapes.input, 1) if ranks.input - 1 > 1 else 0",fftAnyAxisLen2:"dim(shapes.input, 2) if ranks.input - 1 > 2 else 0",fftAnyAxisLen3:"dim(shapes.input, 3) if ranks.input - 1 > 3 else 0",fftAnyAxisLen4:"dim(shapes.input, 4) if ranks.input - 1 > 4 else 0",fftAnyAxisLen5:"dim(shapes.input, 5) if ranks.input - 1 > 5 else 0",fftAnyAxisLen6:"dim(shapes.input, 6) if ranks.input - 1 > 6 else 0",fftAnyAxisCand0:"fftAnyAxisLen0 if (fftAnyAxisLen0 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen0) == fftAnyAxisLen0 and fftAnyAxisLen0 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand1:"fftAnyAxisLen1 if (fftAnyAxisLen1 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen1) == fftAnyAxisLen1 and fftAnyAxisLen1 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand2:"fftAnyAxisLen2 if (fftAnyAxisLen2 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen2) == fftAnyAxisLen2 and fftAnyAxisLen2 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand3:"fftAnyAxisLen3 if (fftAnyAxisLen3 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen3) == fftAnyAxisLen3 and fftAnyAxisLen3 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand4:"fftAnyAxisLen4 if (fftAnyAxisLen4 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen4) == fftAnyAxisLen4 and fftAnyAxisLen4 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand5:"fftAnyAxisLen5 if (fftAnyAxisLen5 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen5) == fftAnyAxisLen5 and fftAnyAxisLen5 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisCand6:"fftAnyAxisLen6 if (fftAnyAxisLen6 >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH and pow2ceil(fftAnyAxisLen6) == fftAnyAxisLen6 and fftAnyAxisLen6 * 16 <= device.limits.maxComputeWorkgroupStorageSize) else 0",fftAnyAxisMaxLen:"max(fftAnyAxisCand0, max(fftAnyAxisCand1, max(fftAnyAxisCand2, max(fftAnyAxisCand3, max(fftAnyAxisCand4, max(fftAnyAxisCand5, fftAnyAxisCand6))))))",fftAnyAxisMinLen:"min((fftAnyAxisCand0 if fftAnyAxisCand0 > 0 else 1073741824), min((fftAnyAxisCand1 if fftAnyAxisCand1 > 0 else 1073741824), min((fftAnyAxisCand2 if fftAnyAxisCand2 > 0 else 1073741824), min((fftAnyAxisCand3 if fftAnyAxisCand3 > 0 else 1073741824), min((fftAnyAxisCand4 if fftAnyAxisCand4 > 0 else 1073741824), min((fftAnyAxisCand5 if fftAnyAxisCand5 > 0 else 1073741824), (fftAnyAxisCand6 if fftAnyAxisCand6 > 0 else 1073741824)))))))",fftAnyAxisLogicalCount:"numel(shapes.input) / dim(shapes.input, ranks.input - 1) if ranks.input >= 1 and dim(shapes.input, ranks.input - 1) > 0 else 0",fftAnyAxisTransforms:"fftAnyAxisLogicalCount / fftAnyAxisMinLen if fftAnyAxisMaxLen >= tunables.RUNTIME_AXIS_FFT_MIN_LENGTH else 0",mixedFftRadix2:"pick([[dim(shapes.input, 1) % 256 == 0, 256], [dim(shapes.input, 1) % 128 == 0, 128], [dim(shapes.input, 1) % 64 == 0, 64], [dim(shapes.input, 1) % 32 == 0, 32], [dim(shapes.input, 1) % 16 == 0, 16], [dim(shapes.input, 1) % 8 == 0, 8], [dim(shapes.input, 1) % 4 == 0, 4]], 1)",mixedFftWorkgroupSize:"min(2 * tunables.WORKGROUP_SIZE, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",mixedFftTail:"dim(shapes.input, 1) / mixedFftRadix2",mixedFftTailAfter3:"mixedFftTail / pick([[mixedFftTail % 81 == 0, 81], [mixedFftTail % 27 == 0, 27], [mixedFftTail % 9 == 0, 9], [mixedFftTail % 3 == 0, 3]], 1)",mixedFftTailAfter5:"mixedFftTailAfter3 / pick([[mixedFftTailAfter3 % 125 == 0, 125], [mixedFftTailAfter3 % 25 == 0, 25], [mixedFftTailAfter3 % 5 == 0, 5]], 1)",mixedFftTailAfter7:"mixedFftTailAfter5 / pick([[mixedFftTailAfter5 % 49 == 0, 49], [mixedFftTailAfter5 % 7 == 0, 7]], 1)",mixedFftTailAfter11:"mixedFftTailAfter7 / pick([[mixedFftTailAfter7 % 121 == 0, 121], [mixedFftTailAfter7 % 11 == 0, 11]], 1)",mixedFftSmoothTail:"pow2ceil(mixedFftTailAfter11) == mixedFftTailAfter11",realResidualShapeFits:"not present.axis and not present.dft_length and ranks.input == 3 and ranks.output == 3 and attrs.inverse == 0 and dim(shapes.input, 0) == dim(shapes.output, 0) and dim(shapes.input, 0) > 0 and rank3BatchDispatchFits and dim(shapes.input, 2) == 1 and dim(shapes.output, 2) == 2 and pow2ceil(dim(shapes.input, 1)) == dim(shapes.input, 1) and dim(shapes.input, 1) >= 4 and ((attrs.onesided == 0 and dim(shapes.output, 1) == dim(shapes.input, 1)) or (attrs.onesided != 0 and dim(shapes.output, 1) == floor(dim(shapes.input, 1) / 2) + 1)) and numel(shapes.input) * 4 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) and numel(shapes.output) * 4 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize)",realResidualTwiddlesFit:"mixedFftWorkgroupSize > 0 and (floor(dim(shapes.input, 1) / 2) + 1) * 16 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) and ceilDiv(floor(dim(shapes.input, 1) / 2) + 1, mixedFftWorkgroupSize) <= device.limits.maxComputeWorkgroupsPerDimension",realStoragePacked:"realResidualShapeFits and realResidualTwiddlesFit and dim(shapes.input, 1) * 8 <= device.limits.maxComputeWorkgroupStorageSize and dim(shapes.input, 1) * 16 > device.limits.maxComputeWorkgroupStorageSize",realStorageResidual:"realResidualShapeFits and realResidualTwiddlesFit and dim(shapes.input, 1) * 8 > device.limits.maxComputeWorkgroupStorageSize and dim(shapes.input, 0) * dim(shapes.input, 1) * 4 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize)",residualSpectrumLength:"dim(shapes.input, 1) / (2 if dim(shapes.input, 2) == 1 else 1)",residualTileCapacity:"floor(device.limits.maxComputeWorkgroupStorageSize / 16)",residualTileFullLength:"min(residualSpectrumLength, pow2ceil(residualTileCapacity) / (1 if pow2ceil(residualTileCapacity) == residualTileCapacity else 2))",residualTileHalfCapacity:"floor(device.limits.maxComputeWorkgroupStorageSize / 32)",residualTileHalfLength:"min(residualSpectrumLength, pow2ceil(residualTileHalfCapacity) / (1 if pow2ceil(residualTileHalfCapacity) == residualTileHalfCapacity else 2))",residualMemoryTileLength:"residualTileHalfLength if residualTileHalfLength >= 2 and residualSpectrumLength / residualTileHalfLength <= 2 * log2ceil(residualSpectrumLength) else residualTileFullLength",residualComplexParallelTileLength:"max(2, pow2ceil(residualSpectrumLength * max(1, dim(shapes.input, 0) / 2) / max(1, 2 * log2ceil(residualSpectrumLength))))",residualTileLength:"min(residualComplexParallelTileLength, residualMemoryTileLength) if dim(shapes.input, 2) == 2 else residualMemoryTileLength",residualTileCount:"residualSpectrumLength / max(1, residualTileLength)",residualTileDispatchFits:"dim(shapes.input, 0) * residualTileCount <= device.limits.maxComputeWorkgroupsPerDimension * device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(dim(shapes.input, 0) * dim(shapes.output, 1), mixedFftWorkgroupSize) <= device.limits.maxComputeWorkgroupsPerDimension * device.limits.maxComputeWorkgroupsPerDimension",complexStorageShapeFits:"not present.axis and not present.dft_length and ranks.input == 3 and ranks.output == 3 and attrs.inverse == 0 and attrs.onesided == 0 and dim(shapes.input, 0) == dim(shapes.output, 0) and dim(shapes.input, 0) > 0 and rank3Axis1StorageFftFits and dim(shapes.input, 2) == 2 and dim(shapes.output, 2) == 2 and dim(shapes.output, 1) == dim(shapes.input, 1)",complexStorageResidual:"complexStorageShapeFits and realResidualTwiddlesFit and numel(shapes.input) * 4 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) and numel(shapes.output) * 4 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) and dim(shapes.input, 0) * dim(shapes.input, 1) * 8 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize)",compensatedDft:"(not present.axis and not present.dft_length and ranks.input >= 2 and ranks.input <= 3 and dim(shapes.input, ranks.input - 1) == 1 and attrs.inverse == 0 and dim(shapes.input, ranks.input - 2) >= tunables.TILED_REAL_MIN_LENGTH and dim(shapes.input, ranks.input - 2) * 4 <= device.limits.maxComputeWorkgroupStorageSize and outer(shapes.input, ranks.input - 2) <= device.limits.maxComputeWorkgroupsPerDimension) or realStorageResidual or complexStorageShapeFits",axisRootTableFits:"mixedFftWorkgroupSize > 0 and (floor(dim(shapes.input, dftAxisAttr) / 2) + 1) * 16 <= min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) and ceilDiv(floor(dim(shapes.input, dftAxisAttr) / 2) + 1, mixedFftWorkgroupSize) <= device.limits.maxComputeWorkgroupsPerDimension",packedMixedLength:"dim(shapes.input, 1) / 2",packedMixedRadix2:"pick([[packedMixedLength % 256 == 0, 256], [packedMixedLength % 128 == 0, 128], [packedMixedLength % 64 == 0, 64], [packedMixedLength % 32 == 0, 32], [packedMixedLength % 16 == 0, 16], [packedMixedLength % 8 == 0, 8], [packedMixedLength % 4 == 0, 4], [packedMixedLength % 2 == 0, 2]], 1)",packedMixedRest0:"packedMixedLength / packedMixedRadix2",packedMixedRest2:"packedMixedRest0 / pick([[packedMixedRest0 % 256 == 0, 256], [packedMixedRest0 % 128 == 0, 128], [packedMixedRest0 % 64 == 0, 64], [packedMixedRest0 % 32 == 0, 32], [packedMixedRest0 % 16 == 0, 16], [packedMixedRest0 % 8 == 0, 8], [packedMixedRest0 % 4 == 0, 4], [packedMixedRest0 % 2 == 0, 2]], 1)",packedMixedRest3:"packedMixedRest2 / pick([[packedMixedRest2 % 6561 == 0, 6561], [packedMixedRest2 % 2187 == 0, 2187], [packedMixedRest2 % 729 == 0, 729], [packedMixedRest2 % 243 == 0, 243], [packedMixedRest2 % 81 == 0, 81], [packedMixedRest2 % 27 == 0, 27], [packedMixedRest2 % 9 == 0, 9], [packedMixedRest2 % 3 == 0, 3]], 1)",packedMixedRest5:"packedMixedRest3 / pick([[packedMixedRest3 % 625 == 0, 625], [packedMixedRest3 % 125 == 0, 125], [packedMixedRest3 % 25 == 0, 25], [packedMixedRest3 % 5 == 0, 5]], 1)",packedMixedRest7:"packedMixedRest5 / pick([[packedMixedRest5 % 2401 == 0, 2401], [packedMixedRest5 % 343 == 0, 343], [packedMixedRest5 % 49 == 0, 49], [packedMixedRest5 % 7 == 0, 7]], 1)",packedMixedRest11:"packedMixedRest7 / pick([[packedMixedRest7 % 1331 == 0, 1331], [packedMixedRest7 % 121 == 0, 121], [packedMixedRest7 % 11 == 0, 11]], 1)",packedMixedRest19:"packedMixedRest11 / pick([[packedMixedRest11 % 6859 == 0, 6859], [packedMixedRest11 % 361 == 0, 361], [packedMixedRest11 % 19 == 0, 19]], 1)",packedMixedWorkgroup:"ceilDiv(tunables.WORKGROUP_SIZE, device.adapterInfo.subgroupMinSize) * device.adapterInfo.subgroupMaxSize if variableSubgroup16To32 else tunables.WORKGROUP_SIZE"},bindings:{input:{elementType:"$T"},output:{elementType:"$T"},params__uniform:{name:"params",struct:[{name:"count",type:"u32",value:"outputRows"},{name:"rank",type:"u32",value:"4"},{name:"inputComponents",type:"u32",value:"dim(shapes.input, ranks.input - 1)"},{name:"outputComponents",type:"u32",value:"dim(shapes.output, ranks.output - 1)"},{name:"inverse",type:"u32",value:"attrs.inverse"},{name:"onesided",type:"u32",value:"attrs.onesided"},{name:"inD0",type:"u32",value:1},{name:"inD1",type:"u32",value:1},{name:"inD2",type:"u32",value:1},{name:"inD3",type:"u32",value:1},{name:"inD4",type:"u32",value:"dftAttrOuter"},{name:"inD5",type:"u32",value:"dim(shapes.input, dftAxisAttr)"},{name:"inD6",type:"u32",value:"dftAttrInputInner"},{name:"outD0",type:"u32",value:1},{name:"outD1",type:"u32",value:1},{name:"outD2",type:"u32",value:1},{name:"outD3",type:"u32",value:1},{name:"outD4",type:"u32",value:"dftAttrOutputOuter"},{name:"outD5",type:"u32",value:"dim(shapes.output, dftAxisAttr)"},{name:"outD6",type:"u32",value:"dftAttrOutputInner"},{name:"axisAttr",type:"u32",value:"1"}]},params_naive:{name:"params",struct:[{name:"count",type:"u32",value:"outputRows"},{name:"signalLength",type:"u32",value:"dim(shapes.input, dftAxisAttr)"},{name:"inputComponents",type:"u32",value:"dim(shapes.input, ranks.input - 1)"},{name:"outputComponents",type:"u32",value:"dim(shapes.output, ranks.output - 1)"},{name:"outputLength",type:"u32",value:"dim(shapes.output, dftAxisAttr)"},{name:"inverse",type:"u32",value:"attrs.inverse"},{name:"onesided",type:"u32",value:"attrs.onesided"}]}},variants:[{id:"contiguous_naive",priority:10,when:["not present.axis","not present.dft_length","ranks.input <= 3","ranks.output == ranks.input","ranks.input == 2 or dim(shapes.input, 0) == dim(shapes.output, 0)","dim(shapes.input, ranks.input - 1) >= 1","dim(shapes.input, ranks.input - 1) <= 2","((attrs.onesided == 0 and dim(shapes.output, dftAxisAttr) == dim(shapes.input, dftAxisAttr) and dim(shapes.output, ranks.input - 1) == 2) or (attrs.onesided != 0 and attrs.inverse == 0 and dim(shapes.input, ranks.input - 1) == 1 and dim(shapes.output, dftAxisAttr) == floor(dim(shapes.input, dftAxisAttr) / 2) + 1 and dim(shapes.output, ranks.input - 1) == 2) or (attrs.onesided != 0 and attrs.inverse != 0 and dim(shapes.input, ranks.input - 1) == 2 and dim(shapes.input, dftAxisAttr) == floor(dim(shapes.output, dftAxisAttr) / 2) + 1 and dim(shapes.output, ranks.input - 1) == 1))"],passes:[{id:"main",name:"DFT.ContiguousNaive",shader:"dft-contiguous-naive.wgsl.jinja",derive:{batched:"ranks.input == 3"},bindings:["input","output","params_naive"],dispatch:{threads:"outputRows",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"general_attr",priority:5,when:["not present.axis","not present.dft_length","dftAxisAttr <= ranks.input - 2","ranks.input == ranks.output","dim(shapes.input, ranks.input - 1) >= 1","dim(shapes.input, ranks.input - 1) <= 2","dim(shapes.output, ranks.output - 1) >= 1","dim(shapes.output, ranks.output - 1) <= 2"],passes:[{id:"main",name:"DFT.General",shader:"dft-general.wgsl.jinja",derive:{hasAxisInput:!1,hasDftLengthInput:!1,skipRuntimeAxis1:!1},bindings:["input","output","params__uniform"],dispatch:{threads:"outputRows",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["dft-accumulator.a9662a53ad.wgsl.jinja",`{% set pairedDft = compensatedDft %}{% if compensatedDft %}{% include "ops/_shared/fft-residual-protected-arithmetic.547d7e2074.wgsl.jinja" %}{% include "ops/_shared/fft-quarter-twiddle-pair.efbc1c05d1.wgsl.jinja" %}{% else %}{% include "ops/_shared/dft-precise-twiddle.16fc24b25e.wgsl.jinja" %}{% endif %}{% set dftTwiddleFn = "fft_quarter_twiddle" if compensatedDft else "dft_precise_twiddle" %}{% macro dft_forward_accumulate(phase_name) %}{% if compensatedDft %}let root_pair=fft_quarter_twiddle_pair({{ phase_name }},signal_length);let c=root_pair.xy;let s=root_pair.zw;let xr_pair=vec2<f32>(xr,0.0);let xi_pair=vec2<f32>(xi,0.0);let real_next=fft_pair_add(vec2<f32>(real,real_low),fft_pair_add(fft_pair_mul(xr_pair,c),-fft_pair_mul(xi_pair,s)));let imag_next=fft_pair_add(vec2<f32>(imag,imag_low),fft_pair_add(fft_pair_mul(xr_pair,s),fft_pair_mul(xi_pair,c)));real=real_next.x;real_low=real_next.y;imag=imag_next.x;imag_low=imag_next.y;
{% else %}real=real+xr*c-xi*s;imag=imag+xr*s+xi*c;
{% endif %}{% endmacro %}`],["dft-naive-accumulate.217c876d40.wgsl.jinja",`{% include "dft-accumulator.a9662a53ad.wgsl.jinja" %}{% include "ops/_shared/float-io-cast.9e460ce304.wgsl.jinja" %}{% if not compensatedDft %}fn is_nyquist(freq:u32,signal_length:u32)->bool{return signal_length%2u==0u&&freq*2u==signal_length;}
{% endif %}{% macro dft_naive_accumulate(transform_index, input_axis_length, output_axis_length, batch_index, other_index, include_batch=true, include_other=true) %}var real=0.0;var imag=0.0;
{% if compensatedDft %}var real_low=0.0;var imag_low=0.0;
{% endif %}{% if not compensatedDft %}let inverse=params.inverse!=0u;let onesided_inverse=inverse&&params.onesided!=0u;let signal_length=select({{ input_axis_length }},{{ output_axis_length }},onesided_inverse);if(onesided_inverse){let used_bins=min({{ input_axis_length }},signal_length/2u+1u);for(var freq=0u;freq<used_bins;freq=freq+1u){let input_base=transform_input_base(freq,{{ batch_index }});let xr={{ io_f32("input[input_base]") }};var xi=0.0;if(params.inputComponents==2u){xi={{ io_f32("input[input_base + 1u]") }};}let phase=({{ transform_index }}*freq)%signal_length;let twiddle={{ dftTwiddleFn }}(phase,signal_length,1.0);let c=twiddle.x;let s=twiddle.y;var weight=1.0;if(freq!=0u&&!is_nyquist(freq,signal_length)){weight=2.0;}real=real+weight*(xr*c-xi*s);imag=imag+weight*(xr*s+xi*c);}real=real/f32(signal_length);imag=imag/f32(signal_length);}else{
{% else %}let signal_length={{ input_axis_length }};
{% endif %}{% if not pairedDft %}let sign=select(-1.0,1.0,inverse);
{% endif %}for(var n=0u;n<{{ input_axis_length }};n=n+1u){let input_base=transform_input_base(n,{{ batch_index }});let xr={{ io_f32("input[input_base]") }};var xi=0.0;if(params.inputComponents==2u){xi={{ io_f32("input[input_base + 1u]") }};}let phase=({{ transform_index }}*n)%signal_length;
{% if not pairedDft %}let twiddle={{ dftTwiddleFn }}(phase,signal_length,sign);let c=twiddle.x;let s=twiddle.y;
{% endif %}{{ dft_forward_accumulate("phase") }} }
{% if not compensatedDft %}if(inverse){real=real/f32(signal_length);imag=imag/f32(signal_length);} }
{% endif %}{% if compensatedDft %}real=fma(real,1.0,real_low);imag=fma(imag,1.0,imag_low);
{% endif %}{% endmacro %}`],["dft-contiguous-naive.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "dft-naive-accumulate.217c876d40.wgsl.jinja" %}fn transform_input_base(sample:u32,batch:u32)->u32{return(batch*params.signalLength+sample)*params.inputComponents;}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "index") }} let transform_index=index%params.outputLength;let batch=index/params.outputLength;{{ dft_naive_accumulate("transform_index", "params.signalLength", "params.outputLength", "batch", "0u", true, false) }} let output_base=index*params.outputComponents;output[output_base]={{ io_element("real") }};if(params.outputComponents==2u){output[output_base+1u]={{ io_element("imag") }};} }`],["dft-general.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "dft-accumulator.a9662a53ad.wgsl.jinja" %}{% include "ops/_shared/float-io-cast.9e460ce304.wgsl.jinja" %}{% if not compensatedDft %}fn is_nyquist(freq:u32,signal_length:u32)->bool{return signal_length%2u==0u&&freq*2u==signal_length;}
{% endif %}fn resolve_axis()->u32{return params.axisAttr;}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "index") }} let in_dims=array<u32,7>(params.inD0,params.inD1,params.inD2,params.inD3,params.inD4,params.inD5,params.inD6);let out_dims=array<u32,7>(params.outD0,params.outD1,params.outD2,params.outD3,params.outD4,params.outD5,params.outD6);let n_logical=params.rank-1u;let pad=7u-n_logical;let resolved_axis=resolve_axis();let transform_slot=resolved_axis+pad;var coords=array<u32,7>(0u,0u,0u,0u,0u,0u,0u);var rem=index;for(var slot=7u;slot>pad;slot=slot-1u){let dim=out_dims[slot-1u];coords[slot-1u]=rem%dim;rem=rem/dim;}let transform_index=coords[transform_slot];let input_axis_length=in_dims[transform_slot];
{% if not compensatedDft %}let output_axis_length=out_dims[transform_slot];
{% endif %}{% if not compensatedDft %}let inverse=params.inverse!=0u;let onesided_inverse=inverse&&params.onesided!=0u;
{% endif %}var signal_length=input_axis_length;
{% if not compensatedDft %}if(onesided_inverse){signal_length=output_axis_length;}
{% endif %}var real=0.0;var imag=0.0;
{% if compensatedDft %}var real_low=0.0;var imag_low=0.0;
{% endif %}{% if not compensatedDft %}if(onesided_inverse){let used_bins=min(input_axis_length,signal_length/2u+1u);for(var freq=0u;freq<used_bins;freq=freq+1u){var in_coords=coords;in_coords[transform_slot]=freq;var in_linear=0u;for(var slot=0u;slot<7u;slot=slot+1u){in_linear=in_linear*in_dims[slot]+in_coords[slot];}let in_base=in_linear*params.inputComponents;let xr={{ io_f32("input[in_base]") }};var xi=0.0;if(params.inputComponents==2u){xi={{ io_f32("input[in_base + 1u]") }};}let kn_mod=(transform_index*freq)%signal_length;let twiddle={{ dftTwiddleFn }}(kn_mod,signal_length,1.0);let c=twiddle.x;let s=twiddle.y;var weight=1.0;if(freq!=0u&&!is_nyquist(freq,signal_length)){weight=2.0;}real=real+weight*(xr*c-xi*s);imag=imag+weight*(xr*s+xi*c);}real=real/f32(signal_length);imag=imag/f32(signal_length);}else{
{% endif %}{% if not pairedDft %}let sign=select(-1.0,1.0,inverse);
{% endif %}for(var n=0u;n<signal_length;n=n+1u){if(n>=input_axis_length){continue;}var in_coords=coords;in_coords[transform_slot]=n;var in_linear=0u;for(var slot=0u;slot<7u;slot=slot+1u){in_linear=in_linear*in_dims[slot]+in_coords[slot];}let in_base=in_linear*params.inputComponents;let xr={{ io_f32("input[in_base]") }};var xi=0.0;if(params.inputComponents==2u){xi={{ io_f32("input[in_base + 1u]") }};}let kn_mod=(transform_index*n)%signal_length;
{% if not pairedDft %}let twiddle={{ dftTwiddleFn }}(kn_mod,signal_length,sign);let c=twiddle.x;let s=twiddle.y;
{% endif %}{{ dft_forward_accumulate("kn_mod") }} }
{% if not compensatedDft %}if(inverse){real=real/f32(signal_length);imag=imag/f32(signal_length);} }
{% endif %}{% if compensatedDft %}real=fma(real,1.0,real_low);imag=fma(imag,1.0,imag_low);
{% endif %}let output_base=index*params.outputComponents;output[output_base]={{ io_element("real") }};if(params.outputComponents==2u){output[output_base+1u]={{ io_element("imag") }};} }`]],assumes:[["dft-contiguous-naive.wgsl.jinja",{ioScalar:"f32",batched:!0}],["dft-general.wgsl.jinja",{ioScalar:"f32"}]]}],["ai.onnx.Exp",{manifest:{domain:"ai.onnx",name:"Exp",sinceVersion:13,inputs:{x:{onnx:"input",dtype:"T"}},outputs:{y:{onnx:"output",dtype:"T",rank:"ranks.x",shape:"shapes.x"}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_layout_vec4",priority:20,when:["numel(shapes.x) > 0","numel(shapes.x) % 4 == 0","numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Exp.Vec4",shader:"ops/_shared/unary-vec4.wgsl.jinja",derive:{op:'"exp"',vec4PerThread:"4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1"},bindings:["x","y","params_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y) / 4, 4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"scalar",when:["numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Exp",shader:"ops/_shared/unary-scalar.wgsl.jinja",derive:{op:'"exp"',itemsPerInvocation:4},bindings:["x_scalar","y_scalar","params_2_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{x:{buffer:"read-only-storage",elementType:"$vectorScalar"},y:{buffer:"storage",elementType:"$vectorScalar"},params_unary:{struct:[{name:"count",type:"u32",value:"numel(shapes.y) / 4"}],name:"params"},x_scalar:{buffer:"read-only-storage",name:"x",elementType:"$scalar"},y_scalar:{buffer:"storage",name:"y",elementType:"$scalar"},params_2_unary:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}}},assets:[["ops/_shared/unary-vec4.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}  let xv = x[i];
y[i]={{ vectorScalar }}(exp(vec4<f32>(xv)));
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/unary-scalar.wgsl.jinja",'{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} {{ flat_tail_open() }} y[i]={{ scalar }}(exp(f32(x[i])));{{ flat_tail_close() }} }']],assumes:[["ops/_shared/unary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>"}],["ops/_shared/unary-scalar.wgsl.jinja",{scalar:"f32"}]]}],["ai.onnx.Gather",{manifest:{domain:"ai.onnx",name:"Gather",sinceVersion:13,inputs:{data:{dtype:"T"},indices:{dtype:"I"}},outputs:{output:{dtype:"T",rank:"ranks.data + ranks.indices - 1",shape:"prefix(shapes.data, attrs.axis if attrs.axis >= 0 else attrs.axis + ranks.data) + shapes.indices + suffix(shapes.data, (attrs.axis if attrs.axis >= 0 else attrs.axis + ranks.data) + 1)"}},attributes:{axis:{default:0}},typeConstraints:{T:["float32","float16","uint32","int32","int16","uint8","int8","bool","int64"],I:["int32"]},tunables:{WORKGROUP_SIZE:{default:256}},derive:{indexScalar:"dtypes.I"},when:["ranks.output == ranks.data + ranks.indices - 1","f16Ok(dtypes.T)"],variants:[{id:"row_fast_vec4",priority:25,when:['dtypes.T != "vec2<u32>"',"ranks.data >= 2","(attrs.axis == 0 or attrs.axis + ranks.data == 0)","numel(shapes.indices) > 0","inner(shapes.data, 0) % 4 == 0","numel(shapes.output) == numel(shapes.indices) * inner(shapes.data, 0)"],tunables:{WORKGROUP_SIZE:{default:64}},derive:{vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Gather.RowVec4",shader:"ops/_shared/datamove-gather-row.wgsl.jinja",derive:{numIndices:"numel(shapes.indices)",axisDimSpec:"dim(shapes.data, 0)",rowVecs:"inner(shapes.data, 0) / 4"},bindings:[{arg:"data",elementType:"$vectorScalar"},"indices",{arg:"output",elementType:"$vectorScalar"}],dispatch:{x:"min(numel(shapes.indices), device.limits.maxComputeWorkgroupsPerDimension)",y:1,z:1}}]},{id:"generic",when:["ranks.data >= 1","attrs.axis + ranks.data >= 0","attrs.axis < ranks.data"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Gather",shader:"gather.wgsl.jinja",derive:{dataShape:"shapes.data",indicesShape:"shapes.indices",outputShape:"shapes.output",dataRank:"ranks.data",indicesRank:"ranks.indices",outputRank:"ranks.output",axisSpec:"attrs.axis if attrs.axis >= 0 else attrs.axis + ranks.data"},bindings:[{arg:"data",elementType:"$scalar"},"indices",{arg:"output",elementType:"$scalar"},{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.output)"}]}],dispatch:{threads:"numel(shapes.output)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["ops/_shared/datamove-gather-row.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const WG:u32={{ tunables.WORKGROUP_SIZE }}u;const NUM_INDICES:u32={{ numIndices }}u;const AXIS_DIM:u32={{ axisDimSpec }}u;const ROW_VECS:u32={{ rowVecs }}u;const INVALID_INDEX:u32=0xffffffffu;
{% include "ops/_shared/normalize-index.fbcdd05a27.wgsl.jinja" %}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){for(var row=wid.x;row<NUM_INDICES;row+=nwg.x){let gather_index=normalize_index(indices[row],AXIS_DIM);let out_base=row*ROW_VECS;if(gather_index==INVALID_INDEX){for(var j=lid.x;j<ROW_VECS;j+=WG){output[out_base+j]={{ vectorScalar }}();} }else{let src_base=gather_index*ROW_VECS;for(var j=lid.x;j<ROW_VECS;j+=WG){output[out_base+j]=data[src_base+j];} } } }`],["gather.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const AXIS_DIM:u32={{ dataShape[axisSpec] }}u;const INVALID_INDEX:u32=0xffffffffu;
{% include "ops/_shared/normalize-index.fbcdd05a27.wgsl.jinja" %}fn index_offset(out_index:u32)->u32{var rem=out_index;var offset=0u;
{% for out_axis in range(outputRank) %}{% set out_stride = namespace(value=1) %}{% for j in range(out_axis + 1, outputRank) %}{% set out_stride.value = out_stride.value * outputShape[j] %}{% endfor %}{% if out_axis >= axisSpec and out_axis < axisSpec + indicesRank %}let coord{{ out_axis }}=rem/{{ max(out_stride.value, 1) }}u;
{% endif %}rem=rem% {{ max(out_stride.value, 1) }}u;
{% if out_axis >= axisSpec and out_axis < axisSpec + indicesRank %}{% set idx_axis = out_axis - axisSpec %}{% set idx_stride = namespace(value=1) %}{% for j in range(idx_axis + 1, indicesRank) %}{% set idx_stride.value = idx_stride.value * indicesShape[j] %}{% endfor %}offset=offset+coord{{ out_axis }}*{{ idx_stride.value }}u;
{% endif %}{% endfor %}return offset;}fn data_offset(out_index:u32,gather_index:u32)->u32{var rem=out_index;var offset=0u;
{% for out_axis in range(outputRank) %}{% set out_stride = namespace(value=1) %}{% for j in range(out_axis + 1, outputRank) %}{% set out_stride.value = out_stride.value * outputShape[j] %}{% endfor %}{% set used = out_axis < axisSpec or out_axis >= axisSpec + indicesRank %}{% if used %}let dcoord{{ out_axis }}=rem/{{ max(out_stride.value, 1) }}u;
{% endif %}rem=rem% {{ max(out_stride.value, 1) }}u;
{% endfor %}{% for data_axis in range(dataRank) %}{% set data_stride = namespace(value=1) %}{% for j in range(data_axis + 1, dataRank) %}{% set data_stride.value = data_stride.value * dataShape[j] %}{% endfor %}{% if data_axis < axisSpec %}offset=offset+dcoord{{ data_axis }}*{{ data_stride.value }}u;
{% elif data_axis == axisSpec %}offset=offset+gather_index*{{ data_stride.value }}u;
{% else %}{% set out_axis = data_axis + indicesRank - 1 %}offset=offset+dcoord{{ out_axis }}*{{ data_stride.value }}u;
{% endif %}{% endfor %}return offset;}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE) }} let gather_index=normalize_index(indices[index_offset(i)],AXIS_DIM);if(gather_index==INVALID_INDEX){output[i]={{ scalar }}(0.0);return;}output[i]=data[data_offset(i,gather_index)];}`]],assumes:[["ops/_shared/datamove-gather-row.wgsl.jinja",{indexScalar:"i32",vectorScalar:"vec4<f32>"}],["gather.wgsl.jinja",{indexScalar:"i32",scalar:"f32",dataRank:2,indicesRank:1,outputRank:2}]]}],["ai.onnx.Gelu",{manifest:{domain:"ai.onnx",name:"Gelu",sinceVersion:20,inputs:{x:{onnx:"X",dtype:"T"}},outputs:{y:{onnx:"Y",dtype:"T",rank:"ranks.x",shape:"shapes.x"}},attributes:{approximate:{default:"none"}},attributeConstraints:{approximate:{values:["none","tanh"]}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256}},derive:{scalar:"dtypes.T"},when:["numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],bindings:{x:{elementType:"$vectorScalar"},y:{elementType:"$vectorScalar"},params:{struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}},variants:[{id:"same_layout_vec4",priority:20,when:["numel(shapes.x) > 0","numel(shapes.x) % 4 == 0"],derive:{vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Gelu.Vec4",shader:"ops/_shared/unary-vec4.wgsl.jinja",derive:{op:'"gelu"',approximateSpec:"attrs.approximate",vec4PerThread:"4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1"},bindings:["x","y",{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y) / 4"}]}],dispatch:{threads:"ceilDiv(numel(shapes.y) / 4, 4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"scalar",derive:{approximate:"attrs.approximate"},passes:[{id:"main",name:"Gelu",shader:"ops/_shared/unary-scalar.wgsl.jinja",derive:{op:'"gelu"',itemsPerInvocation:4},bindings:[{arg:"x"},{arg:"y"},"params"],dispatch:{threads:"ceilDiv(numel(shapes.x), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["ops/_shared/unary-vec4.wgsl.jinja",`{% macro unary_vec4_map(fn, value="fv") -%}vec4<f32>({{ fn }}({{ value }}.x),{{ fn }}({{ value }}.y),{{ fn }}({{ value }}.z),{{ fn }}({{ value }}.w))
{%- endmacro %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.8bfc3a16e6.wgsl.jinja" %}{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}  let xv = x[i];
let fv=vec4<f32>(xv);y[i]={{ vectorScalar }}({{ unary_vec4_map("gelu_value") }});
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/unary-scalar.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.ba735b4ed2.wgsl.jinja" %}{{ flat_tail_open() }}
y[i]={{ scalar }}(gelu_value(f32(x[i])));{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/unary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>",approximateSpec:"none"}],["ops/_shared/unary-scalar.wgsl.jinja",{scalar:"f32",approximate:"none"}]]}],["ai.onnx.LayerNormalization",{manifest:{domain:"ai.onnx",name:"LayerNormalization",sinceVersion:17,inputs:{x:{onnx:"X",dtype:"T"},scale:{onnx:"Scale",dtype:"T"},b:{onnx:"B",dtype:"T",optional:!0}},outputs:{y:{onnx:"Y",dtype:"T",rank:"ranks.x",shape:"shapes.x"},mean:{onnx:"Mean",dtype:"float32",rank:"ranks.x",optional:!0},invStdDev:{onnx:"InvStdDev",dtype:"float32",rank:"ranks.x",optional:!0}},attributes:{axis:{default:-1},epsilon:{default:1e-5},stash_type:{default:1}},attributeConstraints:{stash_type:{values:[1]}},typeConstraints:{T:["float32","float16"]},tunables:{MAX_WORKGROUP_SIZE:{default:256},SCALAR_FAST_MAX_HIDDEN:{default:4096}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",normWorkgroupCap:"min(tunables.MAX_WORKGROUP_SIZE, deviceWorkgroupCap)",hasSubgroupId:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',lastAxisWg:"min(normWorkgroupCap, pow2ceil(dim(shapes.x, -1)))",lastAxisWgVec4:"min(normWorkgroupCap, pow2ceil(dim(shapes.x, -1) / 4))",lastAxisContractOk:"ranks.x >= 1 and ranks.y == ranks.x and numel(shapes.x) == numel(shapes.y) and (attrs.axis == -1 or attrs.axis == ranks.x - 1)",suffixAxisContractOk:"ranks.x >= 2 and ranks.y == ranks.x and numel(shapes.x) == numel(shapes.y) and attrs.axis + ranks.x >= 0 and attrs.axis < ranks.x and not (attrs.axis == -1 or attrs.axis == ranks.x - 1)",axisNorm:"attrs.axis if attrs.axis >= 0 else attrs.axis + ranks.x",normRows:"numel(shapes.x) / max(1, dim(shapes.x, -1)) if lastAxisContractOk else outer(shapes.x, axisNorm)",normRowStride:"max(1, min(normRows, device.limits.maxComputeWorkgroupsPerDimension))",suffixAxisSize:"numel(shapes.x) / max(1, outer(shapes.x, axisNorm))",suffixAxisWg:"min(normWorkgroupCap, pow2ceil(suffixAxisSize))",suffixAxisWgVec4:"min(normWorkgroupCap, pow2ceil(suffixAxisSize / 4))",genericHiddenSize:"dim(shapes.x, -1) if lastAxisContractOk else suffixAxisSize",genericWorkgroupSize:"lastAxisWg if lastAxisContractOk else suffixAxisWg",scaleExactOk:"ranks.x >= 1 and ranks.scale >= 1 and numel(shapes.scale) == dim(shapes.x, -1) and dim(shapes.scale, -1) == dim(shapes.x, -1)",scaleBroadcastOk:"ranks.scale >= 0 and ranks.scale <= ranks.x and broadcastable(shapes.scale, shapes.x)",biasExactOk:"present.b and ranks.x >= 1 and ranks.b >= 1 and numel(shapes.b) == dim(shapes.x, -1) and dim(shapes.b, -1) == dim(shapes.x, -1)",biasBroadcastOk:"present.b and ranks.b >= 0 and ranks.b <= ranks.x and broadcastable(shapes.b, shapes.x)",suffixScaleExactOk:"suffixAxisContractOk and scaleBroadcastOk and numel(shapes.scale) == suffixAxisSize",suffixBiasExactOk:"present.b and suffixAxisContractOk and biasBroadcastOk and numel(shapes.b) == suffixAxisSize",lastAxisExactScaleOk:"lastAxisContractOk and scaleExactOk",lastAxisBroadcastScaleOk:"lastAxisContractOk and scaleBroadcastOk",suffixAxisBroadcastScaleOk:"suffixAxisContractOk and scaleBroadcastOk",suffixAxisExactAffineOk:"suffixAxisContractOk and suffixScaleExactOk and suffixBiasExactOk",lastAxisScalarFastOk:'dtypes.T == "f16" or dim(shapes.x, -1) <= tunables.SCALAR_FAST_MAX_HIDDEN',noStatsOutputs:"not present.mean and not present.invStdDev",meanOnlyOutputs:"present.mean and not present.invStdDev",invStdOnlyOutputs:"not present.mean and present.invStdDev",fullStatsOutputs:"present.mean and present.invStdDev",statsRowsOk:"fullStatsOutputs and ranks.x >= 1 and numel(shapes.mean) == normRows and numel(shapes.invStdDev) == normRows",meanRowsOk:"present.mean and ranks.x >= 1 and numel(shapes.mean) == normRows",invStdRowsOk:"present.invStdDev and ranks.x >= 1 and numel(shapes.invStdDev) == normRows",statsOuterOk:"fullStatsOutputs and ranks.x >= 2 and numel(shapes.mean) == normRows and numel(shapes.invStdDev) == normRows"},when:["f16Ok(dtypes.T)"],bindings:{x:{elementType:"$vectorScalar"},scale:{elementType:"$vectorScalar"},y:{elementType:"$vectorScalar"},params:{struct:[{name:"rows",type:"u32",value:"normRows"},{name:"rowStride",type:"u32",value:"normRowStride"}]},bias:{arg:"b",elementType:"$vectorScalar"}},variants:[{id:"last_axis_bias_row_vec4",priority:111,when:["present.b and noStatsOutputs and biasExactOk","lastAxisExactScaleOk","dim(shapes.x, -1) % 4 == 0"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"LayerNormalization.LastAxisRowVec4",shader:"ops/_shared/norm-row-stats.wgsl.jinja",derive:{modeSpec:'"layer"',compensateHalfStats:!0,vec4:!0,hasBias:"present.b",writeStats:"fullStatsOutputs",hidden:"dim(shapes.x, -1)",wg:"lastAxisWgVec4",epsilon:"attrs.epsilon",hiddenVec:"dim(shapes.x, -1) / 4",vecType:'"vec4<" ~ dtypes.T ~ ">"',combineSubgroups:"hasSubgroupId"},bindings:["x","scale","bias","y","params"],dispatch:{workgroups:"normRows"},subgroupCollectivesWidth:"portable"}]},{id:"last_axis_bias_row",priority:101,when:["present.b and noStatsOutputs and biasExactOk","lastAxisExactScaleOk"],demoteWhen:["not lastAxisScalarFastOk"],derive:{scalar:"dtypes.T",vectorScalar:"dtypes.T"},passes:[{id:"main",name:"LayerNormalization.LastAxisRow",shader:"ops/_shared/norm-row-stats.wgsl.jinja",derive:{modeSpec:'"layer"',compensateHalfStats:!0,vec4:!1,hasBias:"present.b",writeStats:"fullStatsOutputs",hidden:"dim(shapes.x, -1)",wg:"lastAxisWg",epsilon:"attrs.epsilon",combineSubgroups:"hasSubgroupId"},bindings:["x","scale","bias","y","params"],dispatch:{workgroups:"normRows"},subgroupCollectivesWidth:"portable"}]},{id:"last_axis_bias",priority:10,when:["present.b","noStatsOutputs","lastAxisBroadcastScaleOk","biasBroadcastOk"],derive:{hasBias:!0,writeMean:!1,writeInvStdDev:!1,scalar:"dtypes.T",vectorScalar:"dtypes.T",hiddenSize:"dim(shapes.x, -1)",workgroupSize:"lastAxisWg",epsilon:"attrs.epsilon"},passes:[{id:"main",name:"LayerNormalization",shader:"layer-normalization.wgsl.jinja",derive:{xShape:"shapes.x",scaleShape:"shapes.scale",biasShape:"shapes.b"},bindings:["x","scale","bias","y","params"],dispatch:{workgroups:"normRows"}}]},{id:"last_axis_broadcast_bias_row_vec4",priority:91,when:['dtypes.T == "f32"',"lastAxisBroadcastScaleOk","dim(shapes.x, -1) >= 4","dim(shapes.x, -1) % 4 == 0","ranks.scale >= 1","dim(shapes.scale, -1) == dim(shapes.x, -1)","noStatsOutputs","present.b and biasBroadcastOk and ranks.b >= 1 and dim(shapes.b, -1) == dim(shapes.x, -1)","true"],demoteWhen:["false"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"LayerNormalization.BroadcastRowsVec4",shader:"ops/_shared/norm-row-stats.wgsl.jinja",derive:{modeSpec:'"layer"',vec4:!0,hasBias:"present.b",writeStats:!1,hidden:"dim(shapes.x, -1)",wg:"lastAxisWgVec4",epsilon:"attrs.epsilon",hiddenVec:"dim(shapes.x, -1) / 4",vecType:'"vec4<" ~ dtypes.T ~ ">"',combineSubgroups:"hasSubgroupId",affineRowBroadcast:!0,xRowShape:"prefix(shapes.x, ranks.x - 1)",scaleRowShape:"prefix(shapes.scale, ranks.scale - 1)",biasRowShape:"prefix(shapes.b, ranks.b - 1) if present.b else []",batchRows:"1",batchLanes:"lastAxisWgVec4"},bindings:["x","scale","bias","y","params"],dispatch:{workgroups:"normRows"},subgroupCollectivesWidth:"portable"}]}]},assets:[["ops/_shared/norm-row-stats.wgsl.jinja",`{% set combineSubgroups = combineSubgroups and not (scalar == "f16" and compensateHalfStats is defined and compensateHalfStats and (device.adapterInfo.subgroupMinSize | default(0)) > 32) %}{% set scalarIo = false %}{% set packedF32 = "vec4<f32>" %}{% set reduceThreadParameters = ", sg_id: u32, num_sg: u32, sg_size: u32"
    if combineSubgroups else ", tid: u32" %}{% set reduceThreadArguments = ", sg_id, num_sg, sg_size"
    if combineSubgroups else ", tid" %}{% if combineSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }}
{% set affineRowBroadcast = affineRowBroadcast | default(false) %}{% set batchRows = batchRows | default(1) %}{% set batchLanes = batchLanes | default(0) %}{% set xRowShape = xRowShape | default([]) %}{% set scaleRowShape = scaleRowShape | default([]) %}{% set biasRowShape = biasRowShape | default([]) %}{% if affineRowBroadcast %}{% include "ops/_shared/broadcast-offset.87f1ae294f.wgsl.jinja" %}{{ broadcast_offset_fn("scale_row_offset", scaleRowShape, scaleRowShape | length, xRowShape, xRowShape | length) }}
{{ broadcast_offset_fn("bias_row_offset", biasRowShape, biasRowShape | length, xRowShape, xRowShape | length) }}
{% endif %}const HIDDEN:u32={{ hidden }}u;
{% if vec4 %}const HIDDEN_V:u32={{ hiddenVec }}u;
{% endif %}const WG:u32={{ wg }}u;
{% if batchRows > 1 %}const ROW_LANES:u32={{ batchLanes }}u;const ROW_BATCH:u32={{ batchRows }}u;
{% endif %}const EPSILON:f32={{ epsilon }};
{% if combineSubgroups %}var<workgroup>sg_partials:array<vec2<f32>,WG>;fn reduce_pair(value:vec2<f32>{{ reduceThreadParameters }})->vec2<f32>{let s=vec2<f32>(subgroupAdd(value.x),subgroupAdd(value.y));let fold_count=max(num_sg,(WG+sg_size-1u)>>firstTrailingBit(sg_size));if(fold_count==1u){return s;}if(subgroupElect()){sg_partials[sg_id]=s;}workgroupBarrier();var total=vec2<f32>(0.0);for(var i=0u;i<fold_count;i++){total+=sg_partials[i];}workgroupBarrier();return total;}
{% else %}var<workgroup>tr0:array<f32,WG>;var<workgroup>tr1:array<f32,WG>;fn reduce_pair(value:vec2<f32>,tid:u32)->vec2<f32>{tr0[tid]=value.x;tr1[tid]=value.y;workgroupBarrier();
  var stride: u32 = {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %} / 2u;
loop{if(stride==0u){break;}
    if (tid{% if batchRows > 1 %} % ROW_LANES{% endif %} < stride) {
tr0[tid]=tr0[tid]+tr0[tid+stride];tr1[tid]=tr1[tid]+tr1[tid+stride];}stride=stride/2u;workgroupBarrier();}
{% if batchRows > 1 %}let row_base=tid-tid%ROW_LANES;let reduced=vec2<f32>(tr0[row_base],tr1[row_base]);
{% else %}let reduced=vec2<f32>(tr0[0],tr1[0]);
{% endif %}workgroupBarrier();return reduced;}
{% endif %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg_id:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if combineSubgroups %},
@builtin(subgroup_id)sg_id:u32,@builtin(num_subgroups)num_sg:u32,
  @builtin(subgroup_size) sg_size: u32{% endif %}){
{% if batchRows > 1 %}let row=(wg_id.x+wg_id.y*params.rowStride)*ROW_BATCH+lid.x/ROW_LANES;let row_active=row<params.rows;
{% else %}let row=wg_id.x+wg_id.y*params.rowStride;if(row>=params.rows){return;}
{% endif %}let tid=lid.x;
{% if vec4 and not scalarIo %}let base=row*HIDDEN_V;
{% else %}let base=row*HIDDEN;
{% endif %}{% if affineRowBroadcast %}let scale_base={{ broadcast_offset_call("scale_row_offset", scaleRowShape, xRowShape, "row") }}*HIDDEN_V;let bias_base={{ broadcast_offset_call("bias_row_offset", biasRowShape, xRowShape, "row") }}*HIDDEN_V;
{% endif %}{% if vec4 %}{% if batchRows > 1 %}var shift=0.0;if(row_active){shift=f32(x[base].x);}
{% else %}let shift=f32(x[base].x);
{% endif %}{% else %}let shift=f32(x[base]);
{% endif %}var acc=vec2<f32>(0.0,0.0);
{% if vec4 %}  for (var i = tid{% if batchRows > 1 %} % ROW_LANES{% endif %}; {% if batchRows > 1 %}row_active && {% endif %}i < HIDDEN_V; i = i + {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %}) {
let v={{ packedF32 }}(x[base+i]);let d=v-{{ packedF32 }}(shift);acc.x=acc.x+d.x+d.y+d.z+d.w;acc.y=acc.y+dot(d,d);}
{% else %}for(var i=tid;i<HIDDEN;i=i+WG){let v=f32(x[base+i]);let d=v-shift;acc.x=acc.x+d;acc.y=acc.y+d*d;}
{% endif %}let totals=reduce_pair(acc{{ reduceThreadArguments }});let mean_d=totals.x/f32(HIDDEN);let variance=max(totals.y/f32(HIDDEN)-mean_d*mean_d,0.0);let inv=inverseSqrt(variance+EPSILON);let row_mean=shift+mean_d;
{% if vec4 %}  for (var i = tid{% if batchRows > 1 %} % ROW_LANES{% endif %}; {% if batchRows > 1 %}row_active && {% endif %}i < HIDDEN_V; i = i + {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %}) {
let idx=base+i;let v={{ packedF32 }}(x[idx]);
    var value = (v - {{ packedF32 }}(row_mean)) * inv * {{ packedF32 }}(scale[{% if affineRowBroadcast %}scale_base + {% endif %}i]);
    value = value + {{ packedF32 }}(bias[{% if affineRowBroadcast %}bias_base + {% endif %}i]);
y[idx]={{ vecType }}(value);}
{% else %}for(var i=tid;i<HIDDEN;i=i+WG){let idx=base+i;let v=f32(x[idx]);var value=(v-row_mean)*inv*f32(scale[i]);value=value+f32(bias[i]);y[idx]={{ scalar }}(value);}
{% endif %}}`],["layer-normalization.wgsl.jinja",`{% include "ops/_shared/broadcast-offset.9034aa7baf.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const HIDDEN:u32={{ hiddenSize }}u;const EPSILON:f32={{ epsilon }};const WG:u32={{ workgroupSize }}u;
{% set xNumel = namespace(value=1) %}{% for dim in xShape %}{% set xNumel.value = xNumel.value * dim %}{% endfor %}{% set scaleNumel = namespace(value=1) %}{% for dim in scaleShape %}{% set scaleNumel.value = scaleNumel.value * dim %}{% endfor %}{% if scaleNumel.value != 1 %}{{ offset_fn("scale_offset", scaleShape, scaleShape | length, scaleShape == xShape, scaleNumel.value, xShape, xShape | length, xNumel.value) }}
{% endif %}{% set biasNumel = namespace(value=1) %}{% for dim in biasShape %}{% set biasNumel.value = biasNumel.value * dim %}{% endfor %}{% if biasNumel.value != 1 %}{{ offset_fn("bias_offset", biasShape, biasShape | length, biasShape == xShape, biasNumel.value, xShape, xShape | length, xNumel.value) }}
{% endif %}var<workgroup>partial:array<f32,WG>;var<workgroup>row_mean:f32;var<workgroup>row_inv:f32;
{% include "ops/_shared/reduce-sum.fdec0f92c8.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let row=wg.x+wg.y*params.rowStride;if(row>=params.rows){return;}let tid=lid.x;let base=row*HIDDEN;var local_sum=0.0;for(var d=tid;d<HIDDEN;d=d+WG){let value=f32(x[base+d]);local_sum=local_sum+value;}let sum=reduce_sum(local_sum,tid);if(tid==0u){row_mean=sum/f32(HIDDEN);}workgroupBarrier();var local_var_sum=0.0;for(var d=tid;d<HIDDEN;d=d+WG){let diff=f32(x[base+d])-row_mean;local_var_sum=local_var_sum+diff*diff;}let var_sum=reduce_sum(local_var_sum,tid);if(tid==0u){let variance=var_sum/f32(HIDDEN);row_inv=inverseSqrt(variance+EPSILON);}workgroupBarrier();for(var d=tid;d<HIDDEN;d=d+WG){let index=base+d;let normalized=(f32(x[index])-row_mean)*row_inv;
    var value = normalized * f32(scale[{% if scaleNumel.value == 1 %}0u{% else %}{{ broadcast_offset_call("scale_offset", scaleShape, xShape, "index") }}{% endif %}]);
    value = value + f32(bias[{% if biasNumel.value == 1 %}0u{% else %}{{ broadcast_offset_call("bias_offset", biasShape, xShape, "index") }}{% endif %}]);
y[index]={{ scalar }}(value);} }`]],assumes:[["ops/_shared/norm-row-stats.wgsl.jinja",{scalar:"f32",hasBias:!0,writeStats:!1}],["layer-normalization.wgsl.jinja",{scalar:"f32"}]]}],["ai.onnx.Mul",{manifest:{domain:"ai.onnx",name:"Mul",sinceVersion:14,inputs:{a:{onnx:"A",dtype:"T"},b:{onnx:"B",dtype:"T"}},outputs:{c:{onnx:"C",dtype:"T",rank:"max(ranks.a, ranks.b)",shape:"broadcastShape(shapes.a, shapes.b)"}},typeConstraints:{T:["float32","float16","int32","uint32","int8","uint8"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_shape_vec4",priority:20,when:["sameShape(shapes.a, shapes.c)","sameShape(shapes.b, shapes.c)","numel(shapes.c) > 0","numel(shapes.c) % 4 == 0","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Mul.Vec4",shader:"ops/_shared/binary-vec4.wgsl.jinja",derive:{op:'"mul"',cDtype:"tensorDtypes.c",vec4PerThread:"4 if numel(shapes.c) * dtypeBytes(tensorDtypes.c) <= 16777216 else 1"},bindings:["a","b","c_binary","params"],dispatch:{threads:"ceilDiv(numel(shapes.c) / 4, 4 if numel(shapes.c) * dtypeBytes(tensorDtypes.c) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"broadcast_vec4",priority:10,when:["ranks.a <= ranks.c","ranks.b <= ranks.c","ranks.c >= 1","numel(shapes.c) % 4 == 0","numel(shapes.c) >= 4","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",aElement:'"vec4<" ~ dtypes.T ~ ">" if sameShape(shapes.a, shapes.c) else dtypes.T',bElement:'"vec4<" ~ dtypes.T ~ ">" if sameShape(shapes.b, shapes.c) else dtypes.T',vec4Scalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Mul",shader:"ops/_shared/binary-broadcast-vec4.wgsl.jinja",derive:{aShape:"shapes.a",bShape:"shapes.b",cShape:"shapes.c",aRank:"ranks.a",bRank:"ranks.b",cRank:"ranks.c",op:'"mul"',cDtype:"tensorDtypes.c",periodicTableCap:"min(floor(tunables.WORKGROUP_SIZE / 2), floor(device.limits.maxComputeWorkgroupStorageSize / 32)) if numel(shapes.c) >= 4 * tunables.WORKGROUP_SIZE else 0"},bindings:["a_element","b_element","c_2_binary","params"],dispatch:{threads:"numel(shapes.c) / 4",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"broadcast",when:["ranks.a <= ranks.c","ranks.b <= ranks.c","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Mul",shader:"ops/_shared/binary-broadcast.wgsl.jinja",derive:{aShape:"shapes.a",bShape:"shapes.b",cShape:"shapes.c",aRank:"ranks.a",bRank:"ranks.b",cRank:"ranks.c",op:'"mul"',cDtype:"tensorDtypes.c",itemsPerInvocation:4},bindings:["a_scalar","b_scalar","c_scalar","params_count"],dispatch:{threads:"ceilDiv(numel(shapes.c), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{a:{elementType:"$vectorScalar"},b:{elementType:"$vectorScalar"},c_binary:{elementType:"$vectorScalar",name:"c"},params:{struct:[{name:"count",type:"u32",value:"numel(shapes.c) / 4"}]},b_scalar:{name:"b",elementType:"$scalar"},a_scalar:{name:"a",elementType:"$scalar"},a_element:{name:"a",elementType:"$aElement"},b_element:{name:"b",elementType:"$bElement"},c_2_binary:{name:"c",elementType:"$vec4Scalar"},c_scalar:{name:"c",elementType:"$scalar"},params_count:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.c)"}]}}},assets:[["ops/_shared/binary-vec4.wgsl.jinja",`{% set binaryOperator = "*" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}    let av = a[i];
let bv=b[i];c[i]=av {{ binaryOperator }} bv;
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/binary-broadcast-vec4.wgsl.jinja",`{% include "ops/_shared/broadcast-offset.9034aa7baf.wgsl.jinja" %}{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{% include "ops/_shared/binary-broadcast-operand.5796cb977c.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set a_numel = namespace(value=1) %}{% for d in aShape %}{% set a_numel.value = a_numel.value * d %}{% endfor %}{% set b_numel = namespace(value=1) %}{% for d in bShape %}{% set b_numel.value = b_numel.value * d %}{% endfor %}{% set c_numel = namespace(value=1) %}{% for d in cShape %}{% set c_numel.value = c_numel.value * d %}{% endfor %}{% set a_same = namespace(value=(aRank == cRank)) %}{% if a_same.value %}{% for axis in range(cRank) %}{% if aShape[axis] != cShape[axis] %}{% set a_same.value = false %}{% endif %}{% endfor %}{% endif %}{% set b_same = namespace(value=(bRank == cRank)) %}{% if b_same.value %}{% for axis in range(cRank) %}{% if bShape[axis] != cShape[axis] %}{% set b_same.value = false %}{% endif %}{% endfor %}{% endif %}{% set a_inner = aShape[aRank - 1] if aRank >= 1 else 1 %}{% set b_inner = bShape[bRank - 1] if bRank >= 1 else 1 %}{% set c_inner = cShape[cRank - 1] %}{% set crosses_inner_rows = c_inner % 4 != 0 %}{% if a_numel.value == 1 %}{% set a_mode = "scalar" %}{% elif a_inner == 1 %}{% set a_mode = "splat" %}{% else %}{% set a_mode = "contig" %}{% endif %}{% if b_numel.value == 1 %}{% set b_mode = "scalar" %}{% elif b_inner == 1 %}{% set b_mode = "splat" %}{% else %}{% set b_mode = "contig" %}{% endif %}{% set a_table = namespace(period=1, matching=true, active=false) %}{% set b_table = namespace(period=1, matching=true, active=false) %}{{ periodic_geometry(a_table, aShape, aRank, a_same.value, a_numel.value, cShape, cRank, crosses_inner_rows, periodicTableCap) }} {{ periodic_geometry(b_table, bShape, bRank, b_same.value, b_numel.value, cShape, cRank, crosses_inner_rows, periodicTableCap) }}
{% if not a_same.value and a_mode != "scalar" and not a_table.active %}{{ offset_fn("a_offset", aShape, aRank, false, a_numel.value, cShape, cRank, c_numel.value) }}
{% endif %}{% if not b_same.value and b_mode != "scalar" and not b_table.active and not (nativeTrailingB | default(false)) %}{{ offset_fn("b_offset", bShape, bRank, false, b_numel.value, cShape, cRank, c_numel.value) }}
{% endif %}{% set acc = "f32" %}{% set wrap = "" %}{{ periodic_declaration("a", a_table, acc) }} {{ periodic_declaration("b", b_table, acc) }}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})
fn main(@builtin(global_invocation_id) gid: vec3<u32>{% if a_table.active or b_table.active %}, @builtin(local_invocation_index) lid: u32{% endif %}) {
{{ periodic_stage("a", a_table, acc) }} {{ periodic_stage("b", b_table, acc) }}
{% if a_table.active or b_table.active %}workgroupBarrier();
{% endif %}{{ flat_index_2d(tunables.WORKGROUP_SIZE, "i4") }}
{% set needsBase = c_numel.value != 0 and ((not a_same.value and a_mode != "scalar" and not a_table.active) or (not b_same.value and b_mode != "scalar" and not b_table.active and not (nativeTrailingB | default(false)))) %}{% if needsBase %}let base=i4*4u;
{% endif %}{{ broadcast_vector_load("a", aShape, a_same.value, a_mode, a_table, acc, crosses_inner_rows, cShape) }} {{ broadcast_vector_load("b", bShape, b_same.value, b_mode, b_table, acc, crosses_inner_rows, cShape) }} c[i4]={{ wrap }}(vec4<{{ scalar }}>(av*bv));}`],["ops/_shared/binary-broadcast.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{% include "ops/_shared/broadcast-offset.25bcd5874d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{{ binary_broadcast_offsets() }} {{ flat_tail_open() }} let av=f32(a[{{ broadcast_offset_call("a_offset", aShape, cShape, "i") }}]);let bv=f32(b[{{ broadcast_offset_call("b_offset", bShape, cShape, "i") }}]);c[i]={{ scalar }}(av*bv);{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/binary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>",cDtype:"float32"}],["ops/_shared/binary-broadcast-vec4.wgsl.jinja",{scalar:"f32",cDtype:"float32"}],["ops/_shared/binary-broadcast.wgsl.jinja",{scalar:"f32",cDtype:"float32"}]]}],["ai.onnx.ReduceSum",{manifest:{domain:"ai.onnx",name:"ReduceSum",onnx:{adapt:{"inputs.axes":{to:"attributes.axes",note:"The optional axes tensor is supplied through the `axes` request attribute."}}},sinceVersion:13,inputs:{x:{onnx:"data",dtype:"T"}},outputs:{y:{onnx:"reduced",dtype:"T",rank:"ranks.x if attrs.keepdims == 1 or ((attrs.axes | length) == 0 and attrs.noop_with_empty_axes == 1) else (ranks.x - (attrs.axes | length) if (attrs.axes | length) > 0 else 0)"}},attributes:{keepdims:{default:1},noop_with_empty_axes:{default:0},axes:{default:[]}},attributeConstraints:{keepdims:{values:[0,1]},noop_with_empty_axes:{values:[0,1]}},typeConstraints:{T:["float32","float16","int32"]},tunables:{WORKGROUP_SIZE:{default:256},VECTOR_WIDTH:{default:4},ROW_PARALLEL_MIN_COLS:{default:64},SUBGROUP_MIN_COLS:{default:256},SUBGROUP_SMALL_ROW_LIMIT:{default:32768},AXIS0_SPLIT_MIN_ROWS:{default:8192},AXIS0_SPLIT_TARGET_ROWS:{default:256},AXIS0_MAX_SPLITS:{default:128},AXIS0_TILE_MIN_ROWS:{default:64},AXIS0_TILE_MIN_COLS:{default:16},AXIS0_TILE_COLS:{default:16},AXIS_SPLIT_TILE_COLS:{default:8},FULL_REDUCE_MIN_ELEMENTS:{default:8192},FULL_REDUCE_MAX_SPLITS:{default:256},CONTIGUOUS_SUFFIX_MIN_COLS:{default:256},ROW_SERIAL_MIN_ROWS:{default:8192},ROW_SERIAL_MAX_COLS:{default:1024},MULTI_AXIS_COOP_MAX_OUTPUTS:{default:8192},MULTI_AXIS_COOP_MIN_REDUCED:{default:256}},derive:{reduceOp:'"sum"',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',reduceWorkgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",treeWorkgroupOk:'reduceWorkgroupSize > 0 and pow2ceil(reduceWorkgroupSize) == reduceWorkgroupSize and reduceWorkgroupSize * dtypeBytes("float32") <= device.limits.maxComputeWorkgroupStorageSize',subgroupWorkgroupFloor:"min(reduceWorkgroupSize, max(1, device.adapterInfo.subgroupMaxSize))",lastAxisRows:"rows(shapes.x, ranks.x - 1) if ranks.x > 0 else 1",lastAxisCols:"dim(shapes.x, ranks.x - 1) if ranks.x > 0 else 1",rowSerialPreferred:"lastAxisRows >= tunables.ROW_SERIAL_MIN_ROWS and lastAxisCols <= tunables.ROW_SERIAL_MAX_COLS",axis0Rows:"dim(shapes.x, 0) if ranks.x >= 2 else 0",axis0Cols:"dim(shapes.x, 1) if ranks.x >= 2 else 0",axis0SplitCount:"min(tunables.AXIS0_MAX_SPLITS, pow2ceil(ceilDiv(axis0Rows, tunables.AXIS0_SPLIT_TARGET_ROWS)))",axis0SplitScratchBytes:'axis0SplitCount * axis0Cols * dtypeBytes("float32")',axis0SplitPathFits:"axis0SplitCount <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(ceilDiv(axis0Cols, reduceWorkgroupSize), device.limits.maxComputeWorkgroupsPerDimension) <= device.limits.maxComputeWorkgroupsPerDimension and axis0SplitScratchBytes <= device.limits.maxStorageBufferBindingSize and axis0SplitScratchBytes <= device.limits.maxBufferSize",reduceAxis:"(attrs.axes[0] + ranks.x if attrs.axes[0] < 0 else attrs.axes[0]) if ((attrs.axes | length) == 1 and isUniqueIntList(attrs.axes, 0 - ranks.x, ranks.x, 1)) else ranks.x",axisSplitDim:"dim(shapes.x, reduceAxis) if ranks.x >= 2 and reduceAxis < ranks.x else 0",axisSplitInner:"inner(shapes.x, reduceAxis) if ranks.x >= 2 and reduceAxis < ranks.x else 1",axisSplitOutputs:"numel(shapes.y)",axisSplitCount:"min(tunables.AXIS0_MAX_SPLITS, pow2ceil(ceilDiv(axisSplitDim, tunables.AXIS0_SPLIT_TARGET_ROWS)))",axisSplitScratchBytes:"axisSplitCount * axisSplitOutputs * 4",axisSplitPathFits:"axisSplitCount <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(ceilDiv(axisSplitOutputs, reduceWorkgroupSize), device.limits.maxComputeWorkgroupsPerDimension) <= device.limits.maxComputeWorkgroupsPerDimension and axisSplitScratchBytes <= device.limits.maxStorageBufferBindingSize and axisSplitScratchBytes <= device.limits.maxBufferSize",axis0TilePathFits:"treeWorkgroupOk and tunables.AXIS0_TILE_COLS > 0 and tunables.AXIS0_TILE_COLS <= reduceWorkgroupSize and reduceWorkgroupSize % tunables.AXIS0_TILE_COLS == 0",flatItems:"floor(numel(shapes.x) / tunables.VECTOR_WIDTH)",flatSplitCount:"max(1, min(tunables.FULL_REDUCE_MAX_SPLITS, ceilDiv(flatItems, reduceWorkgroupSize)))",flatScratchBytes:'flatSplitCount * dtypeBytes("float32")',flatPathFits:"treeWorkgroupOk and flatSplitCount <= device.limits.maxComputeWorkgroupsPerDimension and flatScratchBytes <= device.limits.maxStorageBufferBindingSize and flatScratchBytes <= device.limits.maxBufferSize",flatParallelCovered:'(dtypes.T == "f32" or dtypes.T == "f16") and f16Ok(dtypes.T) and numel(shapes.y) == 1 and numel(shapes.x) >= tunables.FULL_REDUCE_MIN_ELEMENTS and flatPathFits',contiguousSuffixParallelCovered:'(dtypes.T == "f32" or dtypes.T == "f16") and f16Ok(dtypes.T) and numel(shapes.y) > 0 and numel(shapes.x) % numel(shapes.y) == 0 and numel(shapes.x) / numel(shapes.y) >= tunables.CONTIGUOUS_SUFFIX_MIN_COLS and ((ranks.x == 3 and hasAxis(attrs.axes, 0, 3) == false and hasAxis(attrs.axes, 1, 3) and hasAxis(attrs.axes, 2, 3) and numel(shapes.y) == dim(shapes.x, 0)) or (ranks.x == 4 and hasAxis(attrs.axes, 0, 4) == false and hasAxis(attrs.axes, 1, 4) == false and hasAxis(attrs.axes, 2, 4) and hasAxis(attrs.axes, 3, 4) and numel(shapes.y) == dim(shapes.x, 0) * dim(shapes.x, 1)) or (ranks.x == 4 and hasAxis(attrs.axes, 0, 4) == false and hasAxis(attrs.axes, 1, 4) and hasAxis(attrs.axes, 2, 4) and hasAxis(attrs.axes, 3, 4) and numel(shapes.y) == dim(shapes.x, 0)))',op:"reduceOp",castF32:'dtypes.T == "f16"',packedColumnWidth:'2 if dtypes.T == "f16" else 4',packedAxisPreferred:'(dtypes.T == "f32" and variableSubgroup16To32) or (not device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize > 0 and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32)'},bindings:{y:{elementType:"$T"},x_t:{name:"x",elementType:"$T"},params_axis0:{name:"params",struct:[{name:"rows",type:"u32",value:"dim(shapes.x, 0)"},{name:"cols",type:"u32",value:"dim(shapes.x, 1)"},{name:"outCount",type:"u32",value:"numel(shapes.y)"}]}},variants:[{id:"axis0",priority:0,when:["not flatParallelCovered","f16Ok(dtypes.T)","ranks.x == 2","reduceAxis == 0","((attrs.keepdims == 0 and ranks.y == 1 and dim(shapes.y, 0) == dim(shapes.x, 1)) or (attrs.keepdims == 1 and ranks.y == 2 and dim(shapes.y, 0) == 1 and dim(shapes.y, 1) == dim(shapes.x, 1)))"],derive:{axis:0,scalar:"dtypes.T"},passes:[{id:"main",name:"ReduceSum.Axis0",shader:"ops/_shared/reduce-serial-axis.wgsl.jinja",derive:{indexing:'"axis2d"',intMode:'dtypes.T == "i32"'},bindings:["x_t","y","params_axis0"],dispatch:{threads:"numel(shapes.y)",workgroupSize:"reduceWorkgroupSize"}}]}]},assets:[["ops/_shared/reduce-serial-axis.wgsl.jinja",`{% set yv = "" %}{% set vy = "" %}{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if op == "logsum" %}{% include "ops/_shared/reduce-log-of-sum.0bd3391873.wgsl.jinja" %}{% endif %}{% include "ops/_shared/reduce-minmax-identity.a93b328a6b.wgsl.jinja" %}{% if not intMode and (op == "logsum" or op == "logsumexp") %}{% include "ops/_shared/float-neg-inf.29b2a70cbe.wgsl.jinja" %}{% endif %}{% if op == "max" or op == "min" %}{{ wgsl_minmax_identity("empty_value", op, "f32" if castF32 else scalar) }}
{% elif op == "logsumexp" %}const F32_MIN:f32=-3.4028234663852886e38;const F32_MAX:f32=3.4028234663852886e38;
{% include "ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja" %}{% endif %}{% set count = "params.rows" %}{% set at = "x[r * params.cols + i]" %}{% set at_first = "x[i]" %}@compute@workgroup_size({{ reduceWorkgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(reduceWorkgroupSize, bound="params.outCount") }}
{% if op == "logsumexp" %}if({{ count }}==0u){y[i]={{ yv }}negative_infinity(){{ vy }};return;}var m=F32_MIN;var has_nan=false;var nan_value=0.0;for(var r=0u;r<{{ count }};r=r+1u){let value={{ at }};if(is_nan_f32(value)){has_nan=true;nan_value=value;}else{m=max(m,value);} }if(has_nan){y[i]={{ yv }}nan_value{{ vy }};return;}if(m>F32_MAX){y[i]={{ yv }}m{{ vy }};return;}var acc=0.0;for(var r=0u;r<{{ count }};r=r+1u){acc=acc+exp({{ at }}-m);}y[i]={{ yv }}m+log(acc){{ vy }};
{% elif op == "max" or op == "min" %}if({{ count }}==0u){y[i]={{ yv }}empty_value(){{ vy }};return;}var acc={{ at_first }};for(var r=1u;r<{{ count }};r=r+1u){acc={{ op }}(acc,{{ at }});}y[i]={{ yv }}acc{{ vy }};
{% else %}{% if op == "mean" %}if({{ count }}==0u){y[i]={{ yv }}0.0{{ vy }};return;}
{% elif op == "logsum" %}if({{ count }}==0u){y[i]={{ yv }}negative_infinity(){{ vy }};return;}
{% endif %}var acc={{ "1.0" if op == "prod" else "0.0" }};for(var r=0u;r<{{ count }};r=r+1u){
{% if op == "prod" %}acc=acc*{{ at }};
{% elif op == "l1" %}acc=acc+abs({{ at }});
{% elif op == "l2" or op == "sumsquare" %}let value={{ at }};acc=acc+value*value;
{% else %}acc=acc+{{ at }};
{% endif %}}
{% if op == "l2" %}y[i]={{ yv }}sqrt(acc){{ vy }};
{% elif op == "logsum" %}y[i]={{ yv }}log_of_sum(acc){{ vy }};
{% elif op == "mean" %}y[i]={{ yv }}acc/f32({{ count }}){{ vy }};
{% else %}y[i]={{ yv }}acc{{ vy }};
{% endif %}{% endif %}}`]],assumes:[["ops/_shared/reduce-serial-axis.wgsl.jinja",{op:"sum",castF32:!1,scalar:"f32",intMode:!1}]]}],["ai.onnx.ScatterElements",{manifest:{domain:"ai.onnx",name:"ScatterElements",sinceVersion:18,inputs:{data:{dtype:"T"},indices:{dtype:"I"},updates:{dtype:"T"}},outputs:{output:{dtype:"T",rank:"ranks.data",shape:"shapes.data"}},attributes:{axis:{default:0},reduction:{default:"none"}},attributeConstraints:{reduction:{values:["none","add","mul","max","min"]}},typeConstraints:{T:["float32","float16","int32","uint32","int8","uint8","int16","bool","int64"],I:["int32"]},tunables:{WORKGROUP_SIZE:{default:256},ATOMIC_WORKGROUP_SIZE:{default:64},F16_ATOMIC_MIN_AXIS:{default:1024},F16_ATOMIC_MAX_SLICES:{default:2048},SUBGROUP_PREAGG_ROUNDS:{default:2},F16_SLAB_MIN_OUTER:{default:8},F16_SLAB_MIN_CELLS:{default:32},F16_SLAB_MAX_SLABS:{default:4},F16_ATOMIC_WORKGROUP_SIZE:{default:256},ATOMIC_UPDATES_PER_LANE:{default:2},HISTOGRAM_TARGET_WORKGROUPS:{default:256},OWNED_ROW_MAX_AXIS:{default:16},ORDERED_GLOBAL_MAX_BYTES:{default:2097152},ORDERED_GLOBAL_LOCAL_CHAIN:{default:16},ORDERED_GLOBAL_MAX_FALLBACK_STEPS:{default:268435456}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",storageBufferLimit:"min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',reportedNonWave32Adapter:'not wave32Adapter and (has(device.adapterInfo, "subgroupMinSize") or has(device.adapterInfo, "subgroupMaxSize"))',variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',narrowIntData:'tensorDtypes.data == "int8" or tensorDtypes.data == "uint8" or tensorDtypes.data == "int16"',indexScalar:"dtypes.I",copyElementType:'("vec4<" ~ dtypes.T ~ ">") if dtypes.T != "vec2<u32>" and numel(shapes.output) % 4 == 0 else dtypes.T',reductionOp:"attrs.reduction",narrowBits:'8 if tensorDtypes.data == "int8" or tensorDtypes.data == "uint8" else (16 if tensorDtypes.data == "int16" else 0)',narrowSigned:'tensorDtypes.data == "int8" or tensorDtypes.data == "int16"',boolData:'tensorDtypes.data == "bool"',dataShape:"shapes.data",updatesShape:"shapes.updates",axis:"attrs.axis if attrs.axis >= 0 else attrs.axis + ranks.data",updateSlices:"numel(shapes.updates) / dim(shapes.updates, axis) if dim(shapes.updates, axis) > 0 else 0",workgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",atomicWorkgroupSize:"min(tunables.ATOMIC_WORKGROUP_SIZE, deviceWorkgroupCap)",f16AtomicWorkgroupSize:"min(tunables.F16_ATOMIC_WORKGROUP_SIZE, deviceWorkgroupCap)",bindingsFit:"numel(shapes.data) * dtypeBytes(dtypes.T) <= storageBufferLimit and numel(shapes.indices) * dtypeBytes(dtypes.I) <= storageBufferLimit and numel(shapes.updates) * dtypeBytes(dtypes.T) <= storageBufferLimit and numel(shapes.output) * dtypeBytes(dtypes.T) <= storageBufferLimit",dispatchFits:"max(numel(shapes.output), numel(shapes.updates)) <= device.limits.maxComputeWorkgroupsPerDimension * device.limits.maxComputeWorkgroupsPerDimension * workgroupSize",f16AccumFits:'numel(shapes.output) * dtypeBytes("uint32") <= storageBufferLimit',validLayout:"workgroupSize > 0 and bindingsFit and dispatchFits and ranks.data == ranks.output and ranks.indices == ranks.updates and ranks.data >= 1 and ranks.indices == ranks.data and attrs.axis + ranks.data >= 0 and attrs.axis < ranks.data and numel(shapes.data) == numel(shapes.output) and numel(shapes.indices) == numel(shapes.updates)",slabInner:"inner(shapes.data, axis)",slabOuter:"outer(shapes.data, axis)",slabCells:"dim(shapes.data, axis) * slabInner",slabBytes:'slabCells * dtypeBytes("uint32")',slabWorkgroupSize:"min(workgroupSize, pow2ceil(max(slabCells, tunables.F16_SLAB_MIN_CELLS)))",slabsPerWorkgroup:"max(1, min(tunables.F16_SLAB_MAX_SLABS, floor(device.limits.maxComputeWorkgroupStorageSize / max(slabBytes, 4))))",slabVec4:"slabCells % 4 == 0",slabUpdatesMatchData:"sameShape(prefix(shapes.updates, axis), prefix(shapes.data, axis)) and sameShape(suffix(shapes.updates, axis + 1), suffix(shapes.data, axis + 1))",slabElement:'"vec4<f16>" if slabVec4 else "f16"',histogramColumnLanes:32,histogramRowLanes:8,histogramColumnGroups:"ceilDiv(max(dim(shapes.data, 1), 1), histogramColumnLanes)",histogramStripeCap:"max(1, floor(dim(shapes.updates, 0) / (4 * max(dim(shapes.data, 0), 1))))",histogramStripes:"max(1, min(ceilDiv(tunables.HISTOGRAM_TARGET_WORKGROUPS, histogramColumnGroups), histogramStripeCap, ceilDiv(dim(shapes.updates, 0), histogramRowLanes)))",updatesPerLane:"tunables.ATOMIC_UPDATES_PER_LANE",rank:"ranks.data",axisSpec:"axis",orderedSlabUpdates:"dim(shapes.updates, axis) * slabInner",orderedSlabHeadWords:"ceilDiv(slabCells, 2)",orderedSlabBytes:"ceilDiv(orderedSlabHeadWords * 4, 16) * 16 + ceilDiv(max(orderedSlabUpdates, 1) * 4, 16) * 16",orderedChunkWorkgroupSize:"workgroupSize / 2 if updateSlices >= workgroupSize and workgroupSize >= 2 else workgroupSize",orderedStageCells:"min(dim(shapes.data, axis), max(1, floor(device.limits.maxComputeWorkgroupStorageSize / 4)))",orderedStagesPerColumn:"ceilDiv(dim(shapes.data, axis), orderedStageCells) if dim(shapes.data, axis) > 0 else 0",orderedGlobalChainCapacity:"min(max(1, floor(tunables.ORDERED_GLOBAL_LOCAL_CHAIN)), device.adapterInfo.subgroupMinSize) if variableSubgroup16To32 else 1",orderedGlobalFallbackSteps:"floor(dim(shapes.updates, axis) / (orderedGlobalChainCapacity + 1)) * dim(shapes.updates, axis)",orderedGlobalBytesPerColumn:"4 * (dim(shapes.data, axis) + dim(shapes.updates, axis))",orderedGlobalColumns:"min(updateSlices, floor(min(tunables.ORDERED_GLOBAL_MAX_BYTES, device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize) / max(1, orderedGlobalBytesPerColumn)), floor(tunables.ORDERED_GLOBAL_MAX_FALLBACK_STEPS / max(1, orderedGlobalFallbackSteps)))",orderedGlobalHeadCells:"orderedGlobalColumns * dim(shapes.data, axis)",orderedGlobalLinkCells:"orderedGlobalColumns * dim(shapes.updates, axis)"},bindings:{indices:{elementType:"$indexScalar"},params_scatter:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.updates)"},{name:"axisDim",type:"u32",value:"dim(shapes.data, axis)"}]},src:{arg:"data",elementType:"$copyElementType"},dst:{arg:"output",elementType:"$copyElementType"},updates_scatter:{name:"updates",elementType:"$updateElement"},output_generic:{name:"output",elementType:"$outputElement"}},when:['dtypes.T != "vec2<u32>" or attrs.reduction == "none"'],variants:[{id:"generic",priority:0,when:['attrs.reduction == "none"',"validLayout","f16Ok(dtypes.T)"],derive:{updateElement:"dtypes.T",outputElement:"dtypes.T"},passes:[{id:"copy",name:"ScatterElements.Copy",shader:"ops/_shared/scatter-flat-copy.wgsl.jinja",derive:{count:'numel(shapes.output) / 4 if dtypes.T != "vec2<u32>" and numel(shapes.output) % 4 == 0 else numel(shapes.output)'},bindings:["src","dst"],dispatch:{threads:'numel(shapes.output) / 4 if dtypes.T != "vec2<u32>" and numel(shapes.output) % 4 == 0 else numel(shapes.output)',workgroupSize:"workgroupSize"}},{id:"scatter",name:"ScatterElements.Scatter",shader:"scatter-elements.wgsl.jinja",bindings:["indices","updates_scatter","output_generic","params_scatter"],dispatch:{threads:"numel(shapes.updates)",workgroupSize:"workgroupSize"}}]}]},assets:[["ops/_shared/scatter-flat-copy.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set wg = workgroupSize if workgroupSize is defined else tunables.WORKGROUP_SIZE %}const COUNT:u32={{ count }}u;const WG:u32={{ wg }}u;@compute@workgroup_size({{ wg }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WG", bound="COUNT") }} dst[i]=src[i];}`],["scatter-elements.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const INVALID_INDEX:u32=0xffffffffu;
{% include "ops/_shared/normalize-index.fbcdd05a27.wgsl.jinja" %}{% include "ops/_shared/scatter-output-offset.620f3e6760.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize) }} let gathered=normalize_index(indices[i],params.axisDim);if(gathered!=INVALID_INDEX){output[output_offset(i,gathered)]=updates[i];} }`]],assumes:[["scatter-elements.wgsl.jinja",{indexScalar:"i32",rank:1}]]}],["ai.onnx.Sigmoid",{manifest:{domain:"ai.onnx",name:"Sigmoid",sinceVersion:13,inputs:{x:{onnx:"X",dtype:"T"}},outputs:{y:{onnx:"Y",dtype:"T",rank:"ranks.x",shape:"shapes.x"}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_layout_vec4",priority:20,when:["numel(shapes.x) > 0","numel(shapes.x) % 4 == 0","numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Sigmoid.Vec4",shader:"ops/_shared/unary-vec4.wgsl.jinja",derive:{op:'"sigmoid"',vec4PerThread:"4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1"},bindings:["x","y","params_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y) / 4, 4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"scalar",when:["numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Sigmoid",shader:"ops/_shared/unary-scalar.wgsl.jinja",derive:{op:'"sigmoid"',itemsPerInvocation:4},bindings:["x_scalar","y_scalar","params_2_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{x:{buffer:"read-only-storage",elementType:"$vectorScalar"},y:{buffer:"storage",elementType:"$vectorScalar"},params_unary:{struct:[{name:"count",type:"u32",value:"numel(shapes.y) / 4"}],name:"params"},x_scalar:{buffer:"read-only-storage",name:"x",elementType:"$scalar"},y_scalar:{buffer:"storage",name:"y",elementType:"$scalar"},params_2_unary:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}}},assets:[["ops/_shared/unary-vec4.wgsl.jinja",`{% macro unary_vec4_map(fn, value="fv") -%}vec4<f32>({{ fn }}({{ value }}.x),{{ fn }}({{ value }}.y),{{ fn }}({{ value }}.z),{{ fn }}({{ value }}.w))
{%- endmacro %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.1773d907a8.wgsl.jinja" %}{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}  let xv = x[i];
let fv=vec4<f32>(xv);y[i]={{ vectorScalar }}({{ unary_vec4_map("sigmoid_safe") }});
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/unary-scalar.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.1773d907a8.wgsl.jinja" %}{{ flat_tail_open() }}
y[i]={{ scalar }}(sigmoid_safe(f32(x[i])));{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/unary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>"}],["ops/_shared/unary-scalar.wgsl.jinja",{scalar:"f32"}]]}],["ai.onnx.Sin",{manifest:{domain:"ai.onnx",name:"Sin",sinceVersion:7,inputs:{x:{onnx:"input",dtype:"T"}},outputs:{y:{onnx:"output",dtype:"T",rank:"ranks.x",shape:"shapes.x"}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:256}},variants:[{id:"same_layout_vec4",priority:20,when:["numel(shapes.x) > 0","numel(shapes.x) % 4 == 0","numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Sin.Vec4",shader:"ops/_shared/unary-vec4.wgsl.jinja",derive:{op:'"sin"',vec4PerThread:"4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1"},bindings:["x","y","params_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y) / 4, 4 if numel(shapes.y) * dtypeBytes(tensorDtypes.y) <= 16777216 else 1)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"elementwise",when:["numel(shapes.x) == numel(shapes.y)","f16Ok(dtypes.T)"],derive:{scalar:"dtypes.T"},passes:[{id:"main",name:"Sin",shader:"ops/_shared/unary-scalar.wgsl.jinja",derive:{op:'"sin"',itemsPerInvocation:4},bindings:["x_scalar","y_scalar","params_2_unary"],dispatch:{threads:"ceilDiv(numel(shapes.y), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}],bindings:{x:{buffer:"read-only-storage",elementType:"$vectorScalar"},y:{buffer:"storage",elementType:"$vectorScalar"},params_unary:{struct:[{name:"count",type:"u32",value:"numel(shapes.y) / 4"}],name:"params"},x_scalar:{buffer:"read-only-storage",name:"x",elementType:"$scalar"},y_scalar:{buffer:"storage",name:"y",elementType:"$scalar"},params_2_unary:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}}},assets:[["ops/_shared/unary-vec4.wgsl.jinja",`{% macro unary_vec4_map(fn, value="fv") -%}vec4<f32>({{ fn }}({{ value }}.x),{{ fn }}({{ value }}.y),{{ fn }}({{ value }}.z),{{ fn }}({{ value }}.w))
{%- endmacro %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.142e982e4e.wgsl.jinja" %}{% include "ops/_shared/elementwise-vec4-loop.139ad6daac.wgsl.jinja" %}  let xv = x[i];
let fv=vec4<f32>(xv);y[i]={{ vectorScalar }}({{ unary_vec4_map("sin_accurate") }});
{% include "ops/_shared/elementwise-vec4-loop-close.5d7a074a4b.wgsl.jinja" %}}`],["ops/_shared/unary-scalar.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/unary-numeric-helpers.c06e05473e.wgsl.jinja" %}{{ flat_tail_open() }}
y[i]={{ scalar }}(sin_accurate(f32(x[i])));{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/unary-vec4.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>"}],["ops/_shared/unary-scalar.wgsl.jinja",{scalar:"f32"}]]}],["ai.onnx.Transpose",{manifest:{domain:"ai.onnx",name:"Transpose",sinceVersion:13,inputs:{x:{onnx:"data",dtype:"T"}},outputs:{y:{onnx:"transposed",dtype:"T",rank:"ranks.x"}},attributes:{perm:{}},typeConstraints:{T:["float32","float16","int32","int16","uint32","uint8","int8","bool","int64"]},tunables:{WORKGROUP_SIZE:{default:256},INNER_VEC4_ITEMS_PER_THREAD:{default:4},INNER_VEC4_MIN_THREADS:{default:65536}},derive:{variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',tileSize:32,dtypeSupported:"f16Ok(dtypes.T)",rank2IO:"ranks.x == 2 and ranks.y == 2",rank3IO:"ranks.x == 3 and ranks.y == 3",rank4IO:"ranks.x == 4 and ranks.y == 4",rank5IO:"ranks.x == 5 and ranks.y == 5",permPresent:'has(attrs, "perm")',permContract:"not permPresent or ((attrs.perm | length) == ranks.x and isUniqueIntList(attrs.perm, 0, ranks.x))",explicitRank2:"rank2IO and permContract and permPresent and dim(shapes.y, 0) == dim(shapes.x, attrs.perm[0]) and dim(shapes.y, 1) == dim(shapes.x, attrs.perm[1])",explicitRank3:"rank3IO and permContract and permPresent and dim(shapes.y, 0) == dim(shapes.x, attrs.perm[0]) and dim(shapes.y, 1) == dim(shapes.x, attrs.perm[1]) and dim(shapes.y, 2) == dim(shapes.x, attrs.perm[2])",explicitRank4:"rank4IO and permContract and permPresent and dim(shapes.y, 0) == dim(shapes.x, attrs.perm[0]) and dim(shapes.y, 1) == dim(shapes.x, attrs.perm[1]) and dim(shapes.y, 2) == dim(shapes.x, attrs.perm[2]) and dim(shapes.y, 3) == dim(shapes.x, attrs.perm[3])",explicitRank5:"rank5IO and permContract and permPresent and dim(shapes.y, 0) == dim(shapes.x, attrs.perm[0]) and dim(shapes.y, 1) == dim(shapes.x, attrs.perm[1]) and dim(shapes.y, 2) == dim(shapes.x, attrs.perm[2]) and dim(shapes.y, 3) == dim(shapes.x, attrs.perm[3]) and dim(shapes.y, 4) == dim(shapes.x, attrs.perm[4])",defaultRank2:"rank2IO and permContract and not permPresent and dim(shapes.y, 0) == dim(shapes.x, 1) and dim(shapes.y, 1) == dim(shapes.x, 0)",defaultPerm:"reversedAxes(ranks.x)",genericPerm:"pick([[permPresent, attrs.perm]], defaultPerm)",genericTransposeOk:"ranks.x == ranks.y and permContract and sameShape(shapes.y, permuteShape(shapes.x, genericPerm))",innerVec4Rank3Ok:"explicitRank3 and attrs.perm[2] == 2 and dim(shapes.x, 2) % 4 == 0 and numel(shapes.y) > 0 and dtypeSupported",innerVec4Rank4Ok:"explicitRank4 and attrs.perm[3] == 3 and dim(shapes.x, 3) % 4 == 0 and numel(shapes.y) > 0 and dtypeSupported",innerVec4Ok:"innerVec4Rank3Ok or innerVec4Rank4Ok",innerVec4Perm:"attrs.perm",tiled2dShapeOk:"(defaultRank2 or (explicitRank2 and attrs.perm[0] == 1 and attrs.perm[1] == 0)) and numel(shapes.x) > 0 and ceil(dim(shapes.x, 0) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 1) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled2dOk:"tiled2dShapeOk and dim(shapes.x, 0) % 4 == 0 and dim(shapes.x, 1) % 4 == 0",tiled3dShapeOk:"explicitRank3 and attrs.perm[0] == 0 and attrs.perm[1] == 2 and attrs.perm[2] == 1 and numel(shapes.x) > 0 and dim(shapes.x, 0) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 1) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 2) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled3dOk:"tiled3dShapeOk and dim(shapes.x, 1) % 4 == 0 and dim(shapes.x, 2) % 4 == 0",tiled4dShapeOk:"explicitRank4 and attrs.perm[0] == 0 and attrs.perm[1] == 1 and attrs.perm[2] == 3 and attrs.perm[3] == 2 and numel(shapes.x) > 0 and dim(shapes.x, 0) * dim(shapes.x, 1) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 2) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 3) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled4dOk:"tiled4dShapeOk and dim(shapes.x, 2) % 4 == 0 and dim(shapes.x, 3) % 4 == 0",tiled5dNcdhwToNdhwcShapeOk:"explicitRank5 and attrs.perm[0] == 0 and attrs.perm[1] == 2 and attrs.perm[2] == 3 and attrs.perm[3] == 4 and attrs.perm[4] == 1 and numel(shapes.x) > 0 and dim(shapes.x, 0) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 1) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil((dim(shapes.x, 2) * dim(shapes.x, 3) * dim(shapes.x, 4)) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled5dNcdhwToNdhwcOk:"tiled5dNcdhwToNdhwcShapeOk and dim(shapes.x, 1) % 4 == 0 and (dim(shapes.x, 2) * dim(shapes.x, 3) * dim(shapes.x, 4)) % 4 == 0",tiled5dNdhwcToNcdhwShapeOk:"explicitRank5 and attrs.perm[0] == 0 and attrs.perm[1] == 4 and attrs.perm[2] == 1 and attrs.perm[3] == 2 and attrs.perm[4] == 3 and numel(shapes.x) > 0 and dim(shapes.x, 0) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 1) * dim(shapes.x, 2) * dim(shapes.x, 3) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 4) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled5dNdhwcToNcdhwOk:"tiled5dNdhwcToNcdhwShapeOk and dim(shapes.x, 1) * dim(shapes.x, 2) * dim(shapes.x, 3) % 4 == 0 and dim(shapes.x, 4) % 4 == 0",memcpyPerm3:"explicitRank3 and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[1]) == 1 or attrs.perm[0] < attrs.perm[1]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[0] < attrs.perm[2]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[1] < attrs.perm[2])",memcpyPerm4:"explicitRank4 and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[1]) == 1 or attrs.perm[0] < attrs.perm[1]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[0] < attrs.perm[2]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[0] < attrs.perm[3]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[1] < attrs.perm[2]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[1] < attrs.perm[3]) and (dim(shapes.x, attrs.perm[2]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[2] < attrs.perm[3])",memcpyPerm5:"explicitRank5 and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[1]) == 1 or attrs.perm[0] < attrs.perm[1]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[0] < attrs.perm[2]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[0] < attrs.perm[3]) and (dim(shapes.x, attrs.perm[0]) == 1 or dim(shapes.x, attrs.perm[4]) == 1 or attrs.perm[0] < attrs.perm[4]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[2]) == 1 or attrs.perm[1] < attrs.perm[2]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[1] < attrs.perm[3]) and (dim(shapes.x, attrs.perm[1]) == 1 or dim(shapes.x, attrs.perm[4]) == 1 or attrs.perm[1] < attrs.perm[4]) and (dim(shapes.x, attrs.perm[2]) == 1 or dim(shapes.x, attrs.perm[3]) == 1 or attrs.perm[2] < attrs.perm[3]) and (dim(shapes.x, attrs.perm[2]) == 1 or dim(shapes.x, attrs.perm[4]) == 1 or attrs.perm[2] < attrs.perm[4]) and (dim(shapes.x, attrs.perm[3]) == 1 or dim(shapes.x, attrs.perm[4]) == 1 or attrs.perm[3] < attrs.perm[4])",memcpyPermOk:"memcpyPerm3 or memcpyPerm4 or memcpyPerm5",identityPermutation:"genericTransposeOk and sameShape(genericPerm, reversedAxes(ranks.x) | reverse)",identityCopyOk:"numel(shapes.y) > 0 and numel(shapes.y) % 4 == 0 and (identityPermutation or memcpyPermOk) and dtypeSupported",tiled4dNchwToNhwcShapeOk:"explicitRank4 and attrs.perm[0] == 0 and attrs.perm[1] == 2 and attrs.perm[2] == 3 and attrs.perm[3] == 1 and numel(shapes.x) > 0 and dim(shapes.x, 0) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 1) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil((dim(shapes.x, 2) * dim(shapes.x, 3)) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled4dNchwToNhwcOk:"tiled4dNchwToNhwcShapeOk and dim(shapes.x, 1) % 4 == 0 and (dim(shapes.x, 2) * dim(shapes.x, 3)) % 4 == 0",tiled4dNhwcToNchwShapeOk:"explicitRank4 and attrs.perm[0] == 0 and attrs.perm[1] == 3 and attrs.perm[2] == 1 and attrs.perm[3] == 2 and numel(shapes.x) > 0 and dim(shapes.x, 0) <= device.limits.maxComputeWorkgroupsPerDimension and ceil((dim(shapes.x, 1) * dim(shapes.x, 2)) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(dim(shapes.x, 3) / tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and dtypeSupported",tiled4dNhwcToNchwOk:"tiled4dNhwcToNchwShapeOk and (dim(shapes.x, 1) * dim(shapes.x, 2)) % 4 == 0 and dim(shapes.x, 3) % 4 == 0",tiledTransposeOk:"tiled2dOk or tiled3dOk or tiled4dOk or tiled5dNcdhwToNdhwcOk or tiled5dNdhwcToNcdhwOk or tiled4dNchwToNhwcOk or tiled4dNhwcToNchwOk",tiledScalarTransposeOk:"tiled2dShapeOk or tiled3dShapeOk or tiled4dShapeOk or tiled5dNcdhwToNdhwcShapeOk or tiled5dNdhwcToNcdhwShapeOk or tiled4dNchwToNhwcShapeOk or tiled4dNhwcToNchwShapeOk",tiledRows:"dim(shapes.x, 0) if tiled2dShapeOk else dim(shapes.x, 1) if tiled3dShapeOk else dim(shapes.x, 2) if tiled4dShapeOk else dim(shapes.x, 1) if tiled5dNcdhwToNdhwcShapeOk else dim(shapes.x, 1) * dim(shapes.x, 2) * dim(shapes.x, 3) if tiled5dNdhwcToNcdhwShapeOk else dim(shapes.x, 1) if tiled4dNchwToNhwcShapeOk else dim(shapes.x, 1) * dim(shapes.x, 2) if tiled4dNhwcToNchwShapeOk else 1",tiledCols:"dim(shapes.x, 1) if tiled2dShapeOk else dim(shapes.x, 2) if tiled3dShapeOk else dim(shapes.x, 3) if tiled4dShapeOk else dim(shapes.x, 2) * dim(shapes.x, 3) * dim(shapes.x, 4) if tiled5dNcdhwToNdhwcShapeOk else dim(shapes.x, 4) if tiled5dNdhwcToNcdhwShapeOk else dim(shapes.x, 2) * dim(shapes.x, 3) if tiled4dNchwToNhwcShapeOk else dim(shapes.x, 3) if tiled4dNhwcToNchwShapeOk else 1",tiledBatch:"1 if tiled2dShapeOk else (dim(shapes.x, 0) if tiled3dShapeOk else (dim(shapes.x, 0) * dim(shapes.x, 1) if tiled4dShapeOk else (dim(shapes.x, 0) if tiled5dNcdhwToNdhwcShapeOk or tiled5dNdhwcToNcdhwShapeOk or tiled4dNchwToNhwcShapeOk or tiled4dNhwcToNchwShapeOk else 1)))",stridedPlaneRank3:"rank3IO and genericTransposeOk and genericPerm[0] == 2 and genericPerm[1] == 1 and genericPerm[2] == 0",stridedPlaneRank4:"rank4IO and genericTransposeOk and genericPerm[0] == 0 and genericPerm[1] == 3 and genericPerm[2] == 2 and genericPerm[3] == 1",stridedPlaneRows:"dim(shapes.x, 0) if stridedPlaneRank3 else dim(shapes.x, 1) if stridedPlaneRank4 else 1",stridedPlaneCols:"dim(shapes.x, 2) if stridedPlaneRank3 else dim(shapes.x, 3) if stridedPlaneRank4 else 1",stridedPlaneMiddle:"dim(shapes.x, 1) if stridedPlaneRank3 else dim(shapes.x, 2) if stridedPlaneRank4 else 1",stridedPlaneBatch:"1 if stridedPlaneRank3 else dim(shapes.x, 0) if stridedPlaneRank4 else 1",stridedPlaneGroups:"stridedPlaneBatch * stridedPlaneMiddle",stridedPlaneOk:'(stridedPlaneRank3 or stridedPlaneRank4) and numel(shapes.x) > 0 and dtypeSupported and ceilDiv(stridedPlaneRows, tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(stridedPlaneCols, tileSize) <= device.limits.maxComputeWorkgroupsPerDimension and stridedPlaneGroups <= device.limits.maxComputeWorkgroupsPerDimension and device.limits.maxComputeInvocationsPerWorkgroup >= tileSize * tileSize / 4 and device.limits.maxComputeWorkgroupStorageSize >= tileSize * (tileSize + 1) * (2 if dtypes.T == "f16" else 4)',stridedScalarTileSize:"min(tileSize, pow2ceil(min(stridedPlaneRows, stridedPlaneCols)))",stridedScalarRowStep:"min(8, stridedScalarTileSize)",stridedPlaneSparseTile:"ceilDiv(stridedPlaneRows, tileSize) * ceilDiv(stridedPlaneCols, tileSize) * tileSize * tileSize > 2 * stridedPlaneRows * stridedPlaneCols",scalar:"dtypes.T",vectorScalar:'"vec4<" ~ dtypes.T ~ ">"',inputShape:"shapes.x",outputShape:"shapes.y",rank:"ranks.x"},bindings:{src:{arg:"x",elementType:"$vectorScalar"},dst:{arg:"y",elementType:"$vectorScalar"},x:{elementType:"$scalar"},y:{elementType:"$scalar"}},variants:[{id:"tiled_vec4",priority:22,when:['dtypes.T != "vec2<u32>"',"tiledTransposeOk"],passes:[{id:"main",name:"Transpose.Tiled",shader:"ops/_shared/datamove-transpose-2d-tiled.wgsl.jinja",derive:{rowCount:"tiledRows",cols:"tiledCols",transposeCooperativeRows:"variableSubgroup16To32"},bindings:["src","dst"],dispatch:{x:"ceil(tiledCols / tileSize)",y:"ceil(tiledRows / tileSize)",z:"tiledBatch"}}]},{id:"tiled_scalar",priority:19,when:['dtypes.T != "vec2<u32>"',"tiledScalarTransposeOk","dtypeSupported"],passes:[{id:"main",name:"Transpose.TiledScalar",shader:"datamove-transpose-2d-tiled-scalar.wgsl.jinja",derive:{rowCount:"tiledRows",cols:"tiledCols",tileRowStep:'2 if (has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 64) else 8'},bindings:["x","y"],dispatch:{x:"ceil(tiledCols / tileSize)",y:"ceil(tiledRows / tileSize)",z:"tiledBatch"}}]},{id:"generic",when:["genericTransposeOk","dtypeSupported"],passes:[{id:"main",name:"Transpose",shader:"transpose.wgsl.jinja",derive:{perm:"genericPerm",itemsPerInvocation:4},bindings:["x","y",{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.y)"}]}],dispatch:{threads:"ceilDiv(numel(shapes.y), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["ops/_shared/datamove-transpose-2d-tiled.wgsl.jinja",`{% set rowStep = (tileSize / 4)|int if (transposeCooperativeRows | default(false)) and tileSize >= 32 else tileSize %}{% if (device.adapterInfo.subgroupMinSize | default(0)) >= 64 and tileSize > 16 and 256 % tileSize == 0 %}{% set rowStep = 8 %}{% endif %}{% set tileRow = "lid.y + i" if rowStep < tileSize else "lid.y" %}{{ env.wgsl.resourceDeclarations }} const ROWS:u32={{ rowCount }}u;const COLS:u32={{ cols }}u;const ROWS_V:u32={{ ((rowCount / 4) | int) }}u;const COLS_V:u32={{ ((cols / 4) | int) }}u;const MAT_V:u32={{ (((rowCount * cols * (transposePlaneMiddle | default(1))) / 4) | int) }}u;const TILE:u32={{ tileSize }}u;var<workgroup>tile:array<array<{{ scalar }},{{ tileSize + 1 }}>,{{ tileSize }}>;@compute@workgroup_size({{ (tileSize / 4) | int }},{{ rowStep }},1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let batch_base=wid.z*MAT_V;let row_base=wid.y*TILE;let col_base=wid.x*TILE;
{% if rowStep < tileSize %}for(var i=0u;i<TILE;i=i+{{ rowStep }}u){
{% endif %}{% filter indent(2 if rowStep < tileSize else 0, true) %}let r=row_base+{{ tileRow }};let cv=(col_base/4u)+lid.x;if(r<ROWS&&cv<COLS_V){let v=src[batch_base+r*COLS_V+cv];tile[{{ tileRow }}][lid.x*4u+0u]=v.x;tile[{{ tileRow }}][lid.x*4u+1u]=v.y;tile[{{ tileRow }}][lid.x*4u+2u]=v.z;tile[{{ tileRow }}][lid.x*4u+3u]=v.w;}
{% endfilter %}{% if rowStep < tileSize %}}
{% endif %}workgroupBarrier();
{% if rowStep < tileSize %}for(var i=0u;i<TILE;i=i+{{ rowStep }}u){
{% endif %}{% filter indent(2 if rowStep < tileSize else 0, true) %}let out_row=col_base+{{ tileRow }};let rv=(row_base/4u)+lid.x;if(out_row<COLS&&rv<ROWS_V){dst[batch_base+out_row*ROWS_V+rv]={{ vectorScalar }}(tile[lid.x*4u+0u][{{ tileRow }}],tile[lid.x*4u+1u][{{ tileRow }}],tile[lid.x*4u+2u][{{ tileRow }}],tile[lid.x*4u+3u][{{ tileRow }}]);}
{% endfilter %}{% if rowStep < tileSize %}}
{% endif %}}`],["datamove-transpose-2d-tiled-scalar.wgsl.jinja",'{{ env.wgsl.resourceDeclarations }} const ROWS:u32={{ rowCount }}u;const COLS:u32={{ cols }}u;const MAT:u32={{ rowCount * cols }}u;const TILE:u32={{ tileSize }}u;const ROW_STEP:u32={{ tileRowStep }}u;var<workgroup>tile:array<array<{{ scalar }},{{ tileSize + 1 }}>,{{ tileSize }}>;@compute@workgroup_size({{ tileSize }},{{ tileRowStep }},1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let batch_base=wid.z*MAT;let row_base=wid.y*TILE;let col_base=wid.x*TILE;for(var i=0u;i<TILE;i=i+ROW_STEP){let r=row_base+lid.y+i;let c=col_base+lid.x;if(r<ROWS&&c<COLS){tile[lid.y+i][lid.x]=x[batch_base+{{ "(r * MIDDLE + middle)" if transposePlaneMiddle is defined else "r" }}*COLS+c];} }workgroupBarrier();for(var i=0u;i<TILE;i=i+ROW_STEP){let out_row=col_base+lid.y+i;let out_col=row_base+lid.x;if(out_row<COLS&&out_col<ROWS){y[batch_base+{{ "(out_row * MIDDLE + middle)" if transposePlaneMiddle is defined else "out_row" }}*ROWS+out_col]=tile[lid.x][lid.y+i];} } }'],["transpose.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
fn input_offset(out_index:u32)->u32{var rem=out_index;var offset=0u;
{% for out_axis in range(rank) %}{% set out_stride = namespace(value=1) %}{% for j in range(out_axis + 1, rank) %}{% set out_stride.value = out_stride.value * outputShape[j] %}{% endfor %}{% set out_stride_divisor = out_stride.value if out_stride.value != 0 else 1 %}{% set in_axis = perm[out_axis] %}{% set in_stride = namespace(value=1) %}{% for j in range(in_axis + 1, rank) %}{% set in_stride.value = in_stride.value * inputShape[j] %}{% endfor %}let coord{{ out_axis }}=rem/{{ out_stride_divisor }}u;rem=rem% {{ out_stride_divisor }}u;offset=offset+coord{{ out_axis }}*{{ in_stride.value }}u;
{% endfor %}return offset;} {{ flat_tail_open() }} y[i]=x[input_offset(i)];{{ flat_tail_close() }} }`]],assumes:[["ops/_shared/datamove-transpose-2d-tiled.wgsl.jinja",{scalar:"f32",vectorScalar:"vec4<f32>"}],["datamove-transpose-2d-tiled-scalar.wgsl.jinja",{scalar:"f32"}],["transpose.wgsl.jinja",{rank:2}]]}],["ai.onnx.Where",{manifest:{domain:"ai.onnx",name:"Where",sinceVersion:16,inputs:{condition:{dtype:"C"},x:{onnx:"X",dtype:"T"},y:{onnx:"Y",dtype:"T"}},outputs:{output:{dtype:"T",rank:"max(max(ranks.condition, ranks.x), ranks.y)",shape:"broadcastShape(broadcastShape(shapes.condition, shapes.x), shapes.y)"}},typeConstraints:{C:["bool"],T:["float32","float16","uint32","int32","int16","uint8","int8","bool","int64"]},tunables:{WORKGROUP_SIZE:{default:256}},derive:{scalar:"dtypes.T",cShape:"shapes.condition",xShape:"shapes.x",yShape:"shapes.y",oShape:"shapes.output",cRank:"ranks.condition",xRank:"ranks.x",yRank:"ranks.y",oRank:"ranks.output"},when:["f16Ok(dtypes.T)"],bindings:{condition:{elementType:"u32"},output_main:{name:"output",elementType:"$vectorScalar"},params_main:{name:"params",struct:[{name:"count",type:"u32",value:"numel(shapes.output) / 4"}]},condition_main:{name:"condition",elementType:"vec4<u32>"},x_main:{name:"x",elementType:"$vectorScalar"},y_main:{name:"y",elementType:"$vectorScalar"}},variants:[{id:"broadcast",when:["ranks.condition <= ranks.output","ranks.x <= ranks.output","ranks.y <= ranks.output"],derive:{vec4Out:'dtypes.T != "vec2<u32>" and ranks.output >= 1 and numel(shapes.output) > 0 and dim(shapes.output, ranks.output - 1) % 4 == 0 and (ranks.condition == 0 or dim(shapes.condition, ranks.condition - 1) == 1 or dim(shapes.condition, ranks.condition - 1) == dim(shapes.output, ranks.output - 1)) and (ranks.x == 0 or dim(shapes.x, ranks.x - 1) == 1 or dim(shapes.x, ranks.x - 1) == dim(shapes.output, ranks.output - 1)) and (ranks.y == 0 or dim(shapes.y, ranks.y - 1) == 1 or dim(shapes.y, ranks.y - 1) == dim(shapes.output, ranks.output - 1))',vectorScalar:'"vec4<" ~ dtypes.T ~ ">" if vec4Out else dtypes.T',outCount:"numel(shapes.output) / 4 if vec4Out else numel(shapes.output)"},passes:[{id:"main",name:"Where",shader:"where-broadcast.wgsl.jinja",derive:{itemsPerInvocation:4},bindings:["condition","x","y","output_main",{name:"params",struct:[{name:"count",type:"u32",value:"outCount"}]}],dispatch:{threads:"ceilDiv(numel(shapes.output), 4)",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"all_same_shape_vec4",priority:30,when:['dtypes.T != "vec2<u32>" and (sameShape(shapes.condition, shapes.output))',"sameShape(shapes.x, shapes.output)","sameShape(shapes.y, shapes.output)","numel(shapes.output) > 0","numel(shapes.output) % 4 == 0"],derive:{vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Where.Vec4",shader:"where-vec4.wgsl.jinja",derive:{condMode:'"vector"'},bindings:["condition_main","x_main","y_main","output_main","params_main"],dispatch:{threads:"numel(shapes.output) / 4",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"scalar_condition_vec4",priority:25,when:['dtypes.T != "vec2<u32>" and (numel(shapes.condition) == 1)',"sameShape(shapes.x, shapes.output)","sameShape(shapes.y, shapes.output)","numel(shapes.output) > 0","numel(shapes.output) % 4 == 0"],derive:{vectorScalar:'"vec4<" ~ dtypes.T ~ ">"'},passes:[{id:"main",name:"Where.ScalarCondVec4",shader:"where-vec4.wgsl.jinja",derive:{condMode:'"scalar"'},bindings:["condition","x_main","y_main","output_main","params_main"],dispatch:{threads:"numel(shapes.output) / 4",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["where-broadcast.wgsl.jinja",`{% include "ops/_shared/elementwise-tail-loop.71bf384cac.wgsl.jinja" %}{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/broadcast-offset.87f1ae294f.wgsl.jinja" %}{{ broadcast_offset_fn("c_offset", cShape, cRank, oShape, oRank) }}
{{ broadcast_offset_fn("x_offset", xShape, xRank, oShape, oRank) }} {{ broadcast_offset_fn("y_offset", yShape, yRank, oShape, oRank) }}
{% if vec4Out %}{% set o_last = oShape[oRank - 1] %}{% set c_inner = cRank > 0 and cShape[cRank - 1] == o_last %}{% set x_inner = xRank > 0 and xShape[xRank - 1] == o_last %}{% set y_inner = yRank > 0 and yShape[yRank - 1] == o_last %}@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE, "vi") }} let base=vi*4u;let cb={{ broadcast_offset_call("c_offset", cShape, oShape, "base") }};let xb={{ broadcast_offset_call("x_offset", xShape, oShape, "base") }};let yb={{ broadcast_offset_call("y_offset", yShape, oShape, "base") }};
{% if c_inner %}let cond=vec4<u32>(condition[cb],condition[cb+1u],condition[cb+2u],condition[cb+3u]);
{% else %}let cond=vec4<u32>(condition[cb]);
{% endif %}{% if x_inner %}let xv=vec4<{{ scalar }}>(x[xb],x[xb+1u],x[xb+2u],x[xb+3u]);
{% else %}let xv=vec4<{{ scalar }}>(x[xb]);
{% endif %}{% if y_inner %}let yv=vec4<{{ scalar }}>(y[yb],y[yb+1u],y[yb+2u],y[yb+3u]);
{% else %}let yv=vec4<{{ scalar }}>(y[yb]);
{% endif %}output[vi]=select(yv,xv,cond!=vec4<u32>(0u));}
{% else %}{{ flat_tail_open() }} output[i]=select(y[{{ broadcast_offset_call("y_offset", yShape, oShape, "i") }}],x[{{ broadcast_offset_call("x_offset", xShape, oShape, "i") }}],condition[{{ broadcast_offset_call("c_offset", cShape, oShape, "i") }}]!=0u);{{ flat_tail_close() }} }
{% endif %}`],["where-vec4.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(tunables.WORKGROUP_SIZE) }}
{% if condMode == "scalar" %}output[i]=select(y[i],x[i],condition[0]!=0u);
{% else %}output[i]=select(y[i],x[i],condition[i]!=vec4<u32>(0u));
{% endif %}}`]]}],["co.huggingface.GatedMlpActivation",{manifest:{domain:"co.huggingface",name:"GatedMlpActivation",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X",rank:2,optional:!0},gateT:{onnx:"Gate",dtype:"T",rank:2,optional:!0},upT:{onnx:"Up",dtype:"T",rank:2,optional:!0},geluLutT:{onnx:"GeluLut",dtype:"L",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:2,shape:["args.rows","args.dstStride if args.dstStride is defined else args.mlpInner"]}},args:{rows:{kind:"u32"},mlpInner:{kind:"u32"},srcStride:{kind:"u32",required:!1},srcStart:{kind:"u32",required:!1},dstStride:{kind:"u32",required:!1},dstStart:{kind:"u32",required:!1},activation:{kind:"string",required:!1,oneOf:["silu","fast_silu","gelu_tanh"]},split:{kind:"string",required:!1,oneOf:["chunk","interleaved"]},clampLimit:{kind:"f32",required:!1},alpha:{kind:"f32",required:!1},upBias:{kind:"f32",required:!1},gateScale:{kind:"f32",onnx:"input_scale",required:!1}},typeConstraints:{X:["float32","float16"],Y:["float32","float16"],T:["float32"],L:["float32"]},derive:{packedSource:'present.xT and not present.gateT and not present.upT and not present.geluLutT and args.srcStride is not defined and args.srcStart is not defined and args.dstStride is not defined and args.dstStart is not defined and args.gateScale is not defined and (args.activation is not defined or args.activation == "silu" or args.activation == "fast_silu")',stridedDestinationOk:"(args.dstStride is not defined and args.dstStart is not defined) or (args.dstStride is defined and args.dstStart is defined and args.dstStart + args.mlpInner <= args.dstStride)",stridedSource:'present.xT and not present.gateT and not present.upT and not present.geluLutT and args.srcStride is defined and args.srcStart is defined and args.gateScale is not defined and (args.activation is not defined or args.activation == "silu") and args.split is not defined and args.clampLimit is not defined and args.alpha is not defined and args.upBias is not defined',separateSource:'not present.xT and present.gateT and present.upT and present.geluLutT and args.srcStride is not defined and args.srcStart is not defined and args.dstStride is not defined and args.dstStart is not defined and args.activation == "gelu_tanh" and args.split is not defined and args.clampLimit is not defined and args.alpha is not defined and args.upBias is not defined',interleavedSplit:'args.split == "interleaved"',gluAlpha:"args.alpha if args.alpha is defined else 1",gluUpBias:"args.upBias if args.upBias is defined else 0",gluClamp:"args.clampLimit if args.clampLimit is defined else 0",hasClampArg:"args.clampLimit is defined",generalizedGlu:"interleavedSplit or hasClampArg or (gluAlpha != 1) or (gluUpBias != 0)"},when:["ranks.yT == 2","dim(shapes.yT, 0) == args.rows","dim(shapes.yT, 1) == (args.dstStride if args.dstStride is defined else args.mlpInner)"],bindings:{x:{arg:"xT",elementType:"$xElement"},y:{arg:"yT",elementType:"$yElement"},params:{struct:[{name:"rows",type:"u32"}]}},variants:[{id:"scalar",priority:0,when:["packedSource","ranks.xT == 2","args.rows == dim(shapes.xT, 0)","dim(shapes.xT, 1) == args.mlpInner * 2","f16Ok(dtypes.X) and f16Ok(dtypes.Y)"],tunables:{WG:{default:256}},derive:{vectorized:!1,fastSilu:'args.activation == "fast_silu"',interleaved:"interleavedSplit",generalized:"generalizedGlu",hasClamp:"hasClampArg",clampLimit:"gluClamp",alpha:"gluAlpha",upBias:"gluUpBias",xElement:"dtypes.X",yElement:"dtypes.Y",yScalar:"dtypes.Y",mlpInner:"args.mlpInner",wg:"tunables.WG"},passes:[{id:"main",name:"GatedMlpActivation",shader:"gated-mlp-activation.wgsl.jinja",bindings:["x","y","params"],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil(args.mlpInner / tunables.WG)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]},{id:"vec4",priority:10,when:["packedSource","dtypes.X == dtypes.Y","f16Ok(dtypes.X)","args.mlpInner % 4 == 0","ranks.xT == 2","args.rows == dim(shapes.xT, 0)","dim(shapes.xT, 1) == args.mlpInner * 2"],tunables:{WG:{default:256}},derive:{vectorized:!0,fastSilu:'args.activation == "fast_silu"',generalized:"generalizedGlu",hasClamp:"hasClampArg",clampLimit:"gluClamp",alpha:"gluAlpha",upBias:"gluUpBias",xElement:'"vec4<" ~ dtypes.X ~ ">"',yElement:'"vec4<" ~ dtypes.Y ~ ">"',yScalar:"dtypes.Y",mlpInnerV4:"args.mlpInner / 4",wg:"tunables.WG",interleaved:"interleavedSplit"},passes:[{id:"main",name:"GatedMlpActivation",shader:"gated-mlp-activation.wgsl.jinja",bindings:["x","y","params"],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil((args.mlpInner / 4) / tunables.WG)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]}]},assets:[["gated-mlp-activation.wgsl.jinja",`{% include "ops/_shared/swiglu-value.ebb7c81802.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if vectorized %}const MLP_V4:u32={{ mlpInnerV4 }}u;
{% else %}const MLP:u32={{ mlpInner }}u;
{% endif %}const WG:u32={{ wg }}u;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let r=wg.x+wg.z*{{ DISPATCH_FOLD_WIDTH }}u;if(r>=params.rows){return;}let i0=wg.y*WG+lid.x;
{% if vectorized %}if(i0>=MLP_V4){return;}let row_base=r*2u*MLP_V4;let x1=vec4<f32>(x[row_base+i0]);let x2=vec4<f32>(x[row_base+MLP_V4+i0]);y[r*MLP_V4+i0]={{ yElement }}(({{ swiglu_gate("x1", "vec4<f32>(1.0)", "vec4<f32>(1.0)") }})*x2);
{% else %}if(i0>=MLP){return;}let x1=f32(x[r*2u*MLP+i0]);let x2=f32(x[r*2u*MLP+MLP+i0]);y[r*MLP+i0]={{ yScalar }}(({{ swiglu_gate("x1", "1.0", "1.0") }})*x2);
{% endif %}}`]],assumes:[["gated-mlp-activation.wgsl.jinja",{fastSilu:!1,interleaved:!1,generalized:!1,hasClamp:!1,clampLimit:0,alpha:1,upBias:0,yScalar:"f32",wg:256}]]}],["co.huggingface.RMSNorm",{manifest:{domain:"co.huggingface",name:"RMSNorm",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X"},wT:{onnx:"W",dtype:"W",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:"ranks.xT",shape:"shapes.xT"}},args:{rows:{kind:"u32"},dim:{kind:"u32"},eps:{kind:"f32",required:!1},weightBaseOffset:{kind:"u32",onnx:"weight_base_offset",required:!1},weightOffset:{kind:"u32",onnx:"weight_offset",required:!1},castBeforeScale:{kind:"bool",onnx:"cast_before_scale",required:!1},exact:{kind:"u32",onnx:"exact_reference_order",required:!1}},typeConstraints:{X:["float32","float16"],W:["float32","float16","uint32"],Y:["float32","float16"]},when:['not present.wT or tensorDtypes.wT != "uint32" or args.weightBaseOffset is defined'],tunables:{WG_CAP:{default:1024},ROW_WG:{default:64}},derive:{subgroupsWide32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32',normWgCap:"tunables.WG_CAP if subgroupsWide32 else min(128, tunables.WG_CAP)",normWg:"min(normWgCap, max(32, pow2ceil(args.dim / 8)))",hasSubgroupId:'device.wgslLanguageFeatures.has("subgroup_id")',weightPlusOne:"args.weightOffset == 1",applyCastBeforeScale:'(args.castBeforeScale == true) and present.wT and (dtypes.Y != "f32")',arenaMode:"args.weightBaseOffset is defined and (not args.exact) and (not args.weightOffset) and (not args.castBeforeScale) and present.wT and args.dim > 0",arenaWeightElements:'(dim(shapes.wT, 0) * (2 if tensorDtypes.wT == "uint32" else 1)) if present.wT else 0',arenaDecodeContract:'arenaMode and ranks.xT == 1 and ranks.wT == 1 and ranks.yT == 1 and tensorDtypes.xT == "float32" and (tensorDtypes.wT == "float32" or tensorDtypes.wT == "float16" or tensorDtypes.wT == "uint32") and f16Ok(dtypes.W) and f16Ok(dtypes.Y) and args.rows == 1 and args.dim % 4 == 0 and args.weightBaseOffset % 4 == 0 and dim(shapes.xT, 0) == args.dim and dim(shapes.yT, 0) == args.dim and arenaWeightElements >= args.weightBaseOffset + args.dim',arenaRowsContract:'arenaMode and ranks.xT == 2 and ranks.wT == 1 and ranks.yT == 2 and (tensorDtypes.xT == "float32" or tensorDtypes.xT == "float16") and (tensorDtypes.wT == "float32" or tensorDtypes.wT == "float16" or tensorDtypes.wT == "uint32") and f16Ok(dtypes.X) and f16Ok(dtypes.W) and f16Ok(dtypes.Y) and args.rows > 0 and numel(shapes.xT) >= args.rows * args.dim and arenaWeightElements >= args.weightBaseOffset + args.dim and numel(shapes.yT) >= args.rows * args.dim'},bindings:{x:{arg:"xT",elementType:"$xScalar"},y:{arg:"yT",elementType:"$yScalar"},params_main:{name:"params",struct:[{name:"rows",type:"u32"},{name:"rowStride",type:"u32",value:"min(args.rows, device.limits.maxComputeWorkgroupsPerDimension)"}]}},variants:[{id:"arena_decode",priority:40,when:["arenaDecodeContract"],derive:{weightVec4:'"vec4<f16>" if tensorDtypes.wT == "float16" else "vec2<u32>" if tensorDtypes.wT == "uint32" else "vec4<f32>"',normedVec4:'"vec4<f16>" if tensorDtypes.yT == "float16" else "vec4<f32>"',hiddenSize:"args.dim",hiddenSizeVec4:"args.dim / 4",rmsEps:"args.eps if args.eps is defined else 0.000001",normOffsetVec4:"args.weightBaseOffset / 4",HIDDEN_VEC4_LEN:"args.dim / 4",ARENA_W_VEC4_LEN:"(args.weightBaseOffset + args.dim) / 4",workgroupSize:256},passes:[{id:"main",name:"LlamaDecodeRmsNorm",shader:"rms-norm-arena-decode.wgsl.jinja",bindings:[{arg:"xT",name:"hidden",elementType:"vec4<f32>",length:"$HIDDEN_VEC4_LEN"},{arg:"wT",name:"weights",elementType:"$weightVec4",length:"$ARENA_W_VEC4_LEN"},{arg:"yT",name:"normed",elementType:"$normedVec4",length:"$HIDDEN_VEC4_LEN"}],dispatch:{x:1}}]},{id:"arena_rows",priority:40,when:["arenaRowsContract"],derive:{xScalar:"dtypes.X",wScalar:"dtypes.W",yScalar:"dtypes.Y",dimSize:"args.dim",workgroupSize:256},passes:[{id:"main",name:"LlamaPrefillRmsNorm",shader:"rms-norm-arena-rows.wgsl.jinja",bindings:["x",{arg:"wT",name:"weights",elementType:"$wScalar"},"y",{name:"params",struct:[{name:"norm_offset",type:"u32",value:"args.weightBaseOffset"},{name:"rows",type:"u32"},{name:"eps",type:"f32",value:"args.eps if args.eps is defined else 0.000001"}]}],dispatch:{workgroups:"args.rows"}}]},{id:"unweighted",priority:0,when:["args.weightBaseOffset is undefined and (not args.exact) and (not present.wT and args.rows >= 0 and args.dim > 0 and numel(shapes.xT) >= args.rows * args.dim and numel(shapes.yT) >= args.rows * args.dim and f16Ok(dtypes.X) and f16Ok(dtypes.Y))"],derive:{hasWeight:!1,weightPlusOne:!1,castBeforeScale:!1,xScalar:"dtypes.X",yScalar:"dtypes.Y",dimSize:"args.dim",eps:"args.eps if args.eps is defined else 0.000001",rowWg:"tunables.ROW_WG"},passes:[{id:"main",name:"RMSNorm",shader:"rms-norm.wgsl.jinja",bindings:["x","y","params_main"],dispatch:{workgroups:"args.rows"}}]}]},assets:[["rms-norm-arena-decode.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.629ddaa87a.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;fn sum4(value:vec4<f32>)->f32{return value.x+value.y+value.z+value.w;}fn load_weight(value:{{ weightVec4 }})->vec4<f32>{return {{ "unpack_bf16_x4" if weightVec4 == "vec2<u32>" else "vec4<f32>" }}(value);}fn store_normed(value:vec4<f32>)->{{ normedVec4 }} {return {{ normedVec4 }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var sum=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let value=hidden[dim];sum=sum+sum4(value*value);}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"]) }} let variance=partials[0];let scale=inverseSqrt(variance/f32(HIDDEN_SIZE)+RMS_EPS);for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){normed[dim]=store_normed(hidden[dim]*vec4<f32>(scale)*load_weight(weights[W_NORM_VEC4+dim]));} }`],["rms-norm-arena-rows.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.629ddaa87a.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if wScalar == "u32" %}{% include "ops/_shared/bf16.721558b02a.wgsl.jinja" %}{% include "ops/_shared/bf16.3b861ea8df.wgsl.jinja" %}{% endif %}const DIM:u32={{ dimSize }}u;const WG:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let row=wid.x+wid.y*{{ DISPATCH_FOLD_WIDTH }}u;if(row>=params.rows){return;}let lid=lid3.x;let base=row*DIM;var sum=0.0;for(var d=lid;d<DIM;d=d+WG){let v=f32(x[base+d]);sum=sum+v*v;}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"], wg="WG") }} let scale=inverseSqrt(partials[0]/f32(DIM)+params.eps);for(var d=lid;d<DIM;d=d+WG){
{% if wScalar == "u32" %}let w=load_bf16_word(weights[(params.norm_offset+d)>>1u],params.norm_offset+d);y[base+d]={{ yScalar }}(f32(x[base+d])*scale*w);
{% else %}y[base+d]={{ yScalar }}(f32(x[base+d])*scale*f32(weights[params.norm_offset+d]));
{% endif %}} }`],["rms-norm.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const DIM:u32={{ dimSize }}u;const EPS:f32={{ eps }};const WG:u32={{ rowWg }}u;var<workgroup>partial:array<f32,WG>;
{% include "ops/_shared/reduce-sum.fdec0f92c8.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let rowStride=select(params.rowStride,params.rows,params.rowStride==0u);let row=wg.x+wg.y*rowStride;if(row>=params.rows){return;}let tid=lid.x;let base=row*DIM;var acc:f32=0.0;var i:u32=tid;loop{if(i>=DIM){break;}let v=f32(x[base+i]);acc=acc+v*v;i=i+WG;}let scale=inverseSqrt(reduce_sum(acc,tid)/f32(DIM)+EPS);var j:u32=tid;loop{if(j>=DIM){break;}let xv=f32(x[base+j]);y[base+j]={{ yScalar }}(xv*scale);j=j+WG;} }`]],assumes:[["rms-norm-arena-decode.wgsl.jinja",{weightVec4:"vec4<f32>",normedVec4:"vec4<f32>"}],["rms-norm-arena-rows.wgsl.jinja",{yScalar:"f32"}],["rms-norm.wgsl.jinja",{yScalar:"f32",rowWg:64}]]}],["com.xenova.Attention",{manifest:{domain:"com.xenova",name:"Attention",sinceVersion:1,inputs:{qT:{onnx:"Q",dtype:"T",optional:!0},kT:{onnx:"K",dtype:"T"},vT:{onnx:"V",dtype:"T"},maskKT:{onnx:"MaskK",dtype:"T",rank:1,optional:!0},posT:{onnx:"Pos",dtype:"T",rank:1,optional:!0},thetaT:{onnx:"Theta",dtype:"T",rank:1,optional:!0},qPartT:{onnx:"QPart",dtype:"T",rank:2,optional:!0},qbT:{onnx:"QB",dtype:"T",rank:1,optional:!0},relKT:{onnx:"RelK",dtype:"T",rank:2,optional:!0},relVT:{onnx:"RelV",dtype:"T",rank:2,optional:!0},maskT:{onnx:"Mask",dtype:"T",rank:1,optional:!0}},outputs:{outT:{onnx:"Out",dtype:"U",rank:"2 if crossMode else ranks.qT",shape:"[args.batch * args.seqQ, args.heads * args.headDim] if crossMode else shapes.qT"}},args:{seq:{kind:"u32",required:!1},qHeads:{kind:"u32",onnx:"q_heads",required:!1},kvHeads:{kind:"u32",onnx:"kv_heads",required:!1},headDim:{kind:"u32",onnx:"head_dim"},qTile:{kind:"u32",onnx:"query_tile",required:!1},kTile:{kind:"u32",onnx:"key_tile",required:!1},scale:{kind:"f32",required:!1},causal:{kind:"bool",required:!1},keyLimit:{kind:"u32",onnx:"key_limit",required:!1},useSubgroups:{kind:"bool",onnx:"use_subgroups",required:!1},useHalfQk:{kind:"bool",onnx:"use_half_qk",required:!1},batch:{kind:"u32",required:!1},seqQ:{kind:"u32",required:!1},seqKv:{kind:"u32",required:!1},heads:{kind:"u32",required:!1},splitKIn:{kind:"u32",required:!1},window:{kind:"u32",required:!1}},typeConstraints:{T:["float32","float16"],U:["float32","float16"]},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupMatrixTopologyBytes:"3 * 16",relativePositionMode:"present.relKT or present.relVT or present.maskT or args.window is defined",crossMode:"not relativePositionMode and (args.batch is defined or args.seqQ is defined or args.seqKv is defined or args.heads is defined or args.splitKIn is defined or present.maskKT or present.posT or present.thetaT or present.qPartT or present.qbT)",genericMode:"not relativePositionMode and not crossMode and present.qT and args.seq is defined and args.qHeads is defined and args.kvHeads is defined",callerOwnedParams:"genericMode and present.params",relativePositionContract:'relativePositionMode and present.qT and present.relKT and present.relVT and present.maskT and args.batch is defined and args.seq is defined and args.heads is defined and args.window is defined and args.scale is defined and args.seqQ is undefined and args.seqKv is undefined and args.qHeads is undefined and args.kvHeads is undefined and args.qTile is undefined and args.kTile is undefined and args.causal is undefined and args.keyLimit is undefined and args.useSubgroups is undefined and args.useHalfQk is undefined and args.splitKIn is undefined and not present.maskKT and not present.posT and not present.thetaT and not present.qPartT and not present.qbT and tensorDtypes.qT == "float32" and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and tensorDtypes.relKT == "float32" and tensorDtypes.relVT == "float32" and tensorDtypes.maskT == "float32" and tensorDtypes.outT == "float32"',crossGeometryMode:"crossMode and args.batch is defined and args.seqQ is defined and args.seqKv is defined and args.heads is defined and args.scale is defined and args.seq is undefined and args.qHeads is undefined and args.kvHeads is undefined and args.qTile is undefined and args.kTile is undefined and args.causal is undefined and args.keyLimit is undefined and args.useSubgroups is undefined and args.useHalfQk is undefined",crossDirectQMode:"crossGeometryMode and present.qT and not present.qPartT and not present.qbT and args.splitKIn is undefined",crossQPartMode:"crossGeometryMode and not present.qT and present.qPartT and present.qbT and args.splitKIn is defined",crossTanhKDirectMode:"crossDirectQMode and not present.maskKT and not present.posT and not present.thetaT",crossTanhKQPartMode:"crossQPartMode and not present.maskKT and not present.posT and not present.thetaT",crossKeyMaskDirectMode:"crossDirectQMode and present.maskKT and not present.posT and not present.thetaT",crossKeyMaskRotaryDirectMode:"crossDirectQMode and present.maskKT and present.posT and present.thetaT",crossKeyMaskRotaryQPartMode:"crossQPartMode and present.maskKT and present.posT and present.thetaT",crossDtypeContract:'crossMode and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and tensorDtypes.outT == "float32" and (not present.qT or tensorDtypes.qT == "float32") and (not present.maskKT or tensorDtypes.maskKT == "float32") and (not present.posT or tensorDtypes.posT == "float32") and (not present.thetaT or tensorDtypes.thetaT == "float32") and (not present.qPartT or tensorDtypes.qPartT == "float32") and (not present.qbT or tensorDtypes.qbT == "float32")',explicitFlash:"genericMode and args.qTile is defined",explicitFullAttention:"genericMode and (args.causal is undefined or not args.causal) and (args.keyLimit is undefined or args.keyLimit == args.seq)",geometryOk:"genericMode and args.seq > 0 and args.qHeads > 0 and args.kvHeads > 0 and args.headDim > 0 and args.headDim % 4 == 0 and args.qHeads % args.kvHeads == 0",keyTileOk:"genericMode and (args.kTile == 32 or args.kTile == 64 or args.kTile == 128 or args.kTile == 256)",dtypeContractOk:'genericMode and tensorDtypes.qT == tensorDtypes.kT and tensorDtypes.qT == tensorDtypes.vT and (tensorDtypes.qT == "float32" or tensorDtypes.qT == "float16") and f16Ok(dtypes.T) and f16Ok(dtypes.U)',bufferContractOk:"genericMode and numel(shapes.qT) >= args.seq * args.qHeads * args.headDim and numel(shapes.kT) >= args.seq * args.kvHeads * args.headDim and numel(shapes.vT) >= args.seq * args.kvHeads * args.headDim and numel(shapes.outT) >= args.seq * args.qHeads * args.headDim",baseEligible:"genericMode and not callerOwnedParams and explicitFlash and explicitFullAttention and geometryOk and keyTileOk and dtypeContractOk and bufferContractOk",f16Contract:'genericMode and tensorDtypes.qT == "float16" and tensorDtypes.kT == "float16" and tensorDtypes.vT == "float16"',flashWideNoncausal:'genericMode and not callerOwnedParams and (args.causal is undefined or not args.causal) and tensorDtypes.qT == "float32" and args.headDim == 64 and args.qHeads == args.kvHeads and args.seq >= 512 and args.qHeads * ceil(args.seq / 64) >= 128',flashUseSubgroups:'device.features.has("subgroups") and args.headDim % 32 == 0',flashLanesPerQuery:"(4 if flashWideNoncausal else 8) if flashUseSubgroups else 2",flashQueryTile:"(64 if flashWideNoncausal else 16) if flashUseSubgroups else 64",flashKeyTile:'8 if flashWideNoncausal else ((32 if 2 * 32 * args.headDim * 4 <= device.limits.maxComputeWorkgroupStorageSize else 16) if genericMode and flashUseSubgroups and tensorDtypes.qT != "float16" and args.headDim <= 128 else 8)',flashWorkgroupSize:"flashQueryTile * flashLanesPerQuery",flashKvStorageBytes:'2 * flashKeyTile * args.headDim * (2 if tensorDtypes.qT == "float16" else 4)',flashReductionStorageBytes:"0 if flashUseSubgroups else flashKeyTile * flashWorkgroupSize * 4",flashStorageBytes:"flashKvStorageBytes + flashReductionStorageBytes",flashProfileKeys:"min(args.seq, args.keyLimit if args.keyLimit is defined else args.seq) if genericMode and not callerOwnedParams else 0",flashScorePairs:"flashProfileKeys * (flashProfileKeys + 1) / 2 + (args.seq - flashProfileKeys) * flashProfileKeys if genericMode and args.causal else (args.seq * flashProfileKeys if genericMode else 0)",flashResourcesFit:"genericMode and flashWorkgroupSize <= deviceWorkgroupCap and flashStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",baseContract:"crossGeometryMode and crossDtypeContract and args.batch > 0 and args.heads > 0 and args.seqKv <= 2048 and numel(shapes.kT) >= args.batch * args.seqKv * args.heads * args.headDim and numel(shapes.vT) >= args.batch * args.seqKv * args.heads * args.headDim and present.outT and numel(shapes.outT) >= args.batch * args.seqQ * args.heads * args.headDim",crossFlashLanesPerQuery:8,crossFlashQueryTile:"16 if device.limits.maxComputeInvocationsPerWorkgroup >= 128 and device.limits.maxComputeWorkgroupSizeX >= 128 else 8",crossFlashWorkgroupSize:"crossFlashQueryTile * crossFlashLanesPerQuery",crossFlashHeadWords:"args.headDim / 4",crossFlashStagedBytesPerKey:"args.headDim * 8",crossFlashSubgroupTileK:"max(1, min(32, floor(device.limits.maxComputeWorkgroupStorageSize / crossFlashStagedBytesPerKey)))",crossFlashSubgroupStorageBytes:"crossFlashSubgroupTileK * crossFlashStagedBytesPerKey",crossFlashSubgroupClusterOk:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize >= crossFlashLanesPerQuery and device.adapterInfo.subgroupMinSize % crossFlashLanesPerQuery == 0 and device.adapterInfo.subgroupMaxSize % crossFlashLanesPerQuery == 0',crossFlashDispatchFits:"crossMode and ceilDiv(args.seqQ, crossFlashQueryTile) <= device.limits.maxComputeWorkgroupsPerDimension and args.heads <= device.limits.maxComputeWorkgroupsPerDimension and args.batch <= device.limits.maxComputeWorkgroupsPerDimension",crossFlashWorkgroupFits:"crossFlashWorkgroupSize <= deviceWorkgroupCap",materializedInnerTile:16,materializedQueryTile:64,materializedKeyTile:64,materializedApplyTile:64,materializedWorkgroupDim:16,materializedSoftmaxWorkgroupSize:128,materializedScoreBytes:"args.batch * args.heads * args.seqQ * args.seqKv * 4 if crossMode else 0",materializedScoreStorageBytes:"(materializedQueryTile * materializedInnerTile + materializedKeyTile * (materializedInnerTile + 4)) * 4",materializedApplyStorageBytes:"(materializedQueryTile * materializedInnerTile + materializedInnerTile * materializedApplyTile) * 4",materializedWorkgroupSize:"materializedWorkgroupDim * materializedWorkgroupDim",materializedParallelPreferred:'device.features.has("subgroups") or (has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32)',materializedBuffersFit:"materializedScoreBytes <= device.limits.maxStorageBufferBindingSize and materializedScoreBytes <= device.limits.maxBufferSize",materializedDeviceFits:"materializedInnerTile % 4 == 0 and (materializedApplyTile / materializedWorkgroupDim) % 4 == 0 and materializedWorkgroupSize <= device.limits.maxComputeInvocationsPerWorkgroup and materializedWorkgroupDim <= device.limits.maxComputeWorkgroupSizeX and materializedWorkgroupDim <= device.limits.maxComputeWorkgroupSizeY and materializedSoftmaxWorkgroupSize <= deviceWorkgroupCap and materializedScoreStorageBytes <= device.limits.maxComputeWorkgroupStorageSize and materializedApplyStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",materializedDispatchFits:"crossMode and ceilDiv(args.seqKv, materializedKeyTile) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(args.seqQ, materializedQueryTile) <= device.limits.maxComputeWorkgroupsPerDimension and args.batch * args.heads <= device.limits.maxComputeWorkgroupsPerDimension",headDim:"args.headDim",headDimV4:"args.headDim / 4",hasBias:!1,outputScalar:"dtypes.U",vLoad:'"vec4<f32>(v[vBaseV4 + d4])"',outputVec4:'"vec4<f16>" if tensorDtypes.outT == "float16" else "vec4<f32>"'},bindings:{params_key_limit:{name:"params",struct:[{name:"qSeq",type:"u32",value:"args.seq"},{name:"kvSeq",type:"u32",value:"args.seq"},{name:"qHeads",type:"u32"},{name:"kvHeads",type:"u32"},{name:"isCausal",type:"u32",value:"1 if args.causal else 0"},{name:"keyLimit",type:"u32",value:"args.keyLimit if args.keyLimit else 0"}]},query_scalar:{arg:"qT",name:"query",elementType:"$inputScalar"},key_scalar:{arg:"kT",name:"key",elementType:"$inputScalar"},value_scalar:{arg:"vT",name:"value",elementType:"$inputScalar"},output_scalar:{arg:"outT",name:"output",elementType:"$outputScalar"}},variants:[{id:"scalar",priority:0,when:["bufferContractOk","not explicitFlash","args.seq > 0","args.qHeads > 0","args.kvHeads > 0","args.headDim > 0","args.qHeads % args.kvHeads == 0","(args.keyLimit is undefined or args.keyLimit <= args.seq)","tensorDtypes.qT == tensorDtypes.kT","tensorDtypes.qT == tensorDtypes.vT",'(tensorDtypes.qT == "float32" or tensorDtypes.qT == "float16")',"f16Ok(dtypes.T) and f16Ok(dtypes.U)"],derive:{hasCausal:!0,hasKeyLimit:!0,headsFromParams:!0,scaleFallbackRsqrt:!1,inputScalar:"dtypes.T",workgroupSize:64,scale:"args.scale if args.scale else 1.0"},passes:[{id:"main",name:"Attention",shader:"ops/_shared/attn-online-scalar.wgsl.jinja",derive:{layout:'"shd"'},bindings:["query_scalar","key_scalar","value_scalar","output_scalar","params_key_limit"],dispatch:{x:"min(args.seq, device.limits.maxComputeWorkgroupsPerDimension)",y:"args.qHeads"}}]}]},assets:[["ops/_shared/attn-online-scalar.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HEAD_DIM:u32={{ headDim }}u;
{% set qHeads = "params.qHeads" %}{% set kvHeads = "params.kvHeads" %}const WG:u32={{ workgroupSize }}u;
{% include "ops/_shared/compensated-dot.bd8b60df37.wgsl.jinja" %}var<workgroup> partial: array<DotAccumulator, WG>;
var<workgroup>running_max:f32;var<workgroup>running_denom:f32;var<workgroup>running_out:array<f32,HEAD_DIM>;var<workgroup>previous_scale:f32;
{% include "ops/_shared/reduce-tree.2706db3b42.wgsl.jinja" %}{{ wgsl_tree_reduce_f32("reduce_dot", "compensated", "partial", "WG", valueType="DotAccumulator") }}
fn scale_value()->f32{return {{ scale }};}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let h=wg.y;if(h>={{ qHeads }}){return;}let tid=lid.x;let kv_head=h/({{ qHeads }}/{{ kvHeads }});for(var query_token=wg.x;query_token<params.qSeq;query_token=query_token+nwg.x){let qBase=(query_token*{{ qHeads }}+h)*HEAD_DIM;let kvBase=kv_head*HEAD_DIM;let kvTokenStride={{ kvHeads }}*HEAD_DIM;if(tid==0u){running_max=-3.4028234663852886e38;running_denom=0.0;}for(var d:u32=tid;d<HEAD_DIM;d=d+WG){running_out[d]=0.0;}workgroupBarrier();var maxKj=params.kvSeq;var minKj:u32=0u;maxKj=min(maxKj,select(params.kvSeq,query_token+1u,params.isCausal!=0u));maxKj=min(maxKj,select(params.kvSeq,params.keyLimit,params.keyLimit>0u));for(var key_token:u32=minKj;key_token<maxKj;key_token=key_token+1u){let kRow=kvBase+key_token*kvTokenStride;var partial_dot=DotAccumulator(0.0,0.0);for(var d:u32=tid;d<HEAD_DIM;d=d+WG){var q_value=f32(query[qBase+d]);var k_value=f32(key[kRow+d]);partial_dot=dot_accumulate(partial_dot,q_value,k_value);}let score=dot_value(reduce_dot(partial_dot,tid))*scale_value();if(tid==0u){let next_max=max(running_max,score);let old_factor=exp(running_max-next_max);let new_factor=exp(score-next_max);previous_scale=old_factor;running_max=next_max;running_denom=running_denom*old_factor+new_factor;}workgroupBarrier();let probability_numerator=exp(score-running_max);for(var d:u32=tid;d<HEAD_DIM;d=d+WG){var v_value=f32(value[kRow+d]);running_out[d]=running_out[d]*previous_scale+probability_numerator*v_value;}workgroupBarrier();}let inv_denom=select(0.0,1.0/running_denom,running_denom>0.0);for(var d:u32=tid;d<HEAD_DIM;d=d+WG){output[qBase+d]={{ outputScalar }}(running_out[d]*inv_denom);}workgroupBarrier();} }`]],assumes:[["ops/_shared/attn-online-scalar.wgsl.jinja",{outputScalar:"f32"}]]}],["com.xenova.Axpy",{manifest:{domain:"com.xenova",name:"Axpy",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X"},yT:{onnx:"Y",dtype:"Y"},gateT:{onnx:"gate",dtype:"G",optional:!0}},outputs:{xT:{onnx:"X",dtype:"X",rank:"ranks.xT",shape:"shapes.xT"}},args:{count:{kind:"u32",onnx:"kernel.count"},alpha:{kind:"f32"},rowLength:{kind:"u32",required:!1},gateLayout:{kind:"string",required:!1,oneOf:["row","column"]},gateOffset:{kind:"u32",required:!1}},attributes:{alpha:{default:1}},typeConstraints:{X:["float32","float16"],Y:["float32","float16"],G:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:64}},derive:{gated:"present.gateT",columnGate:'gated and args.gateLayout == "column"',rowLength:"args.rowLength if args.rowLength is defined else 0",gateOffset:"args.gateOffset if args.gateOffset is defined else 0",sizeOk:"numel(shapes.xT) >= args.count and numel(shapes.yT) >= args.count",dtypeOk:"f16Ok(dtypes.X) and f16Ok(dtypes.Y)",gateContract:"gated and f16Ok(dtypes.G) and rowLength > 0"},when:["args.count >= 0","sizeOk","dtypeOk"],bindings:{x:{arg:"xT",elementType:"$X"},y:{arg:"yT",elementType:"$Y"}},variants:[{id:"scalar",when:["not gated"],derive:{hasGate:!1},passes:[{id:"main",name:"scalar",shader:"axpy.wgsl.jinja",bindings:["x","y",{name:"params",struct:[{name:"count",type:"u32"},{name:"alpha",type:"f32"},{name:"wgY",type:"u32",value:"min(ceil(args.count / tunables.WORKGROUP_SIZE), device.limits.maxComputeWorkgroupsPerDimension)"}]}],dispatch:{threads:"args.count",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["axpy.wgsl.jinja",'{{ env.wgsl.resourceDeclarations }} const WG:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let wg_idx=wg.x+wg.y*params.wgY;let i=wg_idx*WG+lid.x;if(i>=params.count){return;}x[i]={{ "f16(f32(x[i]) + params.alpha * f32(y[i]))" if dtypes.X == "f16" else "f32(x[i]) + params.alpha * f32(y[i])" }};}']],assumes:[["axpy.wgsl.jinja",{columnGate:!1}]]}],["com.xenova.CachedDecodeAttention",{manifest:{domain:"com.xenova",name:"CachedDecodeAttention",sinceVersion:1,inputs:{qT:{onnx:"Q",dtype:"Q",rank:1,optional:!0},qkvT:{onnx:"Qkv",dtype:"Q",rank:2,optional:!0},gateT:{onnx:"Gate",dtype:"O",optional:!0},sinkT:{onnx:"Sinks",dtype:"S",rank:1,optional:!0},normWeightsT:{onnx:"NormWeights",dtype:"N",rank:1,optional:!0},cosT:{onnx:"RopeCos",dtype:"float32",optional:!0},sinT:{onnx:"RopeSin",dtype:"float32",optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1}},outputs:{attnT:{onnx:"Attn",dtype:"O",rank:"2 if present.qkvT else 1",shape:"[args.seqLen, args.numHeads * args.headDim] if present.qkvT else [args.numHeads * args.headDim]"},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0,shape:"shapes.cacheKeysT"},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0,shape:"shapes.cacheValuesT"}},args:{layer:{kind:"u32"},cacheLen:{kind:"u32",onnx:"cache_len"},pastLen:{kind:"u32",onnx:"past_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len",required:!1},numHeads:{kind:"u32",onnx:"num_heads"},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},headDim:{kind:"u32",onnx:"head_dim"},scale:{kind:"f32"},workgroupSize:{kind:"u32",onnx:"workgroup_size",required:!1},tileSize:{kind:"u32",onnx:"tile_size",required:!1},numKvChunks:{kind:"u32",onnx:"num_kv_chunks",required:!1},minKeysPerChunk:{kind:"u32",onnx:"min_keys_per_chunk",required:!1},qOffset:{kind:"u32",onnx:"q_offset",required:!1},kOffset:{kind:"u32",onnx:"k_offset",required:!1},vOffset:{kind:"u32",onnx:"v_offset",required:!1},windowSize:{kind:"u32",onnx:"window_size",required:!1},cacheRing:{kind:"u32",onnx:"cache_ring",required:!1},sinkOffset:{kind:"u32",onnx:"sink_offset",required:!1},qNormOffset:{kind:"u32",onnx:"q_norm_offset",required:!1},kNormOffset:{kind:"u32",onnx:"k_norm_offset",required:!1},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1}},typeConstraints:{Q:["float32","float16"],C:["float32","float16"],O:["float32","float16"],S:["float32","float16","uint32"],N:["float32","float16"]},tunables:{DECODE_SCORE_TILE:{default:1024},WORKGROUP_SIZE:{default:128},MAX_SCORE_TILE:{default:4096}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWide32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",subgroupMatrixTopologyBytes:"3 * 16",cacheWriteArgsPresent:"args.qOffset is defined and args.kOffset is defined and args.vOffset is defined",cacheWriteArgsAbsent:"args.qOffset is undefined and args.kOffset is undefined and args.vOffset is undefined",splitArgsPresent:"args.workgroupSize is defined and args.tileSize is defined and args.numKvChunks is defined and args.minKeysPerChunk is defined",splitArgsAbsent:"args.tileSize is undefined and args.numKvChunks is undefined and args.minKeysPerChunk is undefined",splitPartialOutBytes:"args.numHeads * args.numKvChunks * args.headDim * 4 if splitArgsPresent else 0",splitPartialMetaBytes:"args.numHeads * args.numKvChunks * 2 * 4 if splitArgsPresent else 0",splitScratchFits:"splitPartialOutBytes <= device.limits.maxStorageBufferBindingSize and splitPartialOutBytes <= device.limits.maxBufferSize and splitPartialMetaBytes <= device.limits.maxStorageBufferBindingSize and splitPartialMetaBytes <= device.limits.maxBufferSize",decodeMode:"present.qT and not present.qkvT and args.seqLen is undefined",prefillMode:"present.qkvT and not present.qT and args.seqLen is defined and cacheWriteArgsAbsent and splitArgsAbsent",coreShapeContract:'ranks.qT == 1 and (ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5) and (ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5) and ranks.attnT == 1 and (tensorDtypes.qT == "float32" or tensorDtypes.qT == "float16") and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and f16Ok(dtypes.Q) and f16Ok(dtypes.C) and f16Ok(dtypes.O) and args.cacheLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads % args.numKvHeads == 0 and args.headDim > 0 and dim(shapes.attnT, 0) == args.numHeads * args.headDim and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim',coreContract:"decodeMode and cacheWriteArgsAbsent and numel(shapes.qT) >= args.numHeads * args.headDim and coreShapeContract",qkAppendContract:'decodeMode and cacheWriteArgsPresent and args.qOffset == 0 and args.qNormOffset is defined and args.kNormOffset is defined and args.rmsEps is defined and present.normWeightsT and present.cosT and present.sinT and coreShapeContract and tensorDtypes.qT == "float32" and f16Ok(tensorDtypes.normWeightsT) and args.kOffset % 4 == 0 and args.vOffset % 4 == 0 and args.qNormOffset % 4 == 0 and args.kNormOffset % 4 == 0 and args.kOffset >= args.numHeads * args.headDim and args.vOffset >= args.kOffset + args.numKvHeads * args.headDim and numel(shapes.qT) >= args.vOffset + args.numKvHeads * args.headDim and dim(shapes.normWeightsT, 0) >= max(args.qNormOffset, args.kNormOffset) + args.headDim and numel(shapes.cosT) == numel(shapes.sinT) and numel(shapes.cosT) >= args.headDim / 2 and args.headDim % 8 == 0',qBytes:'numel(shapes.qT) * (2 if tensorDtypes.qT == "float16" else 4) if present.qT else 0',cacheKeysBytes:'numel(shapes.cacheKeysT) * (2 if tensorDtypes.cacheKeysT == "float16" else 4)',cacheValuesBytes:'numel(shapes.cacheValuesT) * (2 if tensorDtypes.cacheValuesT == "float16" else 4)',attnBytes:'numel(shapes.attnT) * (2 if tensorDtypes.attnT == "float16" else 4)',buffersFit:"qBytes <= device.limits.maxStorageBufferBindingSize and qBytes <= device.limits.maxBufferSize and cacheKeysBytes <= device.limits.maxStorageBufferBindingSize and cacheKeysBytes <= device.limits.maxBufferSize and cacheValuesBytes <= device.limits.maxStorageBufferBindingSize and cacheValuesBytes <= device.limits.maxBufferSize and attnBytes <= device.limits.maxStorageBufferBindingSize and attnBytes <= device.limits.maxBufferSize",workgroupLimit:"min(256, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",autoWorkgroupSize:"256 if args.headDim > 128 and workgroupLimit >= 256 else 128 if args.headDim > 64 and workgroupLimit >= 128 else 64 if args.headDim > 32 and workgroupLimit >= 64 else 32 if args.headDim > 16 and workgroupLimit >= 32 else 16 if args.headDim > 8 and workgroupLimit >= 16 else 8 if args.headDim > 4 and workgroupLimit >= 8 else 4 if workgroupLimit >= 4 else 0",decodeWorkgroupSize:"args.workgroupSize if args.workgroupSize else autoWorkgroupSize",attentionStorageBytes:"(args.cacheLen + decodeWorkgroupSize) * 4",workgroupSupported:"(decodeWorkgroupSize == 4 or decodeWorkgroupSize == 8 or decodeWorkgroupSize == 16 or decodeWorkgroupSize == 32 or decodeWorkgroupSize == 64 or decodeWorkgroupSize == 128 or decodeWorkgroupSize == 256)",resourcesFitCore:"workgroupSupported and decodeWorkgroupSize <= deviceWorkgroupCap",resourcesFit:"resourcesFitCore and attentionStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",dispatchFits:"args.numHeads <= device.limits.maxComputeWorkgroupsPerDimension",baseOk:"coreContract and buffersFit and resourcesFit and dispatchFits",baseOkCore:"coreContract and buffersFit and resourcesFitCore and dispatchFits",onlineScoreTile:"tunables.DECODE_SCORE_TILE",onlineStorageBytes:"(onlineScoreTile + decodeWorkgroupSize) * 4",onlineResourcesFit:"onlineScoreTile > 0 and pow2ceil(onlineScoreTile) == onlineScoreTile and onlineStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",sharedScoresCrowded:"attentionStorageBytes * 2 > device.limits.maxComputeWorkgroupStorageSize",onlineOk:"baseOkCore and onlineResourcesFit and sharedScoresCrowded",subgroupRangeKnown:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize > 0 and device.adapterInfo.subgroupMinSize <= device.adapterInfo.subgroupMaxSize',shortSubgroupOk:"baseOk and subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= decodeWorkgroupSize and decodeWorkgroupSize % device.adapterInfo.subgroupMinSize == 0 and decodeWorkgroupSize % device.adapterInfo.subgroupMaxSize == 0 and attentionStorageBytes + 8 <= device.limits.maxComputeWorkgroupStorageSize",sharedSplitUseSubgroups:"(subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= args.workgroupSize and args.workgroupSize % device.adapterInfo.subgroupMinSize == 0 and args.workgroupSize % device.adapterInfo.subgroupMaxSize == 0) if splitArgsPresent else false",sharedSplitStorageBytes:'((args.headDim) * 8 + (args.workgroupSize) * 12 + (16 * (args.workgroupSize) if (args.workgroupSize) % ((args.headDim) / 4) == 0 and (args.workgroupSize) / ((args.headDim) / 4) >= 2 else 0) + (4 * (args.workgroupSize) if (sharedSplitUseSubgroups) and ((args.headDim) / 4) >= 8 and not ((tensorDtypes.qT == "float16" or tensorDtypes.cacheKeysT == "float16") and ((args.headDim) / 4) <= 32) and (true) else 0) + (ceilDiv((args.workgroupSize), 32) * 4 + 8 if (sharedSplitUseSubgroups) else 0) if (args.headDim) >= 4 and (args.workgroupSize) > 0 else 0) if splitArgsPresent else 0',sharedSplitResourcesFit:"((args.headDim) >= 4 and (args.workgroupSize) > 0 and (args.workgroupSize) <= deviceWorkgroupCap and (pow2ceil(args.workgroupSize) == (args.workgroupSize) or (sharedSplitUseSubgroups)) and (sharedSplitStorageBytes) <= device.limits.maxComputeWorkgroupStorageSize) if splitArgsPresent else false",scalarWorkgroupLimit:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",scalarWorkgroupSize:"max(1, pow2ceil(scalarWorkgroupLimit + 1) / 2)",scalarScoreTileCapacity:"floor((device.limits.maxComputeWorkgroupStorageSize - scalarWorkgroupSize * 4 - 8) / 4)",scalarScoreTile:"max(1, min(args.cacheLen, tunables.MAX_SCORE_TILE, scalarScoreTileCapacity))",scalarDeviceOk:"tunables.WORKGROUP_SIZE > 0 and tunables.MAX_SCORE_TILE > 0 and scalarWorkgroupLimit > 0 and scalarScoreTileCapacity > 0",fusedWriteContract:'decodeMode and cacheWriteArgsPresent and splitArgsAbsent and not present.gateT and ranks.qT == 1 and ranks.cacheKeysT == 1 and ranks.cacheValuesT == 1 and ranks.attnT == 1 and tensorDtypes.qT == "float32" and tensorDtypes.cacheKeysT == "float32" and tensorDtypes.cacheValuesT == "float32" and tensorDtypes.attnT == "float32" and args.cacheLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads == args.numKvHeads and args.headDim > 0 and dim(shapes.qT, 0) >= args.qOffset + args.numHeads * args.headDim and dim(shapes.qT, 0) >= args.kOffset + args.numKvHeads * args.headDim and dim(shapes.qT, 0) >= args.vOffset + args.numKvHeads * args.headDim and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and dim(shapes.attnT, 0) == args.numHeads * args.headDim',fusedWriteOk:"fusedWriteContract and buffersFit and resourcesFit and dispatchFits",fusedShortSubgroupOk:"fusedWriteOk and subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= decodeWorkgroupSize and decodeWorkgroupSize % device.adapterInfo.subgroupMinSize == 0 and decodeWorkgroupSize % device.adapterInfo.subgroupMaxSize == 0 and attentionStorageBytes + 8 <= device.limits.maxComputeWorkgroupStorageSize",prefillQkvStride:"(args.numHeads + 2 * args.numKvHeads) * args.headDim",prefillQHidden:"args.numHeads * args.headDim",prefillCacheElements:"(args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim",prefillEffectivePastLen:"args.pastLen if args.pastLen else 0",prefillBaseContract:'prefillMode and ranks.qkvT == 2 and (ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5) and (ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5) and ranks.attnT == 2 and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and (tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16") and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and f16Ok(tensorDtypes.qkvT) and f16Ok(tensorDtypes.cacheKeysT) and f16Ok(tensorDtypes.attnT) and args.cacheLen > 0 and args.seqLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads % args.numKvHeads == 0 and args.headDim > 0 and dim(shapes.qkvT, 1) == prefillQkvStride and numel(shapes.qkvT) >= args.seqLen * prefillQkvStride and dim(shapes.attnT, 1) == prefillQHidden and numel(shapes.attnT) >= args.seqLen * prefillQHidden and numel(shapes.cacheKeysT) >= prefillCacheElements and numel(shapes.cacheValuesT) >= prefillCacheElements',prefillGateContract:"present.gateT and ranks.gateT == 2 and tensorDtypes.gateT == tensorDtypes.attnT and f16Ok(tensorDtypes.gateT) and numel(shapes.gateT) >= args.seqLen * prefillQHidden",prefillFlashContract:"prefillBaseContract and args.headDim % 16 == 0",prefillFlashDispatchFits:"ceilDiv(args.seqLen, 16) <= device.limits.maxComputeWorkgroupsPerDimension and args.numHeads <= device.limits.maxComputeWorkgroupsPerDimension",prefillFlashUsesF16:'tensorDtypes.qkvT == "float16" or tensorDtypes.cacheKeysT == "float16" or tensorDtypes.attnT == "float16"',prefillFlashLpq:"8 if args.headDim % 32 == 0 else 4",prefillFlashTileK:"8 if subgroupsWide32 and not prefillFlashUsesF16 and args.headDim <= 16 * prefillFlashLpq else (16 if subgroupsWide32 else (15 if not prefillFlashUsesF16 else 16))",prefillFlashWorkgroupStorageBytes:"2 * prefillFlashTileK * args.headDim * (2 if prefillFlashUsesF16 else 4) + (0 if subgroupsWide32 else 16 * prefillFlashLpq * 4)",headDim:"args.headDim",layer:"args.layer",cacheLen:"args.cacheLen",numKvHeads:"args.numKvHeads",numKvGroups:"args.numHeads / args.numKvHeads",f32Min:'"-3.4028234663852886e38"',qNumHeads:"args.numHeads",kvNumHeads:"args.numKvHeads",headDimV4:"args.headDim / 4",hasBias:!1,hasWindow:!1,noWindow:"args.windowSize is undefined",noSink:"not present.sinkT",sinkContract:'present.sinkT and ranks.sinkT == 1 and f16Ok(tensorDtypes.sinkT) and args.sinkOffset is defined and dim(shapes.sinkT, 0) * (2 if tensorDtypes.sinkT == "uint32" else 1) >= args.sinkOffset + args.numHeads',sinkScalar:'"u32" if tensorDtypes.sinkT == "uint32" else ("f16" if tensorDtypes.sinkT == "float16" else "f32")',ringRows:"args.cacheLen if args.cacheRing else 0",ringOk:"not args.cacheRing or (args.windowSize is defined and pow2ceil(args.cacheLen) == args.cacheLen and args.windowSize + (args.seqLen if args.seqLen else 1) - 1 <= args.cacheLen)",decodeHeadGateContract:'present.gateT and ranks.gateT == 1 and tensorDtypes.gateT == "float32" and dim(shapes.gateT, 0) == args.numHeads',prefillHeadGateContract:"present.gateT and ranks.gateT == 2 and tensorDtypes.gateT == tensorDtypes.attnT and f16Ok(tensorDtypes.gateT) and dim(shapes.gateT, 1) == args.numHeads and numel(shapes.gateT) >= args.seqLen * args.numHeads",cacheVec4:'"vec4<f16>" if tensorDtypes.cacheKeysT == "float16" else "vec4<f32>"',normWeightVec4:'"vec4<f16>" if tensorDtypes.normWeightsT == "float16" else "vec4<f32>"',scaling:"args.scale",numHeads:"args.numHeads",scale:"args.scale",qScalar:"dtypes.Q",cacheScalar:"dtypes.C",outScalar:"dtypes.O"},bindings:{params:{arg:"paramsT",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"}]},partial_out:{scratch:"partialOut",elementType:"vec4<f32>"},partial_out_merge:{scratch:"partialOut",name:"partial_out",buffer:"read-only-storage",elementType:"vec4<f32>"},qkv:{arg:"qkvT",elementType:"$qElement"},cache_keys_main:{arg:"cacheKeysT",name:"cache_keys",buffer:"read-only-storage",elementType:"$cacheElement"},cache_values_main:{arg:"cacheValuesT",name:"cache_values",buffer:"read-only-storage",elementType:"$cacheElement"},attn_out_attn_t:{arg:"attnT",name:"attn_out",elementType:"$outElement"},params_main:{name:"params",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"},{name:"cache_len",type:"u32",value:"args.cacheLen",required:!0,internal:!0},{name:"seq_len",type:"u32",value:"args.seqLen"}]},query:{arg:"qT",elementType:"$qVec4"},partial_meta:{scratch:"partialMeta",name:"partial_stats",elementType:"vec2<f32>"},partial_meta_ro:{scratch:"partialMeta",name:"partial_stats",buffer:"read-only-storage",elementType:"vec2<f32>"},output:{arg:"attnT",elementType:"$outVec4"},gate_head:{arg:"gateT",name:"gate",elementType:"$outScalar"},key_append:{arg:"cacheKeysT",name:"key",elementType:"$cacheVec4"},value_append:{arg:"cacheValuesT",name:"value",elementType:"$cacheVec4"},q_norm_weight:{arg:"normWeightsT",elementType:"$normWeightVec4"},cos_cache:{arg:"cosT",elementType:"f32"},sin_cache:{arg:"sinT",elementType:"f32"}},when:["ringOk"],variants:[{id:"llama_splitk_qk_append_head_gated",priority:20,when:["noSink","decodeHeadGateContract","splitArgsPresent","qkAppendContract","resourcesFitCore","dispatchFits","splitScratchFits","args.headDim % 4 == 0","args.headDim <= args.workgroupSize or args.headDim == args.workgroupSize * 2","args.tileSize > 0","args.workgroupSize >= args.tileSize","args.numKvChunks > 0","args.minKeysPerChunk > 0","args.headDim <= device.limits.maxComputeInvocationsPerWorkgroup","args.headDim <= device.limits.maxComputeWorkgroupSizeX","device.limits.maxComputeWorkgroupsPerDimension is undefined or args.numKvChunks <= device.limits.maxComputeWorkgroupsPerDimension","(args.workgroupSize + args.tileSize + args.headDim) * 4 <= device.limits.maxComputeWorkgroupStorageSize","sharedSplitResourcesFit","(2 * args.workgroupSize + args.tileSize + args.headDim) * 4 + 16 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{qVec4:'"vec4<f16>" if tensorDtypes.qT == "float16" else "vec4<f32>"',outVec4:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',scalar:"dtypes.O",numSplits:"args.numKvChunks",useSubgroups:"sharedSplitUseSubgroups",hasWindow:"args.windowSize is defined",windowSizeConst:"args.windowSize"},intermediates:[{id:"partialOut",dtype:"float32",shape:"[args.numHeads * args.numKvChunks * args.headDim]"},{id:"partialMeta",dtype:"float32",shape:"[args.numHeads * args.numKvChunks * 2]"}],passes:[{id:"split",name:"LlamaDecodeAttentionSplitK",shader:"ops/_shared/attn-flash-decode-splitk.wgsl.jinja",derive:{usesF16:'tensorDtypes.qT == "float16" or tensorDtypes.cacheKeysT == "float16"',layout:'"layer_cache"',workgroupSizeSpec:"args.workgroupSize",minKeysPerChunk:"args.minKeysPerChunk",fusedKvAppend:!0,qkEps:"args.rmsEps",qNormBaseV4:"args.qNormOffset / 4",kNormBaseV4:"args.kNormOffset / 4",kOffsetV4:"args.kOffset / 4",vOffsetV4:"args.vOffset / 4"},bindings:["query","key_append","value_append","q_norm_weight","cos_cache","sin_cache","partial_out","partial_meta","params"],dispatch:{x:"args.numKvChunks",y:"args.numHeads",z:1}},{id:"merge",name:"LlamaDecodeAttentionSplitKMerge",shader:"ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",derive:{layout:'"layer_cache"',hasGate:'"head"'},bindings:["partial_out_merge","partial_meta_ro","gate_head","output"],dispatch:{x:1,y:"args.numHeads",z:1}}]},{id:"prefill_scalar",priority:0,when:["prefillBaseContract","not present.gateT and noSink","args.workgroupSize == 8 or args.workgroupSize == 16 or args.workgroupSize == 32 or args.workgroupSize == 64 or args.workgroupSize == 128 or args.workgroupSize == 256"],derive:{qElement:"dtypes.Q",cacheElement:"dtypes.C",outElement:"dtypes.O",qHidden:"prefillQHidden",qkvStride:"prefillQkvStride",workgroupSize:"args.workgroupSize",hasOutputGate:!1,windowSizeConst:"args.windowSize"},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"prefill-attention.wgsl.jinja",bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main"],dispatch:{x:"args.numHeads",y:"min(args.seqLen, device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"prefill_scalar_head_gated",priority:0,when:["prefillBaseContract","prefillHeadGateContract and noSink","args.workgroupSize == 8 or args.workgroupSize == 16 or args.workgroupSize == 32 or args.workgroupSize == 64 or args.workgroupSize == 128 or args.workgroupSize == 256"],derive:{qElement:"dtypes.Q",cacheElement:"dtypes.C",outElement:"dtypes.O",qHidden:"prefillQHidden",qkvStride:"prefillQkvStride",workgroupSize:"args.workgroupSize",hasOutputGate:'"head"',windowSizeConst:"args.windowSize"},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"prefill-attention.wgsl.jinja",bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","gate_head"],dispatch:{x:"args.numHeads",y:"min(args.seqLen, device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"prefill_flash",priority:1,when:["prefillFlashContract","not present.gateT and noSink","args.layer == 0","ringRows > 0 or prefillEffectivePastLen + args.seqLen <= args.cacheLen","prefillFlashDispatchFits","prefillFlashWorkgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize"],derive:{usesF16:"prefillFlashUsesF16",scalar:"dtypes.O",qHiddenV4:"prefillQHidden / 4",kvHiddenV4:"args.numKvHeads * args.headDim / 4",qkvStrideV4:"prefillQkvStride / 4",hasCausal:!0,causalRightAlign:!0,TILE_Q:16,TILE_K:"prefillFlashTileK",LPQ:"prefillFlashLpq",qElement:'"vec4<f16>" if tensorDtypes.qkvT == "float16" else "vec4<f32>"',cacheElement:'"vec4<f16>" if tensorDtypes.cacheKeysT == "float16" else "vec4<f32>"',outElement:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',useSubgroups:"subgroupsWide32",batchNoSgReduction:!1,hasWindow:"args.windowSize is defined",windowSizeConst:"args.windowSize"},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",derive:{layout:'"bsh"',sourceProfile:1,outputGate:!1,validateClusterLayout:!1},bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main"],dispatch:{x:"ceilDiv(args.seqLen, TILE_Q)",y:"args.numHeads"}}],requires:{}},{id:"prefill_flash_head_gated",priority:1,when:["prefillFlashContract","prefillHeadGateContract and noSink","args.layer == 0","ringRows > 0 or prefillEffectivePastLen + args.seqLen <= args.cacheLen","prefillFlashDispatchFits","prefillFlashWorkgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize"],derive:{usesF16:"prefillFlashUsesF16",scalar:"dtypes.O",qHiddenV4:"prefillQHidden / 4",kvHiddenV4:"args.numKvHeads * args.headDim / 4",qkvStrideV4:"prefillQkvStride / 4",hasCausal:!0,causalRightAlign:!0,TILE_Q:16,TILE_K:"prefillFlashTileK",LPQ:"prefillFlashLpq",qElement:'"vec4<f16>" if tensorDtypes.qkvT == "float16" else "vec4<f32>"',cacheElement:'"vec4<f16>" if tensorDtypes.cacheKeysT == "float16" else "vec4<f32>"',outElement:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',useSubgroups:"subgroupsWide32",batchNoSgReduction:!1,hasWindow:"args.windowSize is defined",windowSizeConst:"args.windowSize"},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",derive:{layout:'"bsh"',sourceProfile:1,outputGate:'"head"',validateClusterLayout:!1},bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","gate_head"],dispatch:{x:"ceilDiv(args.seqLen, TILE_Q)",y:"args.numHeads"}}],requires:{}},{id:"prefill_flash_sgmat_head_gated",priority:3,when:["prefillFlashContract","prefillHeadGateContract and noSink","noWindow or args.windowSize >= 16","args.layer == 0","ringRows > 0 or prefillEffectivePastLen + args.seqLen <= args.cacheLen",'tensorDtypes.qkvT == "float32"','tensorDtypes.cacheKeysT == "float32"','tensorDtypes.attnT == "float32"',"args.headDim == 128 or args.headDim == 256","args.seqLen >= 256","wave32Effective","args.numHeads % args.numKvHeads == 0","(args.numHeads / args.numKvHeads) == 2 or (args.numHeads / args.numKvHeads) == 4 or (args.numHeads / args.numKvHeads) == 6","(args.numHeads / args.numKvHeads) * 32 <= deviceWorkgroupCap","ceilDiv(args.seqLen, 8) <= device.limits.maxComputeWorkgroupsPerDimension","args.numKvHeads <= device.limits.maxComputeWorkgroupsPerDimension","(args.numHeads / args.numKvHeads) * 3744 + 32 + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{qElement:'"f32"',cacheElement:'"f32"',outElement:'"f32"',sgHeadDim:"args.headDim",sgQHeads:"args.numHeads",sgKvHeads:"args.numKvHeads",sgGroup:"args.numHeads / args.numKvHeads",hasOutputGate:'"head"',windowSizeConst:"args.windowSize"},passes:[{id:"main",name:"LlamaPrefillAttention.Sgmat",shader:"prefill-flash-sgmat.wgsl.jinja",bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","gate_head"],dispatch:{x:"ceilDiv(args.seqLen, 8)",y:"args.numKvHeads"},subgroupCollectivesWidth:"portable"}]}]},assets:[["ops/_shared/attn-flash-decode-splitk.wgsl.jinja",`{% set WINDOW_SIZE = (windowSizeConst ~ "u") if (windowSizeConst is defined and windowSizeConst) else "params.windowSize" %}{% set kvRing = ringRows > 0 %}{% set RING_MASK = ((ringRows - 1) ~ "u") if kvRing else "" %}{% if useSubgroups is not defined %}{% set useSubgroups = true %}{% endif %}{% set appendCacheVec4 = "vec4<f32>" %}{% set normWeightOpen = "" %}{% set normWeightClose = "" %}{% set splitKWorkgroupSize = workgroupSizeSpec if workgroupSizeSpec is defined else tunables.WORKGROUP_SIZE %}{% if useSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }} const HEAD_DIM:u32={{ headDim }}u;const HEAD_DIM_V4:u32={{ headDimV4 }}u;const Q_HEADS:u32={{ qNumHeads }}u;const KV_HEADS:u32={{ kvNumHeads }}u;const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const ATTN_SCALE:f32={{ scale }};const WG:u32={{ splitKWorkgroupSize }}u;const NUM_SPLITS:u32={{ numSplits }}u;const HALF_DIM_V4:u32=HEAD_DIM_V4/2u;const QK_EPS:f32={{ qkEps }};const K_OFFSET_V4:u32={{ kOffsetV4 }}u;const V_OFFSET_V4:u32={{ vOffsetV4 }}u;const Q_NORM_BASE_V4:u32={{ qNormBaseV4 }}u;const K_NORM_BASE_V4:u32={{ kNormBaseV4 }}u;
{% include "ops/_shared/softmax-stable-helpers.49d3e37058.wgsl.jinja" %}var<workgroup>q_shared:array<vec4<f32>,HEAD_DIM_V4>;var<workgroup>running_out:array<vec4<f32>,HEAD_DIM_V4>;var<workgroup>probs:array<f32,WG>;
{% set coopQk = useSubgroups and headDimV4 >= 8 and not (usesF16 and headDimV4 <= 32) and (allowCooperativeQk if allowCooperativeQk is defined else true) %}{% set jGroups = (splitKWorkgroupSize / headDimV4)|int %}{% set jSplitV = (splitKWorkgroupSize % headDimV4 == 0) and (jGroups >= 2) %}{% if coopQk %}var<workgroup>sval_sh:array<f32,WG>;
{% endif %}{% if jSplitV %}var<workgroup>vacc_sh:array<vec4<f32>,WG>;
{% endif %}{% set combineSubgroups = useSubgroups %}{% include "ops/_shared/softmax-md-combine.e13c0ec67f.wgsl.jinja" %}{% set ATTN_SCALE_OVERRIDE = "ATTN_SCALE" %}{% include "ops/_shared/attn-scale-value.ae17ac7a82.wgsl.jinja" %}fn load_key4(indexV4:u32)->vec4<f32>{return vec4<f32>(key[indexV4]);}fn load_value4(indexV4:u32)->vec4<f32>{return vec4<f32>(value[indexV4]);}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if coopQk %},
@builtin(subgroup_size)sgSize:u32,
  @builtin(subgroup_invocation_id) sgLane: u32{% endif %}){let split=wg.x;let h=wg.y;let b=wg.z;
  if (h >= Q_HEADS || split >= NUM_SPLITS{% if layout == "layer_cache" and not kvRing %} || params.past_len >= CACHE_LEN{% endif %}) {
return;}let tid=lid.x;
{% if coopQk %}let physicalWidth=max(sgSize,1u);let population=subgroupBallot(true);let populationCount=countOneBits(population.x)+countOneBits(population.y)+countOneBits(population.z)+countOneBits(population.w);let firstLocalIndex=subgroupBroadcastFirst(tid);let ownsLocalBlock=subgroupAll(sgSize>0u&&populationCount==sgSize&&WG%physicalWidth==0u&&tid/physicalWidth==firstLocalIndex/physicalWidth&&sgLane==tid%physicalWidth);if(subgroupElect()&&!ownsLocalBlock){atomicStore(&leaderMask[0],1u);}workgroupBarrier();if(tid==0u){combinedMD=vec2<f32>(f32(atomicExchange(&leaderMask[0],0u)),0.0);}let coopQkAccepted=workgroupUniformLoad(&combinedMD).x==0.0;
{% endif %}let hKv=h/(Q_HEADS/KV_HEADS);let kvSeq=params.past_len+1u;let qBaseV4=h*HEAD_DIM_V4;let kvBaseV4=(LAYER*CACHE_LEN*KV_HEADS+hKv)*HEAD_DIM_V4;let kvTokenStrideV4=KV_HEADS*HEAD_DIM_V4;
{% if hasWindow %}var windowStart:u32=0u;if(kvSeq>{{ WINDOW_SIZE }}){windowStart=kvSeq-{{ WINDOW_SIZE }};}let activeKeys=kvSeq-windowStart;let keysPerSplit=(activeKeys+NUM_SPLITS-1u)/NUM_SPLITS;let splitStart=windowStart+split*keysPerSplit;
{% else %}let evenKeysPerSplit=(kvSeq+NUM_SPLITS-1u)/NUM_SPLITS;let keysPerSplit=max(evenKeysPerSplit,{{ minKeysPerChunk }}u);let splitStart=split*keysPerSplit;
{% endif %}var splitEnd=splitStart+keysPerSplit;if(splitEnd>kvSeq){splitEnd=kvSeq;}let ropeRow=(kvSeq-1u)*(HALF_DIM_V4*4u);var q_ss=0.0;for(var d4=0u;d4<HEAD_DIM_V4;d4=d4+1u){let q4=vec4<f32>(query[qBaseV4+d4]);q_ss=q_ss+dot(q4,q4);}let q_inv=inverseSqrt(q_ss/f32(HEAD_DIM)+QK_EPS);for(var d4=tid;d4<HALF_DIM_V4;d4=d4+WG){let x0=vec4<f32>(query[qBaseV4+d4])*q_inv*{{ normWeightOpen }}q_norm_weight[Q_NORM_BASE_V4+d4]{{ normWeightClose }};let x1=vec4<f32>(query[qBaseV4+d4+HALF_DIM_V4])*q_inv*{{ normWeightOpen }}q_norm_weight[Q_NORM_BASE_V4+d4+HALF_DIM_V4]{{ normWeightClose }};let d=ropeRow+d4*4u;let c=vec4<f32>(cos_cache[d],cos_cache[d+1u],cos_cache[d+2u],cos_cache[d+3u]);let s=vec4<f32>(sin_cache[d],sin_cache[d+1u],sin_cache[d+2u],sin_cache[d+3u]);q_shared[d4]=x0*c-x1*s;q_shared[d4+HALF_DIM_V4]=x1*c+x0*s;}for(var d4=tid;d4<HEAD_DIM_V4;d4=d4+WG){running_out[d4]=vec4<f32>(0.0);}let newPos=kvSeq-1u;if(newPos>=splitStart&&newPos<splitEnd){let kRawV4=K_OFFSET_V4+hKv*HEAD_DIM_V4;let vRawV4=V_OFFSET_V4+hKv*HEAD_DIM_V4;var k_ss=0.0;for(var d4=0u;d4<HEAD_DIM_V4;d4=d4+1u){let k4=vec4<f32>(query[kRawV4+d4]);k_ss=k_ss+dot(k4,k4);}let k_inv=inverseSqrt(k_ss/f32(HEAD_DIM)+QK_EPS);let newRowV4=kvBaseV4+{{ ("(newPos & " ~ RING_MASK ~ ")") if kvRing else "newPos" }}*kvTokenStrideV4;for(var d4=tid;d4<HALF_DIM_V4;d4=d4+WG){let x0=vec4<f32>(query[kRawV4+d4])*k_inv*{{ normWeightOpen }}q_norm_weight[K_NORM_BASE_V4+d4]{{ normWeightClose }};let x1=vec4<f32>(query[kRawV4+d4+HALF_DIM_V4])*k_inv*{{ normWeightOpen }}q_norm_weight[K_NORM_BASE_V4+d4+HALF_DIM_V4]{{ normWeightClose }};let d=ropeRow+d4*4u;let c=vec4<f32>(cos_cache[d],cos_cache[d+1u],cos_cache[d+2u],cos_cache[d+3u]);let s=vec4<f32>(sin_cache[d],sin_cache[d+1u],sin_cache[d+2u],sin_cache[d+3u]);key[newRowV4+d4]={{ appendCacheVec4 }}(x0*c-x1*s);key[newRowV4+d4+HALF_DIM_V4]={{ appendCacheVec4 }}(x1*c+x0*s);}for(var d4=tid;d4<HEAD_DIM_V4;d4=d4+WG){value[newRowV4+d4]={{ appendCacheVec4 }}(query[vRawV4+d4]);}storageBarrier();}workgroupBarrier();let scale=scale_value();var runningMax=-FLT_MAX;var runningDenom=0.0;var kjBase=splitStart;loop{if(kjBase>=splitEnd){break;}let kj=kjBase+tid;var keyAllowed=kj<splitEnd;let tileCount=min(WG,splitEnd-kjBase);var score=-FLT_MAX;var m=-FLT_MAX;var dPart=0.0;
{% if coopQk %}if(coopQkAccepted){let sgPerWg=WG/sgSize;let qkRounds=(tileCount+sgPerWg-1u)/sgPerWg;let lane=tid%sgSize;let sgInWg=tid/sgSize;for(var rr:u32=0u;rr<qkRounds;rr=rr+1u){let j=rr*sgPerWg+sgInWg;var accS:f32=0.0;if(j<tileCount){let kRowV4=kvBaseV4+{{ ("((kjBase + j) & " ~ RING_MASK ~ ")") if kvRing else "(kjBase + j)" }}*kvTokenStrideV4;for(var d4:u32=lane;d4<HEAD_DIM_V4;d4=d4+sgSize){accS=accS+dot(q_shared[d4],load_key4(kRowV4+d4));} }let sj=subgroupAdd(accS);if(lane==0u&&j<tileCount){sval_sh[j]=sj;} }workgroupBarrier();if(keyAllowed){score=sval_sh[tid]*scale;m=score;dPart=1.0;} }else{if(keyAllowed){let kRowV4=kvBaseV4+{{ ("(kj & " ~ RING_MASK ~ ")") if kvRing else "kj" }}*kvTokenStrideV4;var acc:f32=0.0;for(var d4:u32=0u;d4<HEAD_DIM_V4;d4=d4+1u){acc=acc+dot(q_shared[d4],load_key4(kRowV4+d4));}score=acc*scale;m=score;dPart=1.0;} }
{% else %}if(keyAllowed){let kRowV4=kvBaseV4+{{ ("(kj & " ~ RING_MASK ~ ")") if kvRing else "kj" }}*kvTokenStrideV4;var acc:f32=0.0;for(var d4:u32=0u;d4<HEAD_DIM_V4;d4=d4+1u){acc=acc+dot(q_shared[d4],load_key4(kRowV4+d4));}score=acc*scale;m=score;dPart=1.0;}
{% endif %}let tile=combine_partials(m,dPart,tid);
{% include "ops/_shared/attn-online-merge-tile.d088626ed8.wgsl.jinja" %}{% if jSplitV %}const J_GROUPS:u32={{ jGroups }}u;let jg=tid/HEAD_DIM_V4;let d4v=tid%HEAD_DIM_V4;var vacc=vec4<f32>(0.0);var jj=jg;loop{if(jj>=tileCount){break;}vacc=vacc+probs[jj]*load_value4(kvBaseV4+{{ ("((kjBase + jj) & " ~ RING_MASK ~ ")") if kvRing else "(kjBase + jj)" }}*kvTokenStrideV4+d4v);jj=jj+J_GROUPS;}vacc_sh[tid]=vacc;workgroupBarrier();for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){var a4=running_out[d4]*correction;for(var g:u32=0u;g<J_GROUPS;g=g+1u){a4=a4+vacc_sh[g*HEAD_DIM_V4+d4];}running_out[d4]=a4;}workgroupBarrier();
{% else %}for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){var vSum=vec4<f32>(0.0);for(var i:u32=0u;i<tileCount;i=i+1u){vSum=vSum+probs[i]*load_value4(kvBaseV4+{{ ("((kjBase + i) & " ~ RING_MASK ~ ")") if kvRing else "(kjBase + i)" }}*kvTokenStrideV4+d4);}running_out[d4]=running_out[d4]*correction+vSum;}workgroupBarrier();
{% endif %}kjBase=kjBase+WG;}let partialBase=((b*Q_HEADS+h)*NUM_SPLITS+split)*HEAD_DIM_V4;for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){partial_out[partialBase+d4]=running_out[d4];}if(tid==0u){let mdBase=(b*Q_HEADS+h)*NUM_SPLITS+split;partial_stats[mdBase]=vec2<f32>(runningMax,runningDenom);} }`],["ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",`{% if hasGate is not defined %}{% set hasGate = false %}{% endif %}{{ env.wgsl.resourceDeclarations }} const HEAD_DIM_V4:u32={{ headDimV4 }}u;const Q_HEADS:u32={{ qNumHeads }}u;const NUM_SPLITS:u32={{ numSplits }}u;
{% include "ops/_shared/softmax-stable-helpers.49d3e37058.wgsl.jinja" %}@compute @workgroup_size(HEAD_DIM_V4, 1, 1)
fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let h=wg.y;let b=wg.z;let d4=lid.x;if(h>=Q_HEADS||d4>=HEAD_DIM_V4){return;}let mdBase=(b*Q_HEADS+h)*NUM_SPLITS;var globalMax=-FLT_MAX;for(var s:u32=0u;s<NUM_SPLITS;s=s+1u){globalMax=max(globalMax,partial_stats[mdBase+s].x);}var globalDenom=0.0;var outV=vec4<f32>(0.0);for(var s:u32=0u;s<NUM_SPLITS;s=s+1u){let stats=partial_stats[mdBase+s];let w=exp_shift(stats.x,globalMax);globalDenom=globalDenom+stats.y*w;let pBase=((b*Q_HEADS+h)*NUM_SPLITS+s)*HEAD_DIM_V4;outV=outV+partial_out[pBase+d4]*w;}let qBaseV4=h*HEAD_DIM_V4;var outValue=select(vec4<f32>(0.0),outV/globalDenom,globalDenom>0.0);
{% if hasGate == "head" %}outValue=outValue*(1.0/(1.0+exp(-f32(gate[h]))));
{% elif hasGate %}let gateV=vec4<f32>(gate[qBaseV4+d4]);outValue=outValue*(vec4<f32>(1.0)/(vec4<f32>(1.0)+exp(-gateV)));
{% endif %}output[qBaseV4+d4]=vec4<{{ scalar }}>(outValue);}`],["prefill-attention.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const NUM_KV_GROUPS:u32={{ numKvGroups }}u;const HEAD_DIM:u32={{ headDim }}u;const Q_HIDDEN:u32={{ qHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const ATTN_SCALING:f32={{ scaling }};const WG:u32={{ workgroupSize }}u;
{% if windowSizeConst is defined and windowSizeConst %}const WINDOW:u32={{ windowSizeConst }}u;
{% endif %}var<workgroup>partial:array<f32,WG>;var<workgroup>running_max:f32;var<workgroup>running_denom:f32;var<workgroup>running_out:array<f32,HEAD_DIM>;var<workgroup>previous_scale:f32;var<workgroup>current_score:f32;
{% include "ops/_shared/kv-cache-index.26c94f7350.wgsl.jinja" %}{% include "ops/_shared/reduce-sum.fdec0f92c8.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(num_workgroups)ngroups:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let head=wg.x;if(head>=NUM_HEADS){return;}let kv_head=head/NUM_KV_GROUPS;let tid=lid.x;for(var query_token:u32=wg.y;query_token<params.seq_len;query_token=query_token+ngroups.y){let abs_pos=params.past_len+query_token;
{% if not (ringRows is defined and ringRows) %}if(abs_pos>=CACHE_LEN){continue;}
{% endif %}if(tid==0u){running_max=-3.4028234663852886e38;running_denom=0.0;}for(var d:u32=tid;d<HEAD_DIM;d=d+WG){running_out[d]=0.0;}workgroupBarrier();let q_row=query_token*QKV_STRIDE+head*HEAD_DIM;
{% if windowSizeConst is defined and windowSizeConst %}let first_key=select(0u,abs_pos+1u-WINDOW,abs_pos+1u>WINDOW);for(var key_token:u32=first_key;key_token<=abs_pos;key_token=key_token+1u){
{% else %}for(var key_token:u32=0u;key_token<=abs_pos;key_token=key_token+1u){
{% endif %}var partial_dot=0.0;for(var d:u32=tid;d<HEAD_DIM;d=d+WG){partial_dot=partial_dot+f32(qkv[q_row+d])*f32(cache_keys[cache_index(key_token,kv_head,d)]);}let score=reduce_sum(partial_dot,tid)*ATTN_SCALING;if(tid==0u){current_score=score;let next_max=max(running_max,score);let old_factor=exp(running_max-next_max);let new_factor=exp(score-next_max);previous_scale=old_factor;running_max=next_max;running_denom=running_denom*old_factor+new_factor;}workgroupBarrier();let probability_numerator=exp(current_score-running_max);for(var d:u32=tid;d<HEAD_DIM;d=d+WG){running_out[d]=running_out[d]*previous_scale+probability_numerator*f32(cache_values[cache_index(key_token,kv_head,d)]);}workgroupBarrier();}let inv_denom=1.0/running_denom;for(var d:u32=tid;d<HEAD_DIM;d=d+WG){let out_index=query_token*Q_HIDDEN+head*HEAD_DIM+d;
{% if hasOutputGate %}let rounded=f32({{ outScalar }}(running_out[d]*inv_denom));
{% if hasOutputGate == "head" %}let gate_value=1.0/(1.0+exp(-f32(gate[query_token*NUM_HEADS+head])));
{% else %}let gate_value=1.0/(1.0+exp(-f32(gate[out_index])));
{% endif %}attn_out[out_index]={{ outScalar }}(rounded*gate_value);
{% else %}attn_out[out_index]={{ outScalar }}(running_out[d]*inv_denom);
{% endif %}}workgroupBarrier();} }`],["ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",`{% set VALUE_HEAD = valueHeadDim | default(headDim) %}{% set VALUE_SPLIT = VALUE_HEAD != headDim %}{% set VALUE_DIM_NAME = "VALUE_DIM" if VALUE_SPLIT else "HEAD_DIM" %}{% set VALUE_DIM_V4_NAME = "VALUE_DIM_V4" if VALUE_SPLIT else "HEAD_DIM_V4" %}{% set VALUE_SLICE_NAME = "VALUE_SLICE" if VALUE_SPLIT else "SLICE" %}{% set WINDOW_SIZE = (windowSizeConst ~ "u") if (windowSizeConst is defined and windowSizeConst) else "params.windowSize" %}{% set KV_ROW = ("(kj & " ~ (ringRows - 1) ~ "u)") if (ringRows is defined and ringRows) else "kj" %}{% macro cluster_shuffle_sum(value, width) %}{% for stage in range(log2ceil(width)) %}{{ value }} = {{ value }} + subgroupShuffleXor({{ value }}, {{ pow(2, stage)|int }}u);
{% endfor %}{% endmacro %}{% macro cluster_tree_sum(prefix, offset, count) %}{% if count == 1 %}{{ prefix }}{{ offset }}{% else %}{% set left = ((count + 1) / 2)|int %}({{ cluster_tree_sum(prefix, offset, left) }} + {{ cluster_tree_sum(prefix, offset + left, count - left) }}){% endif %}{% endmacro %}{% set QSEQ = "params.seq_len" %}{% set KVSEQ = "(params.past_len + params.seq_len)" %}{% set IS_CAUSAL = "1u" %}{% set Q_STRIDE = "QKV_STRIDE_V4" %}{% set QUERY = "qkv" %}{% set KEY = "cache_keys" %}{% set VALUE = "cache_values" %}{% set OUTPUT = "attn_out" %}{% set HAS_OUTPUT_GATE = outputGate if outputGate is defined else false %}{% set ATTN_SCALE_OVERRIDE = scaling %}{% if useSubgroups is not defined %}{% set useSubgroups = true %}{% endif %}{% if batchNoSgReduction is not defined %}{% set batchNoSgReduction = false %}{% endif %}{% set KVA = KVSEQ %}{% macro score_expr(part) %}{{ part }} * SCALE{% endmacro %}{% if useSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }}
{% set ST = "f32" %}{% set MASK_KEY_RANGE = false %}{% set SLICE_COUNT = ((headDimV4 / LPQ) | int) %}{% set VALUE_SLICE_COUNT = ((VALUE_HEAD / (4 * LPQ)) | int) if VALUE_SPLIT else SLICE_COUNT %}{% set MAX_SLICE_COUNT = max(SLICE_COUNT, VALUE_SLICE_COUNT) %}{% set QPL = (2 if (not usesF16 and not (singleQueryPerLane | default(false)) and MAX_SLICE_COUNT <= 4 and TILE_K <= 16 and TILE_Q % 2 == 0) else 1) if useSubgroups else 1 %}{% macro qn(name, qi) %}{{ name }}{% if qi > 0 %}_q{{ qi }}{% endif %}{% endmacro %}{% macro emit_pair_scores() %}{% for qi in range(QPL) %}{let sc=partv[{{ qi }}];if(kj>={{ qn("minKj", qi) }}&&kj<{{ qn("maxKj", qi) }}){ {{ qn("s", qi) }}[kk]={{ score_expr("sc") }};} }
{% endfor %}{% endmacro %}{% macro emit_tile_softmax() %}{% for qi in range(QPL) %}var {{ qn("tileMax", qi) }}:f32={{ qn("s", qi) }}[0];for(var kk:u32=1u;kk<TILE_K;kk=kk+1u){ {{ qn("tileMax", qi) }}=max({{ qn("tileMax", qi) }},{{ qn("s", qi) }}[kk]);}let {{ qn("newMax", qi) }}=max({{ qn("m", qi) }},{{ qn("tileMax", qi) }});let {{ qn("corr", qi) }}=select(exp({{ qn("m", qi) }}-{{ qn("newMax", qi) }}),0.0,{{ qn("m", qi) }}==NEG_INF);var {{ qn("pSum", qi) }}:f32=0.0;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let pk=select(0.0,exp({{ qn("s", qi) }}[kk]-{{ qn("newMax", qi) }}),{{ qn("s", qi) }}[kk]!=NEG_INF);{{ qn("s", qi) }}[kk]=pk;{{ qn("pSum", qi) }}={{ qn("pSum", qi) }}+pk;} {{ qn("l", qi) }}={{ qn("l", qi) }}*{{ qn("corr", qi) }}+{{ qn("pSum", qi) }};{{ qn("m", qi) }}={{ qn("newMax", qi) }};
{% endfor %}{% endmacro %}const HEAD_DIM:u32={{ headDim }}u;const HEAD_DIM_V4:u32={{ headDimV4 }}u;
{% if VALUE_SPLIT %}const VALUE_DIM: u32 = {{ VALUE_HEAD }}u;
const VALUE_DIM_V4:u32={{ (VALUE_HEAD / 4) | int }}u;const VALUE_SLICE:u32=VALUE_DIM/(4u*LPQ);
{% endif %}const Q_HIDDEN_V4: u32 = {{ qHiddenV4 }}u;   
const QKV_STRIDE_V4:u32={{ qkvStrideV4 }}u;const KV_HIDDEN_V4:u32={{ kvHiddenV4 }}u;const Q_HEADS:u32={{ qNumHeads }}u;const KV_HEADS:u32={{ kvNumHeads }}u;const TILE_Q:u32={{ TILE_Q }}u;const LPQ:u32={{ LPQ }}u;const SLICE:u32=HEAD_DIM/(4u*LPQ);const TILE_K:u32={{ TILE_K }}u;
{% if QPL > 1 %}const QPL:u32={{ QPL }}u;const WG:u32=(TILE_Q/QPL)*LPQ;
{% else %}const WG:u32=TILE_Q*LPQ;
{% endif %}const NEG_INF:f32=-3.4028234663852886e38;var<workgroup>k_tile:array<vec4<{{ ST }}>,TILE_K*(HEAD_DIM/4u)>;var<workgroup>v_tile:array<vec4<{{ ST }}>,TILE_K*({{ VALUE_DIM_NAME }}/4u)>;
{% if not useSubgroups %}{% if batchNoSgReduction %}var<workgroup>red:array<f32,TILE_K*WG>;
{% else %}var<workgroup>red:array<f32,WG>;
{% endif %}{% endif %}{% include "ops/_shared/attn-scale-value.88c0e1c03f.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if useSubgroups %},
  @builtin(subgroup_size) sgSize: u32{% endif %}){let h=wg.y;let b=wg.z;let tid=lid.x;
{% if useSubgroups %}if(sgSize<LPQ||sgSize%LPQ!=0u){return;}
{% endif %}{% if QPL > 1 %}let qSub=(tid/LPQ)*QPL;
{% else %}let qSub=tid/LPQ;
{% endif %}let lane8=tid%LPQ;
{% for qi in range(QPL) %}  let {{ qn("qIdx", qi) }} = wg.x * TILE_Q + qSub{% if qi > 0 %} + {{ qi }}u{% endif %};
let {{ qn("qValid", qi) }}={{ qn("qIdx", qi) }}<{{ QSEQ }}&&h<Q_HEADS;
{% endfor %}let hKv=h/(Q_HEADS/KV_HEADS);let SCALE=scale_value();
{% for qi in range(QPL) %}let {{ qn("qClamped", qi) }}=min({{ qn("qIdx", qi) }},{{ QSEQ }}-1u);let {{ qn("qBase4", qi) }}=(b*{{ QSEQ }}+{{ qn("qClamped", qi) }})*{{ Q_STRIDE }}+h*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}var {{ qn("qr" ~ c, qi) }}=vec4<f32>({{ QUERY }}[{{ qn("qBase4", qi) }}+{{ c }}u]);
{% if not VALUE_SPLIT %}  var {{ qn("o" ~ c, qi) }} = vec4<f32>(0.0);
{% endif %}{% endfor %}{% if VALUE_SPLIT %}{% for c in range(VALUE_SLICE_COUNT) %}  var {{ qn("o" ~ c, qi) }} = vec4<f32>(0.0);
{% endfor %}{% endif %}{% endfor %}{% for qi in range(QPL) %}var {{ qn("m", qi) }}:f32=NEG_INF;var {{ qn("l", qi) }}:f32=0.0;
{% endfor %}let pastLen=select(0u,{{ KVA }}-{{ QSEQ }},{{ KVA }}>={{ QSEQ }});
{% for qi in range(QPL) %}var {{ qn("maxKjV", qi) }}=select({{ KVA }},min(pastLen+{{ qn("qIdx", qi) }}+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);var {{ qn("minKjV", qi) }}:u32=0u;
{% endfor %}let lastQ=min(wg.x*TILE_Q+TILE_Q-1u,{{ QSEQ }}-1u);var wgEndV=select({{ KVA }},min(pastLen+lastQ+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);var wgStartV:u32=0u;
{% if hasWindow %}{% for qi in range(QPL) %}let {{ qn("qAbsP1", qi) }}=pastLen+{{ qn("qIdx", qi) }}+1u;{{ qn("maxKjV", qi) }}=min({{ qn("maxKjV", qi) }},{{ qn("qAbsP1", qi) }});if({{ qn("qAbsP1", qi) }}>{{ WINDOW_SIZE }}){ {{ qn("minKjV", qi) }}={{ qn("qAbsP1", qi) }}-{{ WINDOW_SIZE }};}
{% endfor %}let lastQAbsP1=pastLen+lastQ+1u;wgEndV=min(wgEndV,lastQAbsP1);let firstQAbsP1=pastLen+wg.x*TILE_Q+1u;if(firstQAbsP1>{{ WINDOW_SIZE }}){wgStartV=firstQAbsP1-{{ WINDOW_SIZE }};}
{% endif %}{% for qi in range(QPL) %}let {{ qn("maxKj", qi) }}={{ qn("maxKjV", qi) }};let {{ qn("minKj", qi) }}={{ qn("minKjV", qi) }};
{% endfor %}let wgEnd=wgEndV;let wgStart=wgStartV;
{% if not VALUE_SPLIT %}  let kvBatch4 = b * {{ KVSEQ }} * KV_HIDDEN_V4 + hKv * HEAD_DIM_V4;
{% endif %}  var kStart: u32 = {{ "keyRangeStart" if MASK_KEY_RANGE else "wgStart" }};
loop{if(kStart>={{ "keyRangeEnd" if MASK_KEY_RANGE else "wgEnd" }}){break;}
{% if VALUE_SPLIT %}    
workgroupBarrier();
{% for stream in ["key", "value"] %}{% set isValue = stream == "value" %}{% set dim4 = "VALUE_DIM_V4" if isValue else "HEAD_DIM_V4" %}for(var i:u32=tid;i<TILE_K*{{ dim4 }};i=i+WG){let slot=i/{{ dim4 }};let d4=i% {{ dim4 }};let kj=kStart+slot;if(kj<wgEnd){let base4=((b*KV_HEADS+hKv)*{{ KVSEQ }}+kj)*{{ dim4 }}+d4;{{ "v_tile" if isValue else "k_tile" }}[i]=vec4<{{ ST }}>(padded_load_{{ stream }}(base4));}else{ {{ "v_tile" if isValue else "k_tile" }}[i]=vec4<{{ ST }}>(0.0);} }
{% endfor %}{% else %}    
workgroupBarrier();for(var i:u32=tid;i<TILE_K*HEAD_DIM_V4;i=i+WG){let slot=i/HEAD_DIM_V4;let d4=i%HEAD_DIM_V4;let kj=kStart+slot;if(kj<wgEnd){let base4=kvBatch4+{{ KV_ROW }}*KV_HIDDEN_V4+d4;k_tile[i]=vec4<{{ ST }}>({{ KEY }}[base4]);v_tile[i]=vec4<{{ ST }}>({{ VALUE }}[base4]);}else{k_tile[i]=vec4<{{ ST }}>(0.0);v_tile[i]=vec4<{{ ST }}>(0.0);} }
{% endif %}    workgroupBarrier();
{% if not useSubgroups and batchNoSgReduction %}var s:array<f32,TILE_K>;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){s[kk]=NEG_INF;var part:f32=0.0;let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}part=part+dot(qr{{ c }},vec4<f32>(k_tile[kb+{{ c }}u]));
{% endfor %}red[kk*WG+tid]=part;}workgroupBarrier();let cbase=tid-lane8;for(var kk:u32=lane8;kk<TILE_K;kk=kk+LPQ){
{% for j in range(LPQ) %}let r{{ j }}=red[kk*WG+cbase+{{ j }}u];
{% endfor %}let part={{ cluster_tree_sum("r", 0, LPQ) }};red[kk*WG+cbase]=part;}workgroupBarrier();for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;let part=red[kk*WG+tid-lane8];if(kj>=minKj&&kj<maxKj){s[kk]={{ score_expr("part") }};} }
{% elif QPL > 1 %}{% for qi in range(QPL) %}var {{ qn("s", qi) }}:array<f32,TILE_K>;
{% endfor %}for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;
{% for qi in range(QPL) %}{{ qn("s", qi) }}[kk]=NEG_INF;var {{ qn("part", qi) }}:f32=0.0;
{% endfor %}let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}let kw{{ c }}=vec4<f32>(k_tile[kb+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("part", qi) }}={{ qn("part", qi) }}+dot({{ qn("qr" ~ c, qi) }},kw{{ c }});
{% endfor %}{% endfor %}      var partv = vec{{ QPL }}<f32>({% for qi in range(QPL) %}{{ qn("part", qi) }}{% if not loop.last %}, {% endif %}{% endfor %});
{{ cluster_shuffle_sum("partv", LPQ) | indent(6, true) }} {{ emit_pair_scores() }} }
{% else %}var s:array<f32,TILE_K>;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;s[kk]=NEG_INF;var part:f32=0.0;let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}part=part+dot(qr{{ c }},vec4<f32>(k_tile[kb+{{ c }}u]));
{% endfor %}{% if useSubgroups %}{{ cluster_shuffle_sum("part", LPQ) | indent(6, true) }}
{% else %}red[tid]=part;workgroupBarrier();let cbase=tid-lane8;part=0.0;for(var j:u32=0u;j<LPQ;j=j+1u){part=part+red[cbase+j];}workgroupBarrier();
{% endif %}if(kj>=minKj&&kj<maxKj){s[kk]={{ score_expr("part") }};} }
{% endif %}{{ emit_tile_softmax() }}
{% if QPL > 1 %}    let tileInterior = {% for qi in range(QPL) %}(kStart >= {{ qn("minKj", qi) }} && kStart + TILE_K <= {{ qn("maxKj", qi) }}){% if not loop.last %} && {% endif %}{% endfor %};
{% for c in range(VALUE_SLICE_COUNT) %}{
{% for qi in range(QPL) %}var {{ qn("acc", qi) }}={{ qn("o" ~ c, qi) }}*{{ qn("corr", qi) }};
{% endfor %}if(tileInterior){for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let vw=vec4<f32>(v_tile[kk*{{ VALUE_DIM_V4_NAME }}+lane8*{{ VALUE_SLICE_NAME }}+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("acc", qi) }}={{ qn("acc", qi) }}+{{ qn("s", qi) }}[kk]*vw;
{% endfor %}} }else{for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let vw=vec4<f32>(v_tile[kk*{{ VALUE_DIM_V4_NAME }}+lane8*{{ VALUE_SLICE_NAME }}+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("acc", qi) }}={{ qn("acc", qi) }}+{{ qn("s", qi) }}[kk]*select(vec4<f32>(),vw,kStart+kk>={{ qn("minKj", qi) }}&&kStart+kk<{{ qn("maxKj", qi) }});
{% endfor %}} }
{% for qi in range(QPL) %}{{ qn("o" ~ c, qi) }}={{ qn("acc", qi) }};
{% endfor %}}
{% endfor %}{% else %}let tileInterior=kStart>=minKj&&kStart+TILE_K<=maxKj;
{% for c in range(VALUE_SLICE_COUNT) %}{var acc=o{{ c }}*corr;if(tileInterior){for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){acc=acc+s[kk]*vec4<f32>(v_tile[kk*{{ VALUE_DIM_V4_NAME }}+lane8*{{ VALUE_SLICE_NAME }}+{{ c }}u]);} }else{for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){acc=acc+s[kk]*select(vec4<f32>(),vec4<f32>(v_tile[kk*{{ VALUE_DIM_V4_NAME }}+lane8*{{ VALUE_SLICE_NAME }}+{{ c }}u]),kStart+kk>=minKj&&kStart+kk<maxKj);} }o{{ c }}=acc;}
{% endfor %}{% endif %}kStart=kStart+TILE_K;}
{% macro attention_value(c, qi) %}select({{ qn("o" ~ c, qi) }} * inv, vec4<f32>(), zeroSupport){% endmacro %}{% for qi in range(QPL) %}if({{ qn("qValid", qi) }}){let outBase4=(b*{{ QSEQ }}+{{ qn("qIdx", qi) }})*Q_HIDDEN_V4+h*{{ VALUE_DIM_V4_NAME }}+lane8*{{ VALUE_SLICE_NAME }};let inv=select(0.0,1.0/{{ qn("l", qi) }},{{ qn("l", qi) }}>0.0);let zeroSupport={{ qn("l", qi) }}==0.0;
{% for c in range(VALUE_SLICE_COUNT) %}{% if HAS_OUTPUT_GATE %}{let attentionValue={{ attention_value(c, qi) }};let rounded=vec4<f32>(vec4<{{ scalar }}>(attentionValue));
{% if HAS_OUTPUT_GATE == "head" %}let gateValue=vec4<f32>(1.0/(1.0+exp(-f32(gate[(b*{{ QSEQ }}+{{ qn("qIdx", qi) }})*Q_HEADS+h]))));
{% else %}let gateValue=vec4<f32>(1.0)/(vec4<f32>(1.0)+exp(-vec4<f32>(gate[outBase4+{{ c }}u])));
{% endif %}{{ OUTPUT }}[outBase4+{{ c }}u]=vec4<{{ scalar }}>(rounded*gateValue);}
{% else %}    {{ OUTPUT }}[outBase4 + {{ c }}u] = vec4<{{ scalar }}>({{ attention_value(c, qi) }});{{ "\\n" }}{% endif %}{% endfor %}}
{% endfor %}}`],["prefill-flash-sgmat.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.de53490305.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set DH = (sgHeadDim / 2) | int %}{% set DF = (DH / 8) | int %}{% set CHUNKS = (DH / 32) | int %}{% set TEAMS = (sgGroup / 2) | int %}{% set windowed = windowSizeConst is defined and windowSizeConst %}{% set SPAN = "span" if windowed else "mainEnd" %}{% set KBEGIN = "kBegin + " if windowed else "" %}{% set ring = ringRows is defined and ringRows %}{% set RMASK = (" & " ~ (ringRows - 1) ~ "u") if ring else "" %}const HEAD_DIM:u32={{ sgHeadDim }}u;const GROUP:u32={{ sgGroup }}u;const DH:u32={{ DH }}u;const QKV_STRIDE:u32={{ (sgQHeads + 2 * sgKvHeads) * sgHeadDim }}u;const KV_STRIDE:u32={{ sgKvHeads * sgHeadDim }}u;const OUT_STRIDE:u32={{ sgQHeads * sgHeadDim }}u;const NK:u32=32u;const PART:u32=512u;const DIAG:u32=16u;const SC:u32=32u;const TAU:f32=8.0;const NEG_INF:f32=-3.4028234663852886e38;const SCALE:f32={{ scaling }};
{% if windowed %}const WINDOW:u32={{ windowSizeConst }}u;
{% endif %}var<workgroup>scores:array<f32,{{ TEAMS * 2 * 512 }}>;var<workgroup>probs:array<f32,{{ TEAMS * 512 }}>;var<workgroup>diag:array<f32,{{ sgGroup * 8 * 16 }}>;var<workgroup>rowScale:array<f32,{{ sgGroup * 8 }}>;var<workgroup>red:array<f32,{{ TEAMS * 64 }}>;var<workgroup>rescaleFlag:array<atomic<u32>,2>;var<workgroup>rescaleSeen:u32;
{% include "ops/_shared/subgroup-matrix-topology.5c216ddcce.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.0a6b6c163d.wgsl.jinja" %}fn addf(a:f32,b:f32)->f32{return a+b;}
{% macro row_reduce(sw, op, value) %}{% if sw %}red[teamLane]={{ value }};workgroupBarrier();{{ value }}={{ op }}({{ op }}(red[teamRow],red[teamRow+1u]),{{ op }}(red[teamRow+2u],red[teamRow+3u]));workgroupBarrier();
{% else %}{{ value }}={{ op }}({{ value }},subgroupShuffleXor({{ value }},1u));{{ value }}={{ op }}({{ value }},subgroupShuffleXor({{ value }},2u));
{% endif %}{% endmacro %}{% macro stage_out(sw, r, c) %}{% for n in range(4) %}{{ matrixStore(sw, "row_major", "&scores", "stageAt + " ~ (n * 8) ~ "u", "o" ~ r ~ "_" ~ (c * 4 + n), "SC") }}
{% endfor %}{% endmacro %}{% macro stage_in(sw, r, c) %}{% for n in range(4) %}o{{ r }}_{{ c * 4 + n }}={{ matrixLoad(sw, "result", "f32", "row_major", "&scores", "stageAt + " ~ (n * 8) ~ "u", "SC") }};
{% endfor %}{% endmacro %}{% macro kernel_body(sw) %}let team=matrixRole/2u;let h=matrixRole%2u;let kvHead=workgroup_id.y;let headBase=kvHead*GROUP+team*2u;let seq=params.seq_len;let fullFrags=seq/8u;let q0=select(workgroup_id.x*8u,seq-8u,workgroup_id.x==fullFrags);let qFresh=select(0u,fullFrags*8u-q0,workgroup_id.x==fullFrags);let r0=params.past_len+q0;let mainEnd=r0&~7u;let diagCount=r0+8u-mainEnd;let lane={{ "matrixSoftwareLane" if sw else "subgroupLane" }};let teamIndex=h*32u+lane;let srow=teamIndex/4u;let slane=teamIndex%4u;
{% if sw %}let teamLane=team*64u+teamIndex;let teamRow=team*64u+srow*4u;
{% endif %}let rowSlot=(team*2u+srow/8u)*8u+srow%8u;let scoreBase=team*2u*PART;let probBase=team*PART;let stageAt=scoreBase+h*PART;let kvBase=kvHead*HEAD_DIM;let dBase=h*DH;for(var p=teamIndex;p<2u*8u*DIAG;p+=64u){let hr=p/(8u*DIAG);let rr=(p/DIAG)%8u;let kk=p%DIAG;let key=mainEnd+kk;var s=NEG_INF;if(kk<diagCount&&key<=r0+rr){var acc=0.0;let qAt=(q0+rr)*QKV_STRIDE+(headBase+hr)*HEAD_DIM;let kAt=kvBase+{{ ("(key" ~ RMASK ~ ")") if ring else "key" }}*KV_STRIDE;for(var d=0u;d<HEAD_DIM;d++){acc=fma(qkv[qAt+d],cache_keys[kAt+d],acc);}s=acc*SCALE;}diag[(team*2u+hr)*8u*DIAG+rr*DIAG+kk]=s;}workgroupBarrier();var mUsed=NEG_INF;var l=0.0;{let at=rowSlot*DIAG+slane;let d0=diag[at];let d1=diag[at+4u];let d2=diag[at+8u];let d3=diag[at+12u];var mx=max(max(d0,d1),max(d2,d3));{{ row_reduce(sw, "max", "mx") }} mUsed=mx;let p0=exp(d0-mUsed);let p1=exp(d1-mUsed);let p2=exp(d2-mUsed);let p3=exp(d3-mUsed);var sum=(p0+p1)+(p2+p3);{{ row_reduce(sw, "addf", "sum") }} l=sum;diag[at]=p0;diag[at+4u]=p1;diag[at+8u]=p2;diag[at+12u]=p3;}workgroupBarrier();
{% for r in range(2) %}{% for n in range(DF) %}var o{{ r }}_{{ n }}:{{ matrixType(sw, "result", "f32") }};
{% endfor %}{% endfor %}{% for r in range(2) %}{% for c in range(CHUNKS) %}for(var e=lane;e<8u*SC;e+=32u){let rr=e/SC;let d=dBase+{{ c * 32 }}u+e%SC;var acc=0.0;for(var kk=0u;kk<diagCount;kk++){let key=mainEnd+kk;if(key<=r0+rr){acc=fma(diag[(team*2u+{{ r }}u)*8u*DIAG+rr*DIAG+kk],cache_values[kvBase+{{ ("(key" ~ RMASK ~ ")") if ring else "key" }}*KV_STRIDE+d],acc);} }scores[stageAt+e]=acc;}workgroupBarrier();{{ stage_in(sw, r, c) }} workgroupBarrier();
{% endfor %}{% endfor %}{% if windowed %}let kBegin=select(0u,(r0+1u-WINDOW)&~7u,r0+1u>WINDOW);let span=mainEnd-kBegin;
{% endif %}let fullTiles={{ SPAN }}/NK;let tileCount=({{ SPAN }}+NK-1u)/NK;let slide={{ SPAN }}>=NK;for(var t=0u;t<tileCount;t++){let tail=t==fullTiles&&slide;let k0={{ KBEGIN }}select(t*NK,{{ SPAN }}-NK,tail);let kFresh=select(0u,{{ KBEGIN }}fullTiles*NK-k0,tail);let kf=select(min(4u,(mainEnd-k0)/8u),4u,slide);
{% for r in range(2) %}{% for j in range(4) %}var s{{ r }}_{{ j }}:{{ matrixType(sw, "result", "f32") }};
{% endfor %}{% endfor %}{% for guarded in [false, true] %}{{ "if (slide) {" if not guarded else "} else {" }} for(var k=dBase;k<dBase+DH;k+=8u){
{% for r in range(2) %}let a{{ r }}={{ matrixLoad(sw, "left", "f32", "row_major", "&qkv", "q0 * QKV_STRIDE + (headBase + " ~ r ~ "u) * HEAD_DIM + k", "QKV_STRIDE") }};
{% endfor %}{% for j in range(4) %}{% if guarded and j > 0 %}        if (kf > {{ j }}u) {
{% endif %}{let b={{ matrixLoad(sw, "right", "f32", "col_major", "&cache_keys", "kvBase + " ~ ("((k0 + " ~ (j * 8) ~ "u)" ~ RMASK ~ ")" if ring else "(k0 + " ~ (j * 8) ~ "u)") ~ " * KV_STRIDE + k", "KV_STRIDE") }};
{% for r in range(2) %}s{{ r }}_{{ j }}={{ matrixMma(sw, "f32", "a" ~ r, "b", "s" ~ r ~ "_" ~ j) }};
{% endfor %}}
{% if guarded and j > 0 %}        }
{% endif %}{% endfor %}}
{% endfor %}}
{% for r in range(2) %}{% for j in range(4) %}{{ matrixStore(sw, "row_major", "&scores", "stageAt + " ~ (r * 256 + j * 8) ~ "u", "s" ~ r ~ "_" ~ j, "NK") }}
{% endfor %}{% endfor %}workgroupBarrier();var sv:array<f32,8>;var mx=NEG_INF;for(var i=0u;i<8u;i++){let j=slane+4u*i;let at=scoreBase+srow*NK+j;sv[i]=select(NEG_INF,(scores[at]+scores[at+PART])*SCALE,j>=kFresh&&j<kf*8u{{ " && k0 + j + WINDOW > r0 + srow % 8u" if windowed else "" }});mx=max(mx,sv[i]);} {{ row_reduce(sw, "max", "mx") }} var alpha=1.0;if(mx>mUsed+TAU){alpha=exp(mUsed-mx);mUsed=mx;atomicOr(&rescaleFlag[t&1u],1u);}var sum=0.0;for(var i=0u;i<8u;i++){let pr=exp(sv[i]-mUsed);probs[probBase+srow*NK+slane+4u*i]=pr;sum+=pr;} {{ row_reduce(sw, "addf", "sum") }} l=l*alpha+sum;if(slane==0u){rowScale[rowSlot]=alpha;}
{% if not sw %}workgroupBarrier();
{% endif %}if(local_idx==0u){rescaleSeen=atomicLoad(&rescaleFlag[t&1u]);atomicStore(&rescaleFlag[(t+1u)&1u],0u);}if(workgroupUniformLoad(&rescaleSeen)!=0u){
{% for r in range(2) %}{% for c in range(CHUNKS) %}{{ stage_out(sw, r, c) }} workgroupBarrier();for(var e=lane;e<8u*SC;e+=32u){scores[stageAt+e]*=rowScale[(team*2u+{{ r }}u)*8u+e/SC];}workgroupBarrier();{{ stage_in(sw, r, c) }} workgroupBarrier();
{% endfor %}{% endfor %}}
{% for kk in range(4) %}{% if kk > 0 %}    if (slide || kf > {{ kk }}u) {
{% endif %}{
{% for r in range(2) %}let p{{ r }}={{ matrixLoad(sw, "left", "f32", "row_major", "&probs", "probBase + " ~ (r * 256 + kk * 8) ~ "u", "NK") }};
{% endfor %}let vAt=kvBase+{{ ("((k0 + " ~ (kk * 8) ~ "u)" ~ RMASK ~ ")") if ring else "(k0 + " ~ (kk * 8) ~ "u)" }}*KV_STRIDE+dBase;
{% for n in range(DF) %}{let v={{ matrixLoad(sw, "right", "f32", "row_major", "&cache_values", "vAt + " ~ (n * 8) ~ "u", "KV_STRIDE") }};
{% for r in range(2) %}o{{ r }}_{{ n }}={{ matrixMma(sw, "f32", "p" ~ r, "v", "o" ~ r ~ "_" ~ n) }};
{% endfor %}}
{% endfor %}}
{% if kk > 0 %}    }
{% endif %}{% endfor %}}if(slane==0u){rowScale[rowSlot]=select(1.0/l,0.0,l==0.0);}workgroupBarrier();
{% for r in range(2) %}{% for c in range(CHUNKS) %}{{ stage_out(sw, r, c) }} workgroupBarrier();for(var e=lane;e<8u*SC;e+=32u){let rr=e/SC;if(rr>=qFresh){let at=(q0+rr)*OUT_STRIDE+(headBase+{{ r }}u)*HEAD_DIM+dBase+{{ c * 32 }}u+e%SC;let value=scores[stageAt+e]*rowScale[(team*2u+{{ r }}u)*8u+rr];attn_out[at]=value*(1.0/(1.0+exp(-gate[(q0+rr)*{{ sgQHeads }}u+headBase+{{ r }}u])));} }workgroupBarrier();
{% endfor %}{% endfor %}{% endmacro %}@compute@workgroup_size({{ sgGroup * 32 }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)workgroup_id:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,@builtin(subgroup_size)sg_size:u32,@builtin(subgroup_invocation_id)subgroupLane:u32){let matrixRole=claimMatrixSubgroupRole(sg_size,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,GROUP);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ kernel_body(false) }} }else{let matrixRole=local_idx/32u;{{ kernel_body(true) }} } }`]],assumes:[["ops/_shared/attn-flash-decode-splitk.wgsl.jinja",{cacheVec4:"vec4<f32>",normWeightVec4:"vec4<f32>",usesF16:!1}],["ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",{scalar:"f32"}],["prefill-attention.wgsl.jinja",{outScalar:"f32"}],["ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",{usesF16:!1,scalar:"f32"}]]}],["com.xenova.CandidateArgMax",{manifest:{domain:"com.xenova",name:"CandidateArgMax",sinceVersion:1,inputs:{valuesT:{onnx:"CandidateValues",dtype:"float32",rank:1},indicesT:{onnx:"CandidateIndices",dtype:"uint32",rank:1}},outputs:{outT:{onnx:"Output",dtype:"uint32",rank:1}},args:{count:{kind:"u32",onnx:"candidate_count"},outputOffset:{kind:"u32",onnx:"output_offset",required:!1}},tunables:{WORKGROUP_SIZE:{default:256},CANDIDATES_PER_SEGMENT:{default:4096},SPLIT_MIN_COUNT:{default:16384}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",outputOffset:"args.outputOffset if args.outputOffset else 0",workgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",powerOfTwoWorkgroup:"workgroupSize == 1 or workgroupSize == 2 or workgroupSize == 4 or workgroupSize == 8 or workgroupSize == 16 or workgroupSize == 32 or workgroupSize == 64 or workgroupSize == 128 or workgroupSize == 256",subgroupReduction:'device.features.has("subgroups")',portableWorkgroupStorageOk:"workgroupSize * 12 <= device.limits.maxComputeWorkgroupStorageSize",subgroupWorkgroupStorageOk:"workgroupSize * 12 <= device.limits.maxComputeWorkgroupStorageSize",reductionStorageOk:"subgroupWorkgroupStorageOk if subgroupReduction else portableWorkgroupStorageOk",packedSplit:"ceilDiv(args.count, max(1, tunables.CANDIDATES_PER_SEGMENT))"},when:["ranks.valuesT == 1","ranks.indicesT == 1","ranks.outT == 1",'tensorDtypes.valuesT == "float32"','tensorDtypes.indicesT == "uint32"','tensorDtypes.outT == "uint32"',"args.count > 0","dim(shapes.valuesT, 0) >= args.count","dim(shapes.indicesT, 0) >= args.count","dim(shapes.outT, 0) > outputOffset","powerOfTwoWorkgroup"],variants:[{id:"reduce",priority:0,when:["reductionStorageOk"],derive:{numBlocks:"args.count",useSubgroups:"subgroupReduction",trackValidity:!0,TOKEN_OUT_LEN:"outputOffset + 1"},passes:[{id:"main",name:"CandidateArgMax",shader:"ops/_shared/matmul-argmax-reduce.wgsl.jinja",bindings:[{arg:"valuesT",name:"best_values_in"},{arg:"indicesT",name:"best_ids_in"},{arg:"outT",name:"token_out",length:"$TOKEN_OUT_LEN"}],dispatch:{x:1},subgroupCollectivesWidth:"portable"}]}]},assets:[["ops/_shared/matmul-argmax-reduce.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const NUM_BLOCKS:u32={{ numBlocks }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const OUTPUT_OFFSET:u32={{ outputOffset }}u;
{% include "ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja" %}const INVALID_INDEX: u32 = 0xffffffffu;
{% if useSubgroups %}{% include "ops/_shared/float-neg-inf.29b2a70cbe.wgsl.jinja" %}{% endif %}{% if useSubgroups %}var<workgroup>partial_values:array<f32,{{ workgroupSize }}>;var<workgroup>partial_ids:array<u32,{{ workgroupSize }}>;var<workgroup>partial_valid:array<u32,{{ workgroupSize }}>;
{% else %}var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>best_valid:array<u32,{{ workgroupSize }}>;
{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var value=0.0;var index=INVALID_INDEX;var has_best=false;for(var i=lid;i<NUM_BLOCKS;i=i+WORKGROUP_SIZE){let v=best_values_in[i];let id=best_ids_in[i];if(!is_nan_f32(v)&&(!has_best||v>value||(v==value&&id<index))){value=v;index=id;has_best=true;} }
{% if useSubgroups %}let sg_best=subgroupMax(select(negative_infinity(),value,has_best));let sg_token=subgroupMin(select(INVALID_INDEX,index,has_best&&value==sg_best));let sg_valid=subgroupMax(select(0u,1u,has_best));let elected=subgroupElect();partial_values[lid]=select(negative_infinity(),sg_best,elected);partial_ids[lid]=select(INVALID_INDEX,sg_token,elected);partial_valid[lid]=select(0u,sg_valid,elected);workgroupBarrier();if(lid==0u){var best_value=0.0;var best_index=INVALID_INDEX;var has_valid=false;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){let v=partial_values[i];let id=partial_ids[i];if(partial_valid[i]!=0u&&(!has_valid||v>best_value||(v==best_value&&id<best_index))){best_value=v;best_index=id;has_valid=true;} }token_out[OUTPUT_OFFSET]=best_index;}
{% else %}best_values[lid]=value;best_indices[lid]=index;best_valid[lid]=select(0u,1u,has_best);workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){let other_valid=best_valid[lid+stride];let other_value=best_values[lid+stride];let other_index=best_indices[lid+stride];if(other_valid!=0u&&(best_valid[lid]==0u||other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid]))){best_values[lid]=other_value;best_indices[lid]=other_index;best_valid[lid]=1u;} }workgroupBarrier();stride=stride/2u;}if(lid==0u){token_out[OUTPUT_OFFSET]=best_indices[0];}
{% endif %}}`]]}],["com.xenova.LlamaDecodeConvInProj",{manifest:{domain:"com.xenova",name:"LlamaDecodeConvInProj",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"H",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},weightsT:{onnx:"NormWeights",dtype:"W",rank:1,optional:!0},convWeightT:{onnx:"ConvWeight",dtype:"float32",rank:1,optional:!0},convBiasT:{onnx:"ConvBias",dtype:"float32",rank:1,optional:!0},convPastT:{onnx:"ConvPast",dtype:"float32",rank:1,optional:!0}},outputs:{convProjT:{onnx:"ConvProj",dtype:"float32",rank:1,optional:!0,shape:["args.projSize if args.projSize else 3 * args.hiddenSize"]},convOutT:{onnx:"ConvOut",dtype:"float32",rank:1,optional:!0,shape:["args.convDim"]},convPresentT:{onnx:"ConvPresent",dtype:"float32",rank:1,optional:!0,shape:["args.convDim * 3"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},normOffset:{kind:"u32",onnx:"input_norm_offset",required:!1},preNormed:{kind:"u32",onnx:"pre_normed_input",required:!1},convInOffset:{kind:"u32",onnx:"conv_in_offset",required:!1},projSize:{kind:"u32",onnx:"proj_out_size",required:!1},format:{kind:"string",oneOf:["q4_0","q4_k","q4_k_native_compact","q5_k","q6_k","q2_k","q3_k","q8_0","iq","lut32","lut16","q5_0"]},fuseConv:{kind:"u32",onnx:"fuse_depthwise_conv",required:!1},convZ:{kind:"u32",onnx:"conv_region_start_row",required:!1},convDim:{kind:"u32",onnx:"conv_channels",required:!1},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]}},typeConstraints:{H:["float32","float16"],S:["float32","float16","uint32"],W:["float32","float16"]},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",projSize:"args.projSize if args.projSize else 3 * args.hiddenSize",portableMr4Preferred:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32',q4ProjectionMode:"present.hiddenT and present.bitsT and present.scalesT and present.convProjT and args.convInOffset is defined and ((args.fuseConv == 1 and present.convWeightT and present.convBiasT and present.convPastT and present.convPresentT and present.convOutT) or (not args.fuseConv and not present.convWeightT and not present.convBiasT and not present.convPastT and not present.convPresentT and not present.convOutT))",convProjBlocks:"(args.convInOffset + projSize * (args.hiddenSize / 32)) if q4ProjectionMode else 0",convProjSuperblocks:"ceilDiv(convProjBlocks, 8)",q4knc:'1 if fmt == "q4_k_native_compact" else 0',q4knCompact:"q4knc",kSuper:"q4knc or q6k or q5k or q3k or q2k",kSuperGeometryOk:"(not kSuper) or (args.convInOffset % 8 == 0 and dim(shapes.bitsT, 0) % 4 == 0)",convBitsNeeded:"convProjSuperblocks * 52 if q6k else convProjSuperblocks * 44 if q5k else convProjSuperblocks * 28 if q3k else convProjSuperblocks * 20 if q2k else convProjBlocks * (5 if q5 else quantBits)",convScalesNeeded:"convProjSuperblocks * 5 if q4knc else convProjSuperblocks if q6k else convProjSuperblocks * 2 if q5k else convProjSuperblocks if q3k else convProjSuperblocks * 2 if q2k else convProjBlocks * (2 if (hasMin or iq or lut16) else 1)",baseContract:'q4ProjectionMode and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.convProjT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if q4knc else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and tensorDtypes.convProjT == "float32" and args.hiddenSize > 0 and args.hiddenSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and kSuperGeometryOk and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.convProjT, 0) == projSize and dim(shapes.bitsT, 0) >= convBitsNeeded and dim(shapes.scalesT, 0) >= convScalesNeeded',useSubgroups:"wave32SubgroupsUsable",sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',scaleScalar:"dtypes.S",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",blocksPerRow:"args.hiddenSize / 32",totalOutputs:"args.projSize if args.projSize else 3 * args.hiddenSize",convInOffset:"args.convInOffset",lutId:"args.lut if args.lut else 0",workgroupSize:"subgroupCoveringWorkgroup"},when:["baseContract"],bindings:{hidden:{arg:"hiddenT",elementType:"$hiddenVec4"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},conv_proj:{arg:"convProjT",elementType:"f32"}},variants:[{id:"q4_prenormed",priority:0,when:["not args.fuseConv","args.preNormed == 1","f16Ok(tensorDtypes.hiddenT) and f16Ok(tensorDtypes.scalesT)","(not useSubgroups and not sgRuntime) or 4 * (128 if (0) else workgroupSize) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{preNormed:"1",multiRow:"0",regRows:"1",fuseConv:"0",convZ:"0",convDim:"0"},passes:[{id:"main",name:"LlamaDecodeConvInProj",shader:"decode-conv-in-proj-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","conv_proj"],dispatch:{x:"args.projSize if args.projSize else 3 * args.hiddenSize"},subgroupCollectivesWidth:"portable"}]},{id:"q4_prenormed_mr2",priority:3,when:["not args.fuseConv","projSize % 8 == 0","args.preNormed == 1","f16Ok(tensorDtypes.hiddenT) and f16Ok(tensorDtypes.scalesT)","(not useSubgroups and not sgRuntime) or 4 * (128 if (1) else workgroupSize) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],demoteWhen:["portableMr4Preferred and projSize % 16 == 0"],derive:{preNormed:"1",multiRow:"1",regRows:"2",fuseConv:"0",convZ:"0",convDim:"0"},passes:[{id:"main",name:"LlamaDecodeConvInProj",shader:"decode-conv-in-proj-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","conv_proj"],dispatch:{x:"ceil((args.projSize if args.projSize else 3 * args.hiddenSize) / 8)"},subgroupCollectivesWidth:"portable"}]},{id:"q4_prenormed_mr4",priority:2,when:["not args.fuseConv","projSize % 16 == 0","args.preNormed == 1","f16Ok(tensorDtypes.hiddenT) and f16Ok(tensorDtypes.scalesT)","(not useSubgroups and not sgRuntime) or 4 * (128 if (1) else workgroupSize) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{preNormed:"1",multiRow:"1",regRows:"4",fuseConv:"0",convZ:"0",convDim:"0"},passes:[{id:"main",name:"LlamaDecodeConvInProj",shader:"decode-conv-in-proj-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","conv_proj"],dispatch:{x:"ceil((args.projSize if args.projSize else 3 * args.hiddenSize) / 16)"},subgroupCollectivesWidth:"portable"}]}]},assets:[["decode-conv-in-proj-norm-q4.wgsl.jinja",`{% include "ops/_shared/packed-4x8-compat.cf3c88fe57.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const BPR:u32={{ blocksPerRow }}u;
{% if not multiRow %}const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;
{% endif %}const W_IN_Q4:u32={{ convInOffset }}u;
{% if not multiRow %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% include "ops/_shared/subgroup-runtime-fold.f33ebc1f56.wgsl.jinja" %}{% if useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 128 if multiRow else workgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}{% if multiRow %}{% set partialsRowVector = true %}var<workgroup>partials4:array<vec{{ regRows }}<f32>,128>;
{% else %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% include "ops/_shared/q4-dequant-helpers.245c302430.wgsl.jinja" %}{% if multiRow %}fn store_row(r:u32,total:f32){conv_proj[r]=total;}@compute@workgroup_size(128){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;let dim0=(wid.x*4u+cluster)*{{ regRows }}u;let base0=W_IN_Q4+dim0*BPR;
{% include "ops/_shared/q4-normed-projection-multirow.11b6390ad1.wgsl.jinja" %}{% include "ops/_shared/projection-multirow-reduce.367c218e5c.wgsl.jinja" %}  if (lane == 0u) {
{% for r in range(regRows) %}store_row(dim0+{{ r }}u,total{{ r }});
{% endfor %}} }
{% else %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let index=wid.x;let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(index>=TOTAL_OUTPUTS){return;}let row_block_base=W_IN_Q4+index*BPR;
{% include "ops/_shared/q4-normed-projection-single-row.ce563d4459.wgsl.jinja" %}  if (lid == 0u) {
conv_proj[index]=acc_total;} }
{% endif %}`]],assumes:[["decode-conv-in-proj-norm-q4.wgsl.jinja",{quantBits:4,q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q6k:0,q5k:0,q4knCompact:1,lutId:0}]]}],["com.xenova.LlamaDecodeGateUp",{manifest:{domain:"com.xenova",name:"LlamaDecodeGateUp",sinceVersion:1,inputs:{normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},hiddenT:{onnx:"Hidden",dtype:"H",rank:1,optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},NormWeights:{dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},upBitsT:{onnx:"UpBits",dtype:"uint32",rank:1,optional:!0},upScalesT:{onnx:"UpScales",dtype:"U",rank:1,optional:!0},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0},upGridT:{onnx:"UpGrid",dtype:"uint32",rank:1,optional:!0}},outputs:{intermediateT:{onnx:"Intermediate",dtype:"I",rank:1,shape:["tokenRows * args.intermediateSize"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size",required:!0},intermediateSize:{kind:"u32",onnx:"intermediate_size",required:!0},gateOffset:{kind:"u32",onnx:"gate_proj_offset",required:!0},upOffset:{kind:"u32",onnx:"up_proj_offset",required:!0},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},normOffset:{kind:"u32",onnx:"input_norm_offset",required:!1},activation:{kind:"string",required:!1,oneOf:["silu","fast_silu"]},strategy:{kind:"string",onnx:"schedule_strategy",required:!1,oneOf:["generic","latency"]},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k","q1_0"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},upFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k"]},upLut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},tokens:{kind:"u32",required:!1},upIqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]}},typeConstraints:{N:["float32","float16"],H:["float32","float16"],S:["float32","float16","uint32"],W:["float32","float16","uint32"],I:["float32","float16"],U:["float32","float16","uint32"]},derive:{weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',tokenRows:"args.tokens if args.tokens is defined else 1",fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',iqRawSel:"args.iqRaw if args.iqRaw else 0",upIqRawSel:"args.upIqRaw if args.upIqRaw else 0",gateUpMaxBlocks:"max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 32), args.upOffset + args.intermediateSize * (args.hiddenSize / 32)) if (args.gateOffset is defined and args.upOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",gateUpSuperblocks:"ceilDiv(gateUpMaxBlocks, 8)",gateMaxBlocks:"args.gateOffset + args.intermediateSize * (args.hiddenSize / 32) if (args.gateOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",upMaxBlocks:"args.upOffset + args.intermediateSize * (args.hiddenSize / 32) if (args.upOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",gateSuperblocks:"ceilDiv(gateMaxBlocks, 8)",upSuperblocks:"ceilDiv(upMaxBlocks, 8)",q4kncSel:'1 if fmt == "q4_k_native_compact" else 0',upFmt:"args.upFormat if args.upFormat else fmt",upQuantBits:'8 if (upFmt == "q8_0" or upFmt == "iq") else 4',upHasMin:'1 if upFmt == "q4_k" else 0',upIq:'1 if upFmt == "iq" else 0',upQ6k:'1 if upFmt == "q6_k" else 0',upQ5k:'1 if upFmt == "q5_k" else 0',upQ3k:'1 if upFmt == "q3_k" else 0',upQ2k:'1 if upFmt == "q2_k" else 0',upQ4knc:'1 if upFmt == "q4_k_native_compact" else 0',upLut16:'1 if upFmt == "lut16" else 0',upLut2:'1 if (upFmt == "lut2_32" or upFmt == "lut2_16") else 0',upLutc:'1 if (upFmt == "lut32_compact" or upFmt == "lut16_compact" or upLut2 == 1) else 0',upLutc16:'1 if (upFmt == "lut16_compact" or upFmt == "lut2_16") else 0',upLutcWords:"1 if (upLut2 == 1 and args.upLut == 9) else (5 if upLutc16 == 1 else 3)",gateBitsNeeded:"(gateSuperblocks * 14 if iqRawSel == 8 else gateSuperblocks * 12 if iqRawSel == 7 else gateSuperblocks * 27 if iqRawSel == 4 else gateSuperblocks * 18 if iqRawSel == 6 else gateSuperblocks * 20 if iqRawSel == 5 else gateMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else gateMaxBlocks * 3) if iqRawSel else gateSuperblocks * 52 if q6k else gateSuperblocks * 44 if q5k else gateSuperblocks * 28 if q3k else gateSuperblocks * 20 if q2k else gateMaxBlocks * (2 if lut2 else 4) if (lutc or q4kncSel) else gateMaxBlocks * quantBits",gateScalesNeeded:"gateSuperblocks if iqRawSel else gateSuperblocks if (q6k or q3k) else gateSuperblocks * 2 if (q5k or q2k) else gateSuperblocks * lutcWords if lutc else gateSuperblocks * 5 if q4kncSel else gateMaxBlocks * (2 if (hasMin or iq or lut16) else 1)",upBitsNeeded:"(upSuperblocks * 14 if upIqRawSel == 8 else upSuperblocks * 12 if upIqRawSel == 7 else upSuperblocks * 27 if upIqRawSel == 4 else upSuperblocks * 18 if upIqRawSel == 6 else upSuperblocks * 20 if upIqRawSel == 5 else upMaxBlocks * 2 if (upIqRawSel == 1 or upIqRawSel == 2) else upMaxBlocks * 3) if upIqRawSel else upSuperblocks * 52 if upQ6k else upSuperblocks * 44 if upQ5k else upSuperblocks * 28 if upQ3k else upSuperblocks * 20 if upQ2k else upMaxBlocks * (2 if upLut2 else 4) if (upLutc or upQ4knc) else upMaxBlocks * upQuantBits",upScalesNeeded:"upSuperblocks if upIqRawSel else upSuperblocks if (upQ6k or upQ3k) else upSuperblocks * 2 if (upQ5k or upQ2k) else upSuperblocks * upLutcWords if upLutc else upSuperblocks * 5 if upQ4knc else upMaxBlocks * (2 if (upHasMin or upIq or upLut16) else 1)",gateScalesDtypeOk:'tensorDtypes.scalesT == "uint32" if (lutc or q4kncSel) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',upScalesDtypeOk:'tensorDtypes.upScalesT == "uint32" if (upLutc or upQ4knc) else (tensorDtypes.upScalesT == "float32" or tensorDtypes.upScalesT == "float16")',gateNibbleFam:'0 if iqRawSel else (1 if (fmt == "q4_0" or fmt == "q4_k" or fmt == "q4_k_native_compact") else 0)',upNibbleFam:'0 if upIqRawSel else (1 if (upFmt == "q4_0" or upFmt == "q4_k" or upFmt == "q4_k_native_compact") else 0)',splitRoute:'present.bitsT and present.scalesT and present.upBitsT and present.upScalesT and args.upFormat is defined and fmt != "q1_0" and upFmt != "q1_0" and fmt != "q5_0" and upFmt != "q5_0" and fmt != "q8_0_pair" and upFmt != "q8_0_pair" and (iqRawSel == 0 or iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 3 or iqRawSel == 4 or iqRawSel == 5 or iqRawSel == 6 or iqRawSel == 7 or iqRawSel == 8) and (upIqRawSel == 0 or upIqRawSel == 1 or upIqRawSel == 2 or upIqRawSel == 3 or upIqRawSel == 4 or upIqRawSel == 5 or upIqRawSel == 6 or upIqRawSel == 7 or upIqRawSel == 8)',mixedRawPair:"iqRawSel != 0 and upIqRawSel != 0 and iqRawSel != upIqRawSel",firstRawSel:"iqRawSel if iqRawSel else upIqRawSel",gridElemsNeeded:"1024 if firstRawSel == 6 else 4096 if (firstRawSel == 1 or firstRawSel == 7 or firstRawSel == 8) else 2048 if firstRawSel == 5 else 512 if firstRawSel == 4 else 512 if firstRawSel == 2 else 256",upGridElemsNeeded:"1024 if upIqRawSel == 6 else 4096 if (upIqRawSel == 1 or upIqRawSel == 7 or upIqRawSel == 8) else 2048 if upIqRawSel == 5 else 512 if upIqRawSel == 4 else 512 if upIqRawSel == 2 else 256",splitPackContract:'splitRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.upBitsT == 1 and ranks.upScalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.upBitsT == "uint32" and gateScalesDtypeOk and upScalesDtypeOk and (args.hiddenSize % 256 == 0 if (lutc or q4kncSel or upLutc or upQ4knc or q2k or upQ2k) else args.hiddenSize % 32 == 0) and (args.gateOffset % 8 == 0 if (q6k or q5k or q3k or q2k or lutc or q4kncSel) else true) and (args.upOffset % 8 == 0 if (upQ6k or upQ5k or upQ3k or upQ2k or upLutc or upQ4knc) else true) and (args.lut is defined and args.lut != 0 if lutc else true) and (args.upLut is defined and args.upLut != 0 if upLutc else true) and (quantBits == 4 or quantBits == 8) and (upQuantBits == 4 or upQuantBits == 8) and (dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 if iqRawSel == 0 else true) and (dim(shapes.upBitsT, 0) % (2 if upLut2 else 4) == 0 if upIqRawSel == 0 else true) and dim(shapes.bitsT, 0) >= gateBitsNeeded and dim(shapes.scalesT, 0) >= gateScalesNeeded and dim(shapes.upBitsT, 0) >= upBitsNeeded and dim(shapes.upScalesT, 0) >= upScalesNeeded',preNormalizedMode:"present.normedT and not present.hiddenT and args.rmsEps is undefined and args.normOffset is undefined",fusedRmsNormMode:"present.hiddenT and not present.normedT and present.weightsT and args.rmsEps is defined and args.normOffset is defined",denseRoute:"present.weightsT and not present.bitsT and not present.scalesT and not present.upBitsT and not present.upScalesT",q2Route:'present.bitsT and present.scalesT and present.upBitsT and present.upScalesT and fmt == "q2_k" and iqRawSel == 0',q4Route:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt != "q1_0" and fmt != "q4_k_native_compact" and fmt != "q5_0" and fmt != "q8_0_pair" and lutc == 0 and iqRawSel == 0',q4knCompactRoute:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt == "q4_k_native_compact" and iqRawSel == 0',lutcRoute:"present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and lutc == 1 and iqRawSel == 0",q1Route:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt == "q1_0" and iqRawSel == 0',q2PackContract:'q2Route and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.upBitsT == 1 and ranks.upScalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.upBitsT == "uint32" and tensorDtypes.scalesT == tensorDtypes.upScalesT and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) >= gateSuperblocks * 20 and dim(shapes.scalesT, 0) >= gateSuperblocks * 2 and dim(shapes.upBitsT, 0) >= upSuperblocks * 20 and dim(shapes.upScalesT, 0) >= upSuperblocks * 2',q4PackContract:'q4Route and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k) or (args.gateOffset % 8 == 0 and args.upOffset % 8 == 0)) and dim(shapes.bitsT, 0) >= (gateUpSuperblocks * 52 if q6k else gateUpSuperblocks * 44 if q5k else gateUpSuperblocks * 28 if q3k else gateUpSuperblocks * 20 if q2k else gateUpMaxBlocks * quantBits) and dim(shapes.scalesT, 0) >= (gateUpSuperblocks if (q6k or q3k) else gateUpSuperblocks * 2 if (q5k or q2k) else gateUpMaxBlocks * (2 if (hasMin or iq or lut16) else 1))',q4knCompactPackContract:'q4knCompactRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) % 4 == 0 and dim(shapes.bitsT, 0) >= gateUpMaxBlocks * 4 and dim(shapes.scalesT, 0) >= gateUpSuperblocks * 5',lutcPackContract:'lutcRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.lut is defined and args.lut != 0 and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= gateUpMaxBlocks * (2 if lut2 else 4) and dim(shapes.scalesT, 0) >= gateUpSuperblocks * lutcWords',rawBitsNeeded:"gateUpMaxBlocks * (3 if iqRawSel == 3 else 2) if iqRawSel <= 3 else gateUpSuperblocks * (27 if iqRawSel == 4 else 20 if iqRawSel == 5 else 18 if iqRawSel == 6 else 12 if iqRawSel == 7 else 14)",rawPackContract:'iqRawSel >= 1 and iqRawSel <= 8 and present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) >= rawBitsNeeded and dim(shapes.scalesT, 0) >= gateUpSuperblocks',preNormalizedContract:"preNormalizedMode and ranks.normedT == 1 and ranks.intermediateT == 1 and args.hiddenSize > 0 and args.intermediateSize > 0 and dim(shapes.normedT, 0) == tokenRows * args.hiddenSize and dim(shapes.intermediateT, 0) == tokenRows * args.intermediateSize and tokenRows >= 1 and tokenRows <= 4",fusedRmsNormShape:'fusedRmsNormMode and ranks.hiddenT == 1 and ranks.weightsT == 1 and ranks.intermediateT == 1 and (tensorDtypes.hiddenT == "float32" or tensorDtypes.hiddenT == "float16") and args.hiddenSize > 0 and args.intermediateSize > 0 and args.normOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.intermediateT, 0) == args.intermediateSize',fusedRmsNormContract:'fusedRmsNormShape and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',intermediateScalar:"dtypes.I",intermediateSize:"args.intermediateSize",bitsElem:'"u32" if lut2 else "vec4<u32>"'},bindings:{normed:{arg:"normedT",elementType:"$normedVec4"},intermediate:{arg:"intermediateT",elementType:"$intermediateScalar"},q4_bits:{arg:"bitsT",elementType:"$bitsElem"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"}},variants:[{id:"q4kn_compact",priority:1,when:["preNormalizedContract","q4knCompactPackContract",'tensorDtypes.normedT == "float32" and tensorDtypes.intermediateT == "float32"',"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],tunables:{GATE_UP_TILE_ROWS_Q4KN:{default:1}},derive:{tileRows:"tunables.GATE_UP_TILE_ROWS_Q4KN",scaleScalar:'"u32"',normedVec4:'"vec4<f32>"',lutId:0,preNormed:1,mRows:"tokenRows",q4knCompact:!0,sgSum32:!0,quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpQ4KNativeCompact",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed","q4_bits","q4_scales","intermediate"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]}]},assets:[["decode-gate-up-q4.wgsl.jinja",`{% include "ops/_shared/packed-4x8-compat.cf3c88fe57.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set gateUpSuffixes = [""] if tileRows == 1 else range(tileRows) | list %}{% set xOpen = "vec4<f32>(normed[" %}{% set xClose = "])" %}{% set gateUpTiled = false %}const BPR:u32={{ blocksPerRow }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const W_GATE_Q4:u32={{ gateOffset }}u;const W_UP_Q4:u32={{ upOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% macro act_loads() %}{% for i in range(8) %}let x{{ i }}=vec4<f32>(normed[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% endmacro %}{% include "ops/_shared/subgroup-runtime-fold.58b998281f.wgsl.jinja" %}{% set gateUpOwned = useSubgroups and not sgRuntime and workgroupSize == 32 and (subgroupIdBuiltins | default(false)) %}{% if gateUpOwned %}{% include "ops/_shared/subgroup-active-lanes.97fae981c2.wgsl.jinja" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = workgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}{% for r in gateUpSuffixes %}var<workgroup>partials_g{{ r }}:array<f32,{{ workgroupSize }}>;var<workgroup>partials_u{{ r }}:array<f32,{{ workgroupSize }}>;
{% endfor %}{% endif %}{% include "ops/_shared/silu-fn.d10dc95662.wgsl.jinja" %}{% include "ops/_shared/q4-dequant-helpers.245c302430.wgsl.jinja" %}{% if useSubgroups and sgSum32 is defined and sgSum32 %}{% endif %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_id) sg_id: u32, @builtin(num_subgroups) sg_count: u32" if gateUpOwned else (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "") }}){
{% for sgPath in (["general", "full"] if gateUpOwned else [""]) %}{% set gateUpStride = "activeCount" if sgPath == "general" else "WORKGROUP_SIZE" %}{% set gateUpLeader = "activeRank" if sgPath == "general" else "lid" %}{% if sgPath == "general" %}if(sg_count!=1u){if(sg_id==0u){ {{ active_lanes("sg_lid") }}
{% elif sgPath == "full" %}}else{
{% endif %}{% if gateUpTiled %}let j0=wid.x*TILE_ROWS;let lid={{ "activeRank" if sgPath == "general" else "lid3.x" }};
{% if (useSubgroups or sgRuntime) and not gateUpOwned %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(j0>=INTERMEDIATE_SIZE){return;}
{% for r in gateUpSuffixes %}let row{{ r }}=min(j0+{{ r }}u,INTERMEDIATE_SIZE-1u);let gate_block_base{{ r }}=W_GATE_Q4+row{{ r }}*BPR;let up_block_base{{ r }}=W_UP_Q4+row{{ r }}*BPR;var acc_g{{ r }}=0.0;var acc_u{{ r }}=0.0;
{% endfor %}{% else %}let j=wid.x;let lid={{ "activeRank" if sgPath == "general" else "lid3.x" }};
{% if (useSubgroups or sgRuntime) and not gateUpOwned %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(j>=INTERMEDIATE_SIZE){return;}let gate_block_base=W_GATE_Q4+j*BPR;let up_block_base=W_UP_Q4+j*BPR;var acc_g=0.0;var acc_u=0.0;
{% endif %}{% for m in range(1, mRows) %}{% for r in gateUpSuffixes %}var accM{{ m }}_g{{ r }}=0.0;var accM{{ m }}_u{{ r }}=0.0;
{% endfor %}{% endfor %}for(var b=lid;b<BPR;b=b+{{ gateUpStride }}){let hbase=b*8u;{{ act_loads() | trim }}
{% include "ops/_shared/q4-gate-up-accumulate-reduce.fe1bc3f98d.wgsl.jinja" %}  if ({{ gateUpLeader }} == 0u) {
{% if gateUpTiled %}{% for r in gateUpSuffixes %}if(j0+{{ r }}u<INTERMEDIATE_SIZE){intermediate[j0+{{ r }}u]={{ intermediateScalar }}(silu(g{{ r }})*u{{ r }});}
{% endfor %}{% else %}intermediate[j]={{ intermediateScalar }}(silu(g)*u);
{% endif %}}
{% if sgPath %}}
{% endif %}{% endfor %}}`]],assumes:[["decode-gate-up-q4.wgsl.jinja",{q8Pair:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,upQuantBits:4,upHasMin:0,upIq:0,upQ6k:0,upQ5k:0,upQ3k:0,upQ2k:0,upQ4knc:1,upLut16:0,upLut2:0,upLutc:0,upLutc16:0,gateNibbleFam:1,upNibbleFam:1,intermediateScalar:"f32",tileRows:1,mRows:1}]]}],["com.xenova.LlamaDecodeLmHeadArgmax",{manifest:{domain:"com.xenova",name:"LlamaDecodeLmHeadArgmax",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"float32",optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},normQ8T:{onnx:"NormQ8",dtype:"uint32",rank:1,optional:!0},normScaleT:{onnx:"NormScale",dtype:"float32",rank:1,optional:!0}},outputs:{bestValuesT:{onnx:"BestValues",dtype:"float32",rank:1,shape:["mRowsCount * effectiveNumBlocks * effectiveTopK"]},bestIdsT:{onnx:"BestIds",dtype:"uint32",rank:1,shape:["mRowsCount * effectiveNumBlocks * effectiveTopK"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},vocabSize:{kind:"u32",onnx:"vocab_size"},numBlocks:{kind:"u32",onnx:"num_blocks"},rmsEps:{kind:"f32",onnx:"rms_epsilon",required:!1},finalNormOffset:{kind:"u32",onnx:"final_norm_offset",required:!1},lmHeadOffset:{kind:"u32",onnx:"lm_head_block_offset",required:!1},topK:{kind:"u32",onnx:"candidates_per_block",required:!1},mRows:{kind:"u32",onnx:"hidden_rows",required:!1},quantBits:{kind:"u32",onnx:"packed_quant_bits",required:!1},colMajor:{kind:"u32",onnx:"block_col_major_pack",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q1_0","q8_rows","q8_0","q3_k","q2_k","q6_k","q5_k","q4_k_native_compact"]}},typeConstraints:{W:["float32","float16","uint32"],N:["float32","float16"],S:["float32","float16","uint32"]},tunables:{COL_MAJOR:{default:-1},WORKGROUP_SIZE:{default:256},COOPERATIVE_WORKGROUP_SIZE:{default:256},COOPERATIVE_SUBGROUP_SIZE:{default:32},COOPERATIVE_MIN_HIDDEN_SIZE:{default:2048},KSUPER_WORKGROUP_SIZE:{default:0}},derive:{weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',packedPresent:"present.bitsT and present.scalesT",effectiveQuantBits:"args.quantBits if args.quantBits else 4",effectiveTopK:"args.topK if args.topK else 1",effectiveNumBlocks:"args.numBlocks",mRowsRequested:"args.mRows is defined",mRowsCount:"args.mRows if args.mRows is defined else 1",mRowsShapeOk:"mRowsRequested and args.mRows >= 2 and args.mRows <= 16 and present.hiddenT and ranks.hiddenT == 2 and dim(shapes.hiddenT, 0) == args.mRows and dim(shapes.hiddenT, 1) == args.hiddenSize and dim(shapes.bestValuesT, 0) >= args.mRows * args.numBlocks * effectiveTopK and dim(shapes.bestIdsT, 0) >= args.mRows * args.numBlocks * effectiveTopK",densePresent:"present.hiddenT and present.weightsT and not packedPresent and not present.normedT and not present.normQ8T and not present.normScaleT",q1Present:'present.normedT and packedPresent and fmt == "q1_0" and not present.hiddenT and not present.weightsT',q4Present:'present.hiddenT and present.weightsT and packedPresent and fmt == "q4_0"',q4knc:'1 if fmt == "q4_k_native_compact" else 0',kSuperFmt:"1 if (q6k or q5k or q3k or q2k or q4knc) else 0",kSuperPresent:"present.hiddenT and present.weightsT and packedPresent and kSuperFmt == 1",q8BlockPresent:'present.hiddenT and present.weightsT and packedPresent and fmt == "q8_0"',kSuperSupers:"args.vocabSize * (args.hiddenSize / 256)",kSuperBitsWords:"(kSuperSupers * 52) if q6k else ((kSuperSupers * 44) if q5k else ((kSuperSupers * 32) if q4knc else ((kSuperSupers * 28) if q3k else (kSuperSupers * 20))))",kSuperScaleWords:"(kSuperSupers * 5) if q4knc else ((kSuperSupers * 2) if (q2k or q5k) else kSuperSupers)",kSuperScalesDtype:'"uint32" if q4knc else "float32"',q8Present:'present.normQ8T and present.normScaleT and packedPresent and fmt == "q8_rows" and not present.hiddenT and not present.weightsT and not present.normedT',q8NormPresent:'present.hiddenT and present.weightsT and packedPresent and fmt == "q8_rows" and not present.normQ8T',baseContract:'ranks.bestValuesT == 1 and ranks.bestIdsT == 1 and tensorDtypes.bestValuesT == "float32" and tensorDtypes.bestIdsT == "uint32" and args.hiddenSize > 0 and args.vocabSize > 0 and args.numBlocks > 0 and args.numBlocks <= device.limits.maxComputeWorkgroupsPerDimension and dim(shapes.bestValuesT, 0) >= args.numBlocks * effectiveTopK and dim(shapes.bestIdsT, 0) >= args.numBlocks * effectiveTopK and (not mRowsRequested or mRowsShapeOk)',denseContract:'densePresent and effectiveTopK == 1 and args.rmsEps is defined and args.finalNormOffset is defined and args.lmHeadOffset is defined and ranks.hiddenT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and f16Ok(dtypes.W) and args.hiddenSize % 4 == 0 and args.finalNormOffset % 4 == 0 and args.lmHeadOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and weightElements >= max(args.finalNormOffset + args.hiddenSize, args.lmHeadOffset + args.vocabSize * args.hiddenSize)',q1StorageBytes:"16 * (args.hiddenSize / 4) + 2048 if q1Present else 0",q1Contract:'q1Present and effectiveTopK == 1 and args.lmHeadOffset is defined and ranks.normedT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and (tensorDtypes.normedT == "float32" or tensorDtypes.normedT == "float16") and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.N) and f16Ok(dtypes.S) and args.hiddenSize % 128 == 0 and dim(shapes.normedT, 0) == args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 128)) * 4 and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 128) and q1StorageBytes <= device.limits.maxComputeWorkgroupStorageSize',q4Contract:'q4Present and args.lmHeadOffset is defined and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 32 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 32)) * effectiveQuantBits and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 32)',q8Contract:'q8Present and effectiveTopK == 1 and ranks.normQ8T == 1 and ranks.normScaleT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.normQ8T == "uint32" and tensorDtypes.normScaleT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and args.hiddenSize % 4 == 0 and dim(shapes.normQ8T, 0) == args.hiddenSize / 4 and dim(shapes.normScaleT, 0) == 1 and dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4) and dim(shapes.scalesT, 0) >= args.vocabSize',q8NormContract:'q8NormPresent and effectiveTopK == 1 and args.rmsEps is defined and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.weightsT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and f16Ok(dtypes.W) and args.hiddenSize % 4 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4) and dim(shapes.scalesT, 0) >= args.vocabSize',workgroupSize:"tunables.WORKGROUP_SIZE",cooperativeWorkgroupSize:"tunables.COOPERATIVE_WORKGROUP_SIZE",workgroupStorageBytes:"args.hiddenSize * 4 + workgroupSize * 8 + (args.hiddenSize / 8 if effectiveQuantBits == 2 else 0) if q4Present else 0",geometryOk:"q4Present and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and workgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",cooperativeNumSubgroups:"cooperativeWorkgroupSize / max(1, tunables.COOPERATIVE_SUBGROUP_SIZE)",cooperativeStorageBytes:"args.hiddenSize * 4 + cooperativeWorkgroupSize * 8 + 8 if q4Present else 0",fixedCooperativeSubgroup:'q4Present and device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id") and tunables.COOPERATIVE_SUBGROUP_SIZE > 0 and cooperativeWorkgroupSize % max(1, tunables.COOPERATIVE_SUBGROUP_SIZE) == 0 and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == tunables.COOPERATIVE_SUBGROUP_SIZE and device.adapterInfo.subgroupMaxSize == tunables.COOPERATIVE_SUBGROUP_SIZE',cooperativeGeometryOk:"q4Present and fixedCooperativeSubgroup and cooperativeWorkgroupSize > 0 and cooperativeWorkgroupSize <= deviceWorkgroupCap and cooperativeNumSubgroups > 0 and workgroupSize % cooperativeNumSubgroups == 0 and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and cooperativeStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",scalesPk2:'q4Present and effectiveQuantBits == 4 and tensorDtypes.scalesT == "float16" and (args.hiddenSize / 32) % 2 == 0 and args.lmHeadOffset is defined and args.lmHeadOffset % 2 == 0',q4CooperativeContract:'q4Contract and effectiveQuantBits == 4 and effectiveTopK == 1 and not args.colMajor and cooperativeGeometryOk and args.hiddenSize >= tunables.COOPERATIVE_MIN_HIDDEN_SIZE and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and geometryOk',q4TiledContract:'q4Contract and (((effectiveQuantBits == 4 and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and effectiveTopK <= 8 and (effectiveTopK == 1 or args.vocabSize <= workgroupSize * effectiveNumBlocks)) or (effectiveQuantBits == 2 and tensorDtypes.scalesT == "uint32" and effectiveTopK > 1 and effectiveTopK <= 16 and args.vocabSize <= workgroupSize * effectiveNumBlocks and (args.hiddenSize / 32) % 2 == 0 and args.lmHeadOffset % 2 == 0 and not args.colMajor)) and geometryOk and (effectiveQuantBits != 4 or (not args.colMajor) or args.lmHeadOffset == 0))',headBlocksPerRow:"args.hiddenSize / 32",mRowsBytesPerBlock:"mRowsCount * (132 if effectiveQuantBits == 2 else 128)",mRowsLdsBudget:"device.limits.maxComputeWorkgroupStorageSize - workgroupSize * 8",mRowsChunkBlocks:"pick([[headBlocksPerRow % 64 == 0 and 64 * mRowsBytesPerBlock <= mRowsLdsBudget, 64], [headBlocksPerRow % 32 == 0 and 32 * mRowsBytesPerBlock <= mRowsLdsBudget, 32], [headBlocksPerRow % 16 == 0 and 16 * mRowsBytesPerBlock <= mRowsLdsBudget, 16], [headBlocksPerRow % 8 == 0 and 8 * mRowsBytesPerBlock <= mRowsLdsBudget, 8], [headBlocksPerRow % 4 == 0 and 4 * mRowsBytesPerBlock <= mRowsLdsBudget, 4], [headBlocksPerRow % 2 == 0 and 2 * mRowsBytesPerBlock <= mRowsLdsBudget, 2]], 0)",mRowsStorageBytes:"mRowsChunkBlocks * mRowsBytesPerBlock + workgroupSize * 8",mRowsWidthOk:'(effectiveQuantBits == 4 and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and effectiveTopK <= 8) or (effectiveQuantBits == 2 and tensorDtypes.scalesT == "uint32" and effectiveTopK > 1 and effectiveTopK <= 16 and args.lmHeadOffset is defined and args.lmHeadOffset % 2 == 0)',mRowsGeometryOk:"mRowsChunkBlocks > 0 and mRowsStorageBytes <= device.limits.maxComputeWorkgroupStorageSize and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension",q4MRowsContract:'q4Present and mRowsShapeOk and args.lmHeadOffset is defined and args.finalNormOffset is defined and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 64 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * headBlocksPerRow) * effectiveQuantBits and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * headBlocksPerRow and not args.colMajor and args.vocabSize <= workgroupSize * effectiveNumBlocks and mRowsWidthOk and mRowsGeometryOk',kSuperWorkgroupSize:'tunables.KSUPER_WORKGROUP_SIZE if tunables.KSUPER_WORKGROUP_SIZE != 0 else (min(workgroupSize, deviceWorkgroupCap, 2 * device.adapterInfo.subgroupMaxSize) if q6k and args.hiddenSize / 4 <= workgroupSize and device.features.has("subgroups") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMaxSize > 0 and device.adapterInfo.subgroupMaxSize <= 32 else workgroupSize)',kSuperStorageBytes:"args.hiddenSize * 4 + kSuperWorkgroupSize * 8 if kSuperPresent else 0",q8BlockStorageBytes:"args.hiddenSize * 4 + workgroupSize * 8 if q8BlockPresent else 0",kSuperGeometryOk:"kSuperPresent and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and kSuperWorkgroupSize > 0 and pow2ceil(kSuperWorkgroupSize) == kSuperWorkgroupSize and kSuperWorkgroupSize <= workgroupSize and kSuperWorkgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and kSuperStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",kSuperMRowsGeometryOk:"kSuperPresent and mRowsChunkBlocks > 0 and mRowsStorageBytes <= device.limits.maxComputeWorkgroupStorageSize and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension",kSuperMRowsContract:'kSuperPresent and mRowsShapeOk and effectiveTopK == 1 and not args.colMajor and args.finalNormOffset is defined and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == kSuperScalesDtype and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 256 == 0 and args.finalNormOffset % 4 == 0 and (args.lmHeadOffset if args.lmHeadOffset else 0) == 0 and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= kSuperBitsWords and dim(shapes.scalesT, 0) >= kSuperScaleWords and args.vocabSize <= workgroupSize * effectiveNumBlocks and kSuperMRowsGeometryOk',kSuperContract:'kSuperPresent and effectiveTopK == 1 and not args.colMajor and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == kSuperScalesDtype and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 256 == 0 and args.finalNormOffset % 4 == 0 and (args.lmHeadOffset if args.lmHeadOffset else 0) == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= kSuperBitsWords and dim(shapes.scalesT, 0) >= kSuperScaleWords and kSuperGeometryOk',q8BlockContract:'q8BlockPresent and effectiveTopK == 1 and not mRowsRequested and not args.colMajor and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 32 == 0 and args.finalNormOffset % 4 == 0 and (args.lmHeadOffset if args.lmHeadOffset else 0) == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 32) * 8 and dim(shapes.scalesT, 0) >= args.vocabSize * (args.hiddenSize / 32) and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and q8BlockStorageBytes <= device.limits.maxComputeWorkgroupStorageSize',denseCooperativeContract:'denseContract and device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id") and tunables.COOPERATIVE_SUBGROUP_SIZE > 0 and cooperativeWorkgroupSize % tunables.COOPERATIVE_SUBGROUP_SIZE == 0 and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == tunables.COOPERATIVE_SUBGROUP_SIZE and device.adapterInfo.subgroupMaxSize == tunables.COOPERATIVE_SUBGROUP_SIZE and cooperativeWorkgroupSize <= deviceWorkgroupCap and pow2ceil(cooperativeWorkgroupSize) == cooperativeWorkgroupSize and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and args.hiddenSize * 4 + cooperativeWorkgroupSize * 12 + 32 <= device.limits.maxComputeWorkgroupStorageSize and args.hiddenSize >= tunables.COOPERATIVE_MIN_HIDDEN_SIZE'},when:["baseContract"],bindings:{best_values_out:{arg:"bestValuesT",elementType:"f32"},best_ids_out:{arg:"bestIdsT",elementType:"u32"},hidden_q4:{arg:"hiddenT",name:"hidden",elementType:"f32"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},norm_weight:{arg:"weightsT",elementType:"$normWeightVec4"}},variants:[{id:"ksuper",priority:1,when:["kSuperPresent","kSuperContract"],derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",hiddenSizeBlocks:"args.hiddenSize / 32",lmHeadOffset:"args.lmHeadOffset if args.lmHeadOffset else 0",finalNormOffsetVec4:"args.finalNormOffset / 4",quantBits:"4",q4knCompact:"q4knc",workgroupSize:"kSuperWorkgroupSize",vocabTileSize:"tunables.WORKGROUP_SIZE"},passes:[{id:"main",name:"LlamaDecodeLmHeadArgmax",shader:"decode-lm-head-ksuper-argmax-tiled.wgsl.jinja",derive:{hiddenSizeVec4:"args.hiddenSize / 4",vocabSize:"args.vocabSize",numBlocks:"effectiveNumBlocks",f32Min:'"-3.4028234663852886e38"'},bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],dispatch:{x:"effectiveNumBlocks"}}]}]},assets:[["decode-lm-head-ksuper-argmax-tiled.wgsl.jinja",`{% include "ops/_shared/packed-4x8-compat.ac66f822e2.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const BPR:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD:u32={{ lmHeadOffset }}u;const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% if workgroupSize != vocabTileSize %}const VOCAB_TILE_SIZE:u32={{ vocabTileSize }}u;
{% endif %}const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ hiddenSizeVec4 }}>;
{% include "ops/_shared/q4-dequant-helpers.1f30cb9761.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.c040ca2b55.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;for(var i=lid;i<HIDDEN_SIZE_VEC4;i=i+WORKGROUP_SIZE){shared_acts[i]=hidden4(i)*vec4<f32>(norm_weight[W_NORM_VEC4+i]);}workgroupBarrier();var best_token=0u;var best_value=F32_MIN;
{% if workgroupSize == vocabTileSize %}for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){
{% else %}for(var tile=block*VOCAB_TILE_SIZE;tile<VOCAB_SIZE;tile=tile+VOCAB_TILE_SIZE*NUM_BLOCKS){for(var vocab=tile+lid;vocab<min(tile+VOCAB_TILE_SIZE,VOCAB_SIZE);vocab=vocab+WORKGROUP_SIZE){
{% endif %}var acc0=0.0;let base0=W_LM_HEAD+vocab*HIDDEN_SIZE_BLOCKS;for(var b=0u;b<HIDDEN_SIZE_BLOCKS;b=b+1u){let act=b*8u;
{% for i in range(8) %}let x{{ i }}=shared_acts[act{{ " + " ~ i ~ "u" if i else "" }}];
{% endfor %}{% set regRows = 1 %}{% include "ops/_shared/q4-decode-row-block-accumulate.6b9bc7b528.wgsl.jinja" %}    }
let acc=acc0;if(acc>best_value){best_value=acc;best_token=vocab;} }
{% if workgroupSize != vocabTileSize %}}
{% endif %}{% include "ops/_shared/argmax-wg-tree.13d09b6502.wgsl.jinja" %}}`]],assumes:[["decode-lm-head-ksuper-argmax-tiled.wgsl.jinja",{q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q6k:1,q5k:0,scalesPk2:!1,q4knCompact:0,vocabTileSize:256}]]}],["com.xenova.LlamaDecodeQkvProj",{manifest:{domain:"com.xenova",name:"LlamaDecodeQkvProj",sinceVersion:1,inputs:{normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},hiddenT:{onnx:"Hidden",dtype:"H",rank:1,optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},normWeightsT:{onnx:"NormWeights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},kBitsT:{onnx:"KBits",dtype:"uint32",rank:1,optional:!0},kScalesT:{onnx:"KScales",dtype:"KS",rank:1,optional:!0},vBitsT:{onnx:"VBits",dtype:"uint32",rank:1,optional:!0},vScalesT:{onnx:"VScales",dtype:"VS",rank:1,optional:!0},qT:{onnx:"Q",dtype:"float32",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0},cosT:{onnx:"RopeCos",dtype:"float32",rank:1,optional:!0},sinT:{onnx:"RopeSin",dtype:"float32",rank:1,optional:!0}},outputs:{qT:{onnx:"Q",dtype:"float32",rank:1,optional:!0},qGateT:{onnx:"QGate",dtype:"float32",rank:1,optional:!0},headGateT:{onnx:"HeadGate",dtype:"float32",rank:1,optional:!0},kT:{onnx:"K",dtype:"float32",rank:1,optional:!0},vT:{onnx:"V",dtype:"float32",rank:1,optional:!0},qkvT:{onnx:"QKV",dtype:"O",rank:1,optional:!0},kCacheT:{onnx:"KCache",dtype:"float32",rank:1,optional:!0},vCacheT:{onnx:"VCache",dtype:"float32",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0}},args:{layer:{kind:"u32",required:!1},cacheLen:{kind:"u32",onnx:"cache_len",required:!1},pastLen:{kind:"u32",onnx:"past_len",required:!1},hiddenSize:{kind:"u32",onnx:"hidden_size",required:!1},tokens:{kind:"u32",required:!1},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},numHeads:{kind:"u32",onnx:"num_heads",required:!1},headDim:{kind:"u32",onnx:"head_dim"},rotaryDim:{kind:"u32",onnx:"rotary_dim",required:!1},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},normOffset:{kind:"u32",onnx:"input_norm_offset",required:!1},preNormed:{kind:"u32",onnx:"pre_normed_input",required:!1},qOffset:{kind:"u32",onnx:"q_proj_offset",required:!1},qSize:{kind:"u32",onnx:"q_out_size",required:!1},gateOffset:{kind:"u32",onnx:"gate_offset",required:!1},kOffset:{kind:"u32",onnx:"k_proj_offset",required:!1},vOffset:{kind:"u32",onnx:"v_proj_offset",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q6_k","q5_k","q3_k","q2_k","q1_0"]},fuseKv:{kind:"u32",onnx:"fuse_kv_append",required:!1},kvDstStart:{kind:"u32",onnx:"kv_cache_dst_start",required:!1},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},iqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},gridOffset:{kind:"u32",onnx:"offset",required:!1},kGridOffset:{kind:"u32",onnx:"offset",required:!1},vGridOffset:{kind:"u32",onnx:"offset",required:!1},kFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q8_0","iq","lut32","lut16","q6_k","q5_k","lut32_compact","lut16_compact","lut2_32","lut2_16"]},kLut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},kIqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},vFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q8_0","iq","lut32","lut16","q6_k","q5_k","lut32_compact","lut16_compact","lut2_32","lut2_16","q4_k_native_compact","q3_k","q2_k"]},vLut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},vIqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]}},typeConstraints:{N:["float32","float16"],H:["float32","float16"],S:["float32","float16","uint32"],KS:["float32","float16","uint32"],VS:["float32","float16","uint32"],W:["float32","float16","uint32"],O:["float32","float16"],C:["float32","float16"]},tunables:{GATED_DENSE_WORKGROUP_SIZE:{default:64},GATED_Q4_WORKGROUP_SIZE:{default:32},GATED_Q1_WORKGROUP_SIZE:{default:64},GATED_Q4_TILE_ROWS:{default:2}},derive:{qRows:"args.qSize if args.qSize else args.hiddenSize",weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',tokenRows:"args.tokens if args.tokens is defined else 1",deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',q4knc:'1 if fmt == "q4_k_native_compact" else 0',iqRawSel:"args.iqRaw if args.iqRaw else 0",kFmt:"args.kFormat if args.kFormat else fmt",kQuantBits:'8 if (kFmt == "q8_0" or kFmt == "iq") else 4',kHasMin:'1 if kFmt == "q4_k" else 0',kIq:'1 if kFmt == "iq" else 0',kLut16:'1 if kFmt == "lut16" else 0',kLut2:'1 if (kFmt == "lut2_32" or kFmt == "lut2_16") else 0',kQ6k:'1 if kFmt == "q6_k" else 0',kQ5k:'1 if kFmt == "q5_k" else 0',kScalesPerBlock:"2 if (kHasMin or kIq or kLut16) else 1",kLutc:'1 if (kFmt == "lut32_compact" or kFmt == "lut16_compact" or kLut2 == 1) else 0',kLutc16:'1 if (kFmt == "lut16_compact" or kFmt == "lut2_16") else 0',kLutcWords:"1 if (kLut2 == 1 and args.kLut == 9) else (5 if kLutc16 == 1 else 3)",kIqRawSel:"args.kIqRaw if args.kIqRaw else 0",vFmt:"args.vFormat if args.vFormat else fmt",vQuantBits:'8 if (vFmt == "q8_0" or vFmt == "iq") else 4',vHasMin:'1 if vFmt == "q4_k" else 0',vIq:'1 if vFmt == "iq" else 0',vLut16:'1 if vFmt == "lut16" else 0',vLut2:'1 if (vFmt == "lut2_32" or vFmt == "lut2_16") else 0',vQ6k:'1 if vFmt == "q6_k" else 0',vQ5k:'1 if vFmt == "q5_k" else 0',vQ3k:'1 if vFmt == "q3_k" else 0',vQ2k:'1 if vFmt == "q2_k" else 0',vQ4knc:'1 if vFmt == "q4_k_native_compact" else 0',vQ5:'1 if vFmt == "q5_0" else 0',vScalesPerBlock:"2 if (vHasMin or vIq or vLut16) else 1",vLutc:'1 if (vFmt == "lut32_compact" or vFmt == "lut16_compact" or vLut2 == 1) else 0',vLutc16:'1 if (vFmt == "lut16_compact" or vFmt == "lut2_16") else 0',vLutcWords:"1 if (vLut2 == 1 and args.vLut == 9) else (5 if vLutc16 == 1 else 3)",vIqRawSel:"args.vIqRaw if args.vIqRaw else 0",qIq1Family:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",kIq1Family:"kIqRawSel == 1 or kIqRawSel == 7 or kIqRawSel == 8",vIq1Family:"vIqRawSel == 1 or vIqRawSel == 7 or vIqRawSel == 8",qGridInBitsOk:"(args.gridOffset is defined and args.gridOffset % 2 == 0 and dim(shapes.bitsT, 0) >= args.gridOffset + 4096) if qIq1Family else true",kGridInBitsOk:"(args.kGridOffset is defined and args.kGridOffset % 2 == 0 and dim(shapes.kBitsT, 0) >= args.kGridOffset + 4096) if kIq1Family else true",vGridInBitsOk:"(args.vGridOffset is defined and args.vGridOffset % 2 == 0 and dim(shapes.vBitsT, 0) >= args.vGridOffset + 4096) if vIq1Family else true",splitPacksPresent:"present.kBitsT and present.kScalesT and present.vBitsT and present.vScalesT",projectionArgsPresent:"(args.hiddenSize is defined and args.qOffset is defined and args.kOffset is defined and args.vOffset is defined)",sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',portableMr4Preferred:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32',gateMaxBlocks:"(args.gateOffset + args.numHeads * (args.hiddenSize / 32)) if (projectionArgsPresent and present.headGateT and args.gateOffset is defined and args.numHeads is defined) else 0",qkvMaxBlocks:"(max(args.qOffset + (args.qSize if args.qSize else args.hiddenSize) * (args.hiddenSize / 32), max(args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 32), max(0 if present.vBitsT else args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 32), gateMaxBlocks)))) if projectionArgsPresent else 0",qkvSuperblocks:"ceilDiv(qkvMaxBlocks, 8)",vSplitMaxBlocks:"(args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 32)) if projectionArgsPresent else 0",vSplitSuperblocks:"ceilDiv(vSplitMaxBlocks, 8)",qkvPackContract:'projectionArgsPresent and lut2 == 0 and iqRawSel == 0 and not present.qGateT and not present.kBitsT and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.qT == 1 and ranks.kT == 1 and ranks.vT == 1 and fmt != "q1_0" and tensorDtypes.bitsT == "uint32" and tensorDtypes.qT == "float32" and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and args.hiddenSize > 0 and args.hiddenSize % 32 == 0 and args.numKvHeads > 0 and args.headDim > 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or (args.qOffset % 8 == 0 and args.kOffset % 8 == 0 and (present.vBitsT or args.vOffset % 8 == 0))) and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.qT, 0) == (args.qSize if args.qSize else args.hiddenSize) and dim(shapes.kT, 0) == args.numKvHeads * args.headDim and dim(shapes.vT, 0) == args.numKvHeads * args.headDim and dim(shapes.bitsT, 0) % 4 == 0 and dim(shapes.bitsT, 0) >= (qkvMaxBlocks * 4 if (q4knc or lutc) else qkvSuperblocks * 52 if q6k else qkvSuperblocks * 44 if q5k else qkvSuperblocks * 28 if q3k else qkvSuperblocks * 20 if q2k else qkvMaxBlocks * quantBits) and dim(shapes.scalesT, 0) >= (qkvSuperblocks * 5 if q4knc else qkvSuperblocks * lutcWords if lutc else qkvSuperblocks * 2 if (q2k or q5k) else qkvSuperblocks if (q6k or q3k) else qkvMaxBlocks * (2 if (hasMin or iq or lut16) else 1))',headGateContract:'present.headGateT and args.gateOffset is defined and args.numHeads > 0 and ranks.headGateT == 1 and tensorDtypes.headGateT == "float32" and dim(shapes.headGateT, 0) == args.numHeads and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or args.gateOffset % 8 == 0)',vPackContract:'present.vScalesT and args.vFormat is defined and vLut2 == 0 and vIqRawSel == 0 and vFmt != "q1_0" and ranks.vBitsT == 1 and ranks.vScalesT == 1 and tensorDtypes.vBitsT == "uint32" and (vQuantBits == 4 or vQuantBits == 8) and (not (vQ6k or vQ5k or vQ3k or vQ2k or vQ4knc or vLutc) or args.vOffset % 8 == 0) and (tensorDtypes.vScalesT == "uint32" if (vQ4knc or vLutc) else (tensorDtypes.vScalesT == "float32" or tensorDtypes.vScalesT == "float16")) and dim(shapes.vBitsT, 0) % 4 == 0 and dim(shapes.vBitsT, 0) >= (vSplitMaxBlocks * 4 if (vQ4knc or vLutc) else vSplitSuperblocks * 52 if vQ6k else vSplitSuperblocks * 44 if vQ5k else vSplitSuperblocks * 28 if vQ3k else vSplitSuperblocks * 20 if vQ2k else vSplitMaxBlocks * vQuantBits) and dim(shapes.vScalesT, 0) >= (vSplitSuperblocks * 5 if vQ4knc else vSplitSuperblocks * vLutcWords if vLutc else vSplitSuperblocks * 2 if (vQ2k or vQ5k) else vSplitSuperblocks if (vQ6k or vQ3k) else vSplitMaxBlocks * vScalesPerBlock)',baseContract:"not present.vBitsT and qkvPackContract",headGatedContract:"baseContract and headGateContract",vSplitContract:"present.vBitsT and qkvPackContract and headGateContract and vPackContract",gatedDenseWorkgroupLimit:"min(tunables.GATED_DENSE_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedDenseWorkgroupSize:"max(1, pow2ceil(gatedDenseWorkgroupLimit + 1) / 2)",gatedQ4WorkgroupLimit:"min(tunables.GATED_Q4_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedQ4WorkgroupSize:"max(1, pow2ceil(gatedQ4WorkgroupLimit + 1) / 2)",gatedQ1WorkgroupLimit:"min(tunables.GATED_Q1_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedQ1WorkgroupSize:"max(1, pow2ceil(gatedQ1WorkgroupLimit + 1) / 2)",gatedQSize:"args.qSize if args.qSize else 0",gatedBlocksPerRow:"(args.hiddenSize / 32) if projectionArgsPresent else 0",gatedKvSize:"(args.numKvHeads * args.headDim) if projectionArgsPresent else 0",gatedQMaxBlocks:"(args.qOffset + 2 * gatedQSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedKMaxBlocks:"(args.kOffset + gatedKvSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedVMaxBlocks:"(args.vOffset + gatedKvSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedMaxBlocks:"max(gatedQMaxBlocks, max(gatedKMaxBlocks, gatedVMaxBlocks))",gatedQSupers:"ceilDiv(gatedQMaxBlocks, 8)",gatedKSupers:"ceilDiv(gatedKMaxBlocks, 8)",gatedVSupers:"ceilDiv(gatedVMaxBlocks, 8)",gatedSupers:"ceilDiv(gatedMaxBlocks, 8)",gatedBitsNeeded:"ceilDiv(gatedMaxBlocks, 8) * 12 if iqRawSel == 7 else ceilDiv(gatedMaxBlocks, 8) * 14 if iqRawSel == 8 else ceilDiv(gatedMaxBlocks, 8) * 27 if iqRawSel == 4 else gatedMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else gatedSupers * 52 if q6k else gatedSupers * 44 if q5k else gatedMaxBlocks * (2 if lut2 else quantBits)",gatedScalesNeeded:"ceilDiv(gatedMaxBlocks, 8) if (iqRawSel == 4 or iqRawSel == 2 or iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8) else gatedSupers * lutcWords if lutc else gatedSupers if q6k else gatedSupers * 2 if q5k else gatedMaxBlocks * (2 if (hasMin or iq or lut16) else 1)",gatedQBitsNeeded:"ceilDiv(gatedQMaxBlocks, 8) * 12 if iqRawSel == 7 else ceilDiv(gatedQMaxBlocks, 8) * 14 if iqRawSel == 8 else ceilDiv(gatedQMaxBlocks, 8) * 27 if iqRawSel == 4 else gatedQMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else gatedQSupers * 52 if q6k else gatedQSupers * 44 if q5k else gatedQMaxBlocks * (2 if lut2 else quantBits)",gatedQScalesNeeded:"ceilDiv(gatedQMaxBlocks, 8) if (iqRawSel == 4 or iqRawSel == 2 or iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8) else gatedQSupers * lutcWords if lutc else gatedQSupers if q6k else gatedQSupers * 2 if q5k else gatedQMaxBlocks * (2 if (hasMin or iq or lut16) else 1)",gatedKBitsNeeded:"ceilDiv(gatedKMaxBlocks, 8) * 12 if kIqRawSel == 7 else ceilDiv(gatedKMaxBlocks, 8) * 14 if kIqRawSel == 8 else ceilDiv(gatedKMaxBlocks, 8) * 27 if kIqRawSel == 4 else gatedKMaxBlocks * 2 if (kIqRawSel == 1 or kIqRawSel == 2) else gatedKSupers * 52 if kQ6k else gatedKSupers * 44 if kQ5k else gatedKMaxBlocks * (2 if kLut2 else kQuantBits)",gatedKScalesNeeded:"ceilDiv(gatedKMaxBlocks, 8) if (kIqRawSel == 4 or kIqRawSel == 2 or kIqRawSel == 1 or kIqRawSel == 7 or kIqRawSel == 8) else (gatedKSupers * kLutcWords if kLutc else (gatedKSupers if kQ6k else (gatedKSupers * 2 if kQ5k else gatedKMaxBlocks * kScalesPerBlock)))",gatedVBitsNeeded:"ceilDiv(gatedVMaxBlocks, 8) * 12 if vIqRawSel == 7 else ceilDiv(gatedVMaxBlocks, 8) * 14 if vIqRawSel == 8 else ceilDiv(gatedVMaxBlocks, 8) * 27 if vIqRawSel == 4 else gatedVMaxBlocks * 2 if (vIqRawSel == 1 or vIqRawSel == 2) else gatedVSupers * 52 if vQ6k else gatedVSupers * 44 if vQ5k else gatedVMaxBlocks * (2 if vLut2 else vQuantBits)",gatedVScalesNeeded:"ceilDiv(gatedVMaxBlocks, 8) if (vIqRawSel == 4 or vIqRawSel == 2 or vIqRawSel == 1 or vIqRawSel == 7 or vIqRawSel == 8) else (gatedVSupers * vLutcWords if vLutc else (gatedVSupers if vQ6k else (gatedVSupers * 2 if vQ5k else gatedVMaxBlocks * vScalesPerBlock)))",gatedKSuperGeometry:"(args.hiddenSize % 256 == 0) if projectionArgsPresent else false",gatedTileFits:"projectionArgsPresent and args.qSize is defined and tunables.GATED_Q4_TILE_ROWS >= 1 and args.headDim % tunables.GATED_Q4_TILE_ROWS == 0 and args.qSize % tunables.GATED_Q4_TILE_ROWS == 0 and (args.numKvHeads * args.headDim) % tunables.GATED_Q4_TILE_ROWS == 0",gatedTileRows:"tunables.GATED_Q4_TILE_ROWS if gatedTileFits else 1",gatedDenseResourcesFit:"(gatedDenseWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and 2 * gatedQSize + 2 * args.numKvHeads * args.headDim <= device.limits.maxComputeWorkgroupsPerDimension)",gatedQ4ResourcesFit:"(gatedQ4WorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and 2 * gatedQSize + 2 * args.numKvHeads * args.headDim <= device.limits.maxComputeWorkgroupsPerDimension)",gatedQ1ResourcesFit:"(gatedQ1WorkgroupSize * 16 <= device.limits.maxComputeWorkgroupStorageSize and ceilDiv(2 * gatedQSize + 2 * args.numKvHeads * args.headDim, 4) <= device.limits.maxComputeWorkgroupsPerDimension)",gatedBaseContract:'projectionArgsPresent and (ranks.normedT == 1 and ranks.qT == 1 and ranks.qGateT == 1 and ranks.kT == 1 and ranks.vT == 1 and tensorDtypes.normedT == "float32" and tensorDtypes.qT == "float32" and tensorDtypes.qGateT == "float32" and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and args.qSize > 0 and args.numKvHeads > 0 and args.headDim > 0 and args.qSize % args.headDim == 0 and args.hiddenSize > 0 and dim(shapes.normedT, 0) == tokenRows * args.hiddenSize and dim(shapes.qT, 0) == tokenRows * args.qSize and dim(shapes.qGateT, 0) == tokenRows * args.qSize and dim(shapes.kT, 0) == tokenRows * (args.numKvHeads * args.headDim) and dim(shapes.vT, 0) == tokenRows * (args.numKvHeads * args.headDim) and tokenRows >= 1 and tokenRows <= 4)'},bindings:{q:{arg:"qT",elementType:"f32"},k:{arg:"kT",elementType:"f32"},v:{arg:"vT",elementType:"f32"},hidden:{arg:"hiddenT",elementType:"$hiddenVec4"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},head_gate:{arg:"headGateT",elementType:"f32"}},variants:[{cases:[{rid:"",priority:0,demote:null,multiRow:"0",regRows:"1",guard:[],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (0) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim + args.numHeads"},{rid:"_mr2",priority:3,demote:["portableMr4Preferred"],multiRow:"1",regRows:"2",guard:["(args.qSize if args.qSize else args.hiddenSize) % 2 == 0","(args.numKvHeads * args.headDim) % 2 == 0","args.numHeads % 2 == 0","((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim + args.numHeads) % 8 == 0"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"ceil(((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim + args.numHeads) / 8)"},{rid:"_mr4",priority:2,demote:null,multiRow:"1",regRows:"4",guard:["(args.qSize if args.qSize else args.hiddenSize) % 4 == 0","(args.numKvHeads * args.headDim) % 4 == 0","args.numHeads % 4 == 0","((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim + args.numHeads) % 16 == 0"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"ceil(((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim + args.numHeads) / 16)"}],variants:[{id:"q4_prenormed_head_gated{rid}",priority:"{priority}",when:["args.fuseKv != 1","args.preNormed == 1","headGatedContract","{guard}","f16Ok(dtypes.H) and f16Ok(dtypes.S)","tokenRows == 1","{sgFoldStorageGate0}"],demoteWhen:"{demote}",derive:{scaleScalar:"dtypes.S",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",blocksPerRow:"args.hiddenSize / 32",kvSize:"args.numKvHeads * args.headDim",headGateRows:"args.numHeads",wG:"args.gateOffset",preNormed:"1",multiRow:"{multiRow}",regRows:"{regRows}",fuseKv:"0",lutId:"args.lut if args.lut else 0",workgroupSize:"subgroupCoveringWorkgroup",useSubgroups:"wave32SubgroupsUsable",qSize:"args.qSize if args.qSize else args.hiddenSize",totalOutputs:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim",wQ:"args.qOffset",wK:"args.kOffset",wV:"args.vOffset",q4knCompact:"q4knc"},passes:[{id:"main",name:"LlamaDecodeQkvNormQ4",shader:"decode-qkv-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","q","k","v","head_gate"],dispatch:{x:"{x1}"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{rid:"",priority:0,demote:null,multiRow:"0",regRows:"1",guard:[],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (0) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"(args.qSize if args.qSize else args.hiddenSize) + args.numKvHeads * args.headDim + args.numHeads",x2:"args.numKvHeads * args.headDim"},{rid:"_mr2",priority:3,demote:["portableMr4Preferred"],multiRow:"1",regRows:"2",guard:["(args.qSize if args.qSize else args.hiddenSize) % 2 == 0","(args.numKvHeads * args.headDim) % 2 == 0","args.numHeads % 2 == 0","((args.qSize if args.qSize else args.hiddenSize) + args.numKvHeads * args.headDim + args.numHeads) % 8 == 0","(args.numKvHeads * args.headDim) % 8 == 0"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"ceil(((args.qSize if args.qSize else args.hiddenSize) + args.numKvHeads * args.headDim + args.numHeads) / 8)",x2:"ceil((args.numKvHeads * args.headDim) / 8)"},{rid:"_mr4",priority:2,demote:null,multiRow:"1",regRows:"4",guard:["(args.qSize if args.qSize else args.hiddenSize) % 4 == 0","(args.numKvHeads * args.headDim) % 4 == 0","args.numHeads % 4 == 0","((args.qSize if args.qSize else args.hiddenSize) + args.numKvHeads * args.headDim + args.numHeads) % 16 == 0","(args.numKvHeads * args.headDim) % 16 == 0"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize",x1:"ceil(((args.qSize if args.qSize else args.hiddenSize) + args.numKvHeads * args.headDim + args.numHeads) / 16)",x2:"ceil((args.numKvHeads * args.headDim) / 16)"}],variants:[{id:"q4_prenormed_head_gated_v_split{rid}",priority:"{priority}",when:["args.fuseKv != 1","args.preNormed == 1","vSplitContract","{guard}","f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.VS)","tokenRows == 1","{sgFoldStorageGate0}"],demoteWhen:"{demote}",derive:{scaleScalar:"dtypes.S",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",blocksPerRow:"args.hiddenSize / 32",kvSize:"args.numKvHeads * args.headDim",headGateRows:"args.numHeads",wG:"args.gateOffset",preNormed:"1",multiRow:"{multiRow}",regRows:"{regRows}",fuseKv:"0",lutId:"args.lut if args.lut else 0",workgroupSize:"subgroupCoveringWorkgroup",useSubgroups:"wave32SubgroupsUsable",qSize:"args.qSize if args.qSize else args.hiddenSize",totalOutputs:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim",wQ:"args.qOffset",wK:"args.kOffset",wV:"args.vOffset",q4knCompact:"q4knc"},passes:[{id:"qkg",name:"LlamaDecodeQkvNormQ4SplitQkg",shader:"decode-qkv-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","q","k","head_gate"],dispatch:{x:"{x1}"},subgroupCollectivesWidth:"portable",derive:{vSplit:"1"}},{id:"v",name:"LlamaDecodeQkvNormQ4SplitV",shader:"decode-qkv-norm-q4.wgsl.jinja",bindings:["hidden",{arg:"vBitsT",name:"q4_bits",elementType:"vec4<u32>"},{arg:"vScalesT",name:"q4_scales",elementType:"$scaleScalar"},"v"],dispatch:{x:"{x2}"},subgroupCollectivesWidth:"portable",derive:{vSplit:"2",scaleScalar:"dtypes.VS",quantBits:"vQuantBits",hasMin:"vHasMin",iq:"vIq",lut16:"vLut16",lutc:"vLutc",lutc16:"vLutc16",q6k:"vQ6k",q5k:"vQ5k",q3k:"vQ3k",q2k:"vQ2k",q5:"vQ5",q4knCompact:"vQ4knc",lutId:"args.vLut if args.vLut else 0"}}]}]}]},assets:[["decode-qkv-norm-q4.wgsl.jinja",`{% set emitQuantDotHelpers = true %}{% set nibbleHelpers = not (q2k or q3k or q5k or q6k) %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not lutId and (((quantBits != 8) and nibbleHelpers) or q2k or q3k or q5k or q6k) %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const BPR:u32={{ blocksPerRow }}u;
{% set headGate = (headGateRows | default(0)) > 0 %}{% set vSplit = vSplit | default(0) %}{% if vSplit != 2 %}const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;
{% endif %}{% if not multiRow %}const TOTAL_OUTPUTS:u32={{ kvSize if vSplit == 2 else totalOutputs + (headGateRows | default(0)) - (kvSize if vSplit == 1 else 0) }}u;
{% endif %}{% if vSplit != 2 %}const W_Q_Q4:u32={{ wQ }}u;const W_K_Q4:u32={{ wK }}u;
{% endif %}{% if vSplit != 1 %}const W_V_Q4:u32={{ wV }}u;
{% endif %}{% if headGate and vSplit != 2 %}const W_G_Q4:u32={{ wG }}u;const GATE_START:u32=Q_SIZE+{{ "" if vSplit == 1 else "2u * " }}KV_SIZE;
{% endif %}{% if not multiRow %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% include "ops/_shared/subgroup-runtime-fold.f33ebc1f56.wgsl.jinja" %}{% if useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 128 if multiRow else workgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}{% if multiRow %}{% set partialsRowVector = true %}var<workgroup>partials4:array<vec{{ regRows }}<f32>,128>;
{% else %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% include "ops/_shared/q4-dequant-helpers.33f455d750.wgsl.jinja" %}{% if multiRow %}{% if headGate %}{% endif %}@compute@workgroup_size(128){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;let dim0=(wid.x*4u+cluster)*{{ regRows }}u;
{% if vSplit == 2 %}let seg_base=W_V_Q4;let seg_out0=dim0;
{% else %}var seg_base=W_Q_Q4;var seg_out0=dim0;if(dim0>=Q_SIZE&&dim0<Q_SIZE+KV_SIZE){seg_base=W_K_Q4;seg_out0=dim0-Q_SIZE;
{% if vSplit != 1 %}}else if(dim0>=Q_SIZE+KV_SIZE{{ " && dim0 < GATE_START" if headGate else "" }}){seg_base=W_V_Q4;seg_out0=dim0-Q_SIZE-KV_SIZE;
{% endif %}{% if headGate %}}else if(dim0>=GATE_START){seg_base=W_G_Q4;seg_out0=dim0-GATE_START;
{% endif %}}
{% endif %}let base0=seg_base+seg_out0*BPR;
{% include "ops/_shared/q4-normed-projection-multirow.ed1980421f.wgsl.jinja" %}{% include "ops/_shared/projection-multirow-reduce.367c218e5c.wgsl.jinja" %}  if (lane == 0u) {
{% if vSplit == 2 %}{% for r in range(regRows) %}v[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}{% else %}if(dim0<Q_SIZE){
{% for r in range(regRows) %}q[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}}else if(dim0<Q_SIZE+KV_SIZE){
{% for r in range(regRows) %}k[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}{% if headGate and vSplit != 1 %}}else if(dim0<GATE_START){
{% for r in range(regRows) %}v[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}{% endif %}}else{
{% for r in range(regRows) %}{{ "head_gate" if headGate else "v" }}[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}}
{% endif %}} }
{% else %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let index=wid.x;let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(index>=TOTAL_OUTPUTS){return;}
{% if vSplit == 2 %}let output_index=index;let row_block_base=W_V_Q4+index*BPR;
{% else %}var output_index=index;var row_block_base=W_Q_Q4+index*BPR;if(index>=Q_SIZE&&index<Q_SIZE+KV_SIZE){output_index=index-Q_SIZE;row_block_base=W_K_Q4+output_index*BPR;
{% if vSplit != 1 %}}else if(index>=Q_SIZE+KV_SIZE{{ " && index < GATE_START" if headGate else "" }}){output_index=index-Q_SIZE-KV_SIZE;row_block_base=W_V_Q4+output_index*BPR;
{% endif %}{% if headGate %}}else if(index>=GATE_START){output_index=index-GATE_START;row_block_base=W_G_Q4+output_index*BPR;
{% endif %}}
{% endif %}{% include "ops/_shared/q4-normed-projection-single-row.7d02c37204.wgsl.jinja" %}  if (lid == 0u) {
let out=acc_total;
{% if vSplit == 2 %}v[output_index]=out;
{% else %}if(index<Q_SIZE){q[output_index]=out;}else if(index<Q_SIZE+KV_SIZE){k[output_index]=out;
{% if headGate and vSplit != 1 %}}else if(index<GATE_START){v[output_index]=out;
{% endif %}}else{ {{ "head_gate" if headGate else "v" }}[output_index]=out;}
{% endif %}} }
{% endif %}`]],assumes:[["decode-qkv-norm-q4.wgsl.jinja",{quantBits:4,q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q5k:0,lutId:0}]]}],["com.xenova.LlamaDecodeResidualProjection",{manifest:{domain:"com.xenova",name:"LlamaDecodeResidualProjection",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:1},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},attnT:{onnx:"Attn",dtype:"A",rank:1,optional:!0},intermediateT:{onnx:"Intermediate",dtype:"I",rank:1,optional:!0},upT:{onnx:"Up",dtype:"float32",rank:1,optional:!0},colBitsT:{onnx:"ColBits",dtype:"uint32",rank:1,optional:!0},colScalesT:{onnx:"ColScales",dtype:"S",rank:1,optional:!0},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0}},outputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:"ranks.hiddenT",shape:"shapes.hiddenT"}},when:["args.accumulate is undefined or args.accumulate == 1 or (args.accumulate == 0 and not present.colBitsT)"],args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},inFeatures:{kind:"u32",onnx:"in_features",required:!1},oOffset:{kind:"u32",onnx:"o_proj_offset",required:!1},intermediateSize:{kind:"u32",onnx:"intermediate_size",required:!1},downOffset:{kind:"u32",onnx:"down_proj_offset",required:!1},colOffset:{kind:"u32",onnx:"column_pack_word_offset",required:!1},scaleOffset:{kind:"u32",onnx:"transposed_scale_offset",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k","q1_0"]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},accumulate:{kind:"u32",onnx:"accumulate",required:!1},actRelu2:{kind:"u32",onnx:"activation_relu_squared",required:!1},hasMin:{kind:"u32",onnx:"kquant_has_min",required:!1},colElem:{kind:"u32",onnx:"column_pack_element_major",required:!1,oneOf:[0,1]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},tokens:{kind:"u32",required:!1}},typeConstraints:{W:["float32","float16","uint32"],S:["float32","float16","uint32"],A:["float32","float16"],I:["float32","float16"]},tunables:{OUTPUTS_PER_WG:{default:2},DENSE_WORKGROUP_SIZE:{default:64},Q1_WORKGROUP_SIZE:{default:64},OPROJ_Q1_WORKGROUP_SIZE:{default:32},Q1_GENERAL_WORKGROUP_SIZE:{default:64}},derive:{weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',tokenRows:"args.tokens if args.tokens is defined else 1",deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',q4knc:'1 if fmt == "q4_k_native_compact" else 0',iqRawSel:"args.iqRaw if args.iqRaw else 0",oprojMode:"present.attnT and not present.intermediateT and not present.upT and not present.colBitsT and not present.colScalesT",legacyMode:"args.inFeatures is undefined",generalMode:"args.inFeatures is defined",inputSize:"args.inFeatures if args.inFeatures is defined else args.hiddenSize",denseRoute:"present.weightsT and not present.bitsT and not present.scalesT",q1Route:'not present.weightsT and present.bitsT and present.scalesT and fmt == "q1_0"',q4Route:'not present.weightsT and present.bitsT and present.scalesT and fmt != "q1_0"',oprojFormatOk:'args.format is undefined or args.format == "q4_0" or args.format == "q4_k" or args.format == "q8_0" or args.format == "iq" or args.format == "lut32" or args.format == "lut16" or args.format == "q6_k" or args.format == "q5_k" or args.format == "q3_k" or args.format == "q2_k" or args.format == "q4_k_native_compact" or args.format == "lut32_compact" or args.format == "lut16_compact" or args.format == "lut2_32" or args.format == "lut2_16"',sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',portableMr4Preferred:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32',denseWorkgroupLimit:"min(tunables.DENSE_WORKGROUP_SIZE, deviceWorkgroupCap)",denseWorkgroupSize:"max(1, pow2ceil(denseWorkgroupLimit + 1) / 2)",denseWorkgroupBytes:"denseWorkgroupSize * tunables.OUTPUTS_PER_WG * 4",denseDeviceOk:"tunables.OUTPUTS_PER_WG >= 1 and tunables.OUTPUTS_PER_WG <= denseWorkgroupSize and denseWorkgroupBytes <= device.limits.maxComputeWorkgroupStorageSize",denseGeneralResourcesFit:"denseWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and args.hiddenSize <= device.limits.maxComputeWorkgroupsPerDimension",q1WorkgroupLimit:"min((tunables.OPROJ_Q1_WORKGROUP_SIZE if oprojMode else tunables.Q1_WORKGROUP_SIZE), deviceWorkgroupCap)",q1WorkgroupSize:"max(1, pow2ceil(q1WorkgroupLimit + 1) / 2)",q1WorkgroupBytes:"q1WorkgroupSize * 4 * 4",q1DeviceOk:"q1WorkgroupBytes <= device.limits.maxComputeWorkgroupStorageSize",q1GeneralWorkgroupLimit:"min(tunables.Q1_GENERAL_WORKGROUP_SIZE, deviceWorkgroupCap)",q1GeneralWorkgroupSize:"max(1, pow2ceil(q1GeneralWorkgroupLimit + 1) / 2)",q1GeneralResourcesFit:"q1GeneralWorkgroupSize * 16 <= device.limits.maxComputeWorkgroupStorageSize and ceilDiv(args.hiddenSize, 4) <= device.limits.maxComputeWorkgroupsPerDimension",q4WorkgroupSize:"32",q4MultiRowWorkgroupSize:"q4WorkgroupSize * 4",q4WorkgroupFits:"q4WorkgroupSize == 32 and q4WorkgroupSize <= deviceWorkgroupCap and q4WorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize",q4ResourcesFit:"q4WorkgroupFits and args.hiddenSize <= device.limits.maxComputeWorkgroupsPerDimension",q4MultiRowResourcesFit:"q4WorkgroupFits and q4MultiRowWorkgroupSize <= deviceWorkgroupCap and q4MultiRowWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize",stageActivationsMeasured:'wave32SubgroupsUsable and has(device.adapterInfo, "architecture") and (device.adapterInfo.architecture == "pascal" or device.adapterInfo.architecture == "maxwell") and (fmt == "q4_0" or fmt == "q8_0" or fmt == "q3_k" or fmt == "q4_k" or fmt == "q5_k" or fmt == "q6_k" or fmt == "q4_k_native_compact" or fmt == "lut2_32") and 8192 + 544 <= device.limits.maxComputeWorkgroupStorageSize',oprojMaxBlocks:"(args.oOffset + args.hiddenSize * (inputSize / 32)) if (oprojMode and args.oOffset is defined and args.hiddenSize is defined) else 0",oprojSuperblocks:"ceilDiv(oprojMaxBlocks, 8)",downMaxBlocks:"(args.downOffset + args.hiddenSize * (args.intermediateSize / 32)) if (args.downOffset is defined and args.hiddenSize is defined and args.intermediateSize is defined) else 0",downSuperblocks:"ceilDiv(downMaxBlocks, 8)",oprojBaseContract:'oprojMode and args.oOffset is defined and ranks.hiddenT == 1 and ranks.attnT == 1 and tensorDtypes.hiddenT == "float32" and args.hiddenSize > 0 and inputSize > 0 and dim(shapes.hiddenT, 0) == tokenRows * args.hiddenSize and dim(shapes.attnT, 0) == tokenRows * inputSize and tokenRows >= 1 and tokenRows <= 4',oprojQ4Contract:'iqRawSel == 0 and oprojFormatOk and oprojBaseContract and q4Route and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.attnT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and inputSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or args.oOffset % 8 == 0) and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= (oprojMaxBlocks * (2 if lut2 else 4) if (q4knc or lutc) else oprojSuperblocks * 20 if q2k else oprojSuperblocks * 52 if q6k else oprojSuperblocks * 44 if q5k else oprojSuperblocks * 28 if q3k else oprojMaxBlocks * quantBits) and dim(shapes.scalesT, 0) >= (oprojSuperblocks * 5 if q4knc else oprojSuperblocks * lutcWords if lutc else oprojSuperblocks * 2 if q2k else oprojSuperblocks if q6k else oprojSuperblocks * 2 if q5k else oprojSuperblocks if q3k else oprojMaxBlocks * (2 if (hasMin or iq or lut16) else 1))',oprojIqBitsNeeded:"oprojSuperblocks * (20 if iqRawSel == 5 else (18 if iqRawSel == 6 else 12)) if iqRawSel >= 5 else oprojMaxBlocks * 2",oprojIqRawContract:'(iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 5 or iqRawSel == 6 or iqRawSel == 7) and oprojBaseContract and present.bitsT and present.scalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and tensorDtypes.attnT == "float32" and inputSize % 256 == 0 and args.oOffset % 8 == 0 and dim(shapes.bitsT, 0) >= oprojIqBitsNeeded and dim(shapes.scalesT, 0) >= oprojSuperblocks',downBaseContract:'args.intermediateSize is defined and not present.attnT and present.intermediateT and not present.upT and not present.colBitsT and not present.colScalesT and args.downOffset is defined and ranks.hiddenT == 1 and ranks.intermediateT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.intermediateT == "float32" or tensorDtypes.intermediateT == "float16") and args.hiddenSize > 0 and args.intermediateSize > 0 and dim(shapes.hiddenT, 0) == tokenRows * args.hiddenSize and dim(shapes.intermediateT, 0) == tokenRows * args.intermediateSize and tokenRows >= 1 and tokenRows <= 4',downSparseContract:'args.intermediateSize is defined and not present.attnT and present.upT and present.colBitsT and present.colScalesT and not present.intermediateT and not present.weightsT and not present.bitsT and not present.scalesT and args.colOffset is defined and args.scaleOffset is defined and args.actRelu2 == 1 and ranks.hiddenT == 1 and ranks.upT == 1 and ranks.colBitsT == 1 and ranks.colScalesT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.upT == "float32" and tensorDtypes.colBitsT == "uint32" and (tensorDtypes.colScalesT == "float32" or tensorDtypes.colScalesT == "float16") and f16Ok(dtypes.S) and args.hiddenSize > 0 and args.hiddenSize % 32 == 0 and args.intermediateSize > 0 and args.intermediateSize % 32 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.upT, 0) == args.intermediateSize and dim(shapes.colBitsT, 0) >= args.colOffset + args.intermediateSize * (args.hiddenSize / 8) and dim(shapes.colScalesT, 0) >= args.scaleOffset + (args.intermediateSize / 32) * args.hiddenSize * (2 if args.hasMin else 1)',downQ4Contract:'iqRawSel == 0 and downBaseContract and present.bitsT and fmt != "q1_0" and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT) and args.intermediateSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or args.downOffset % 8 == 0) and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= (downMaxBlocks * (2 if lut2 else 4) if (q4knc or lutc) else downSuperblocks * 20 if q2k else downSuperblocks * 52 if q6k else downSuperblocks * 44 if q5k else downSuperblocks * 28 if q3k else downMaxBlocks * quantBits) and dim(shapes.scalesT, 0) >= (downSuperblocks * 5 if q4knc else downSuperblocks * lutcWords if lutc else downSuperblocks * 2 if q2k else downSuperblocks if q6k else downSuperblocks * 2 if q5k else downSuperblocks if q3k else downMaxBlocks * (2 if (hasMin or iq or lut16) else 1))',downIqBitsNeeded:"downSuperblocks * (20 if iqRawSel == 5 else (18 if iqRawSel == 6 else (12 if iqRawSel == 7 else 14))) if iqRawSel >= 5 else downMaxBlocks * (3 if iqRawSel == 3 else 2)",downIqRawContract:'iqRawSel >= 1 and iqRawSel <= 8 and iqRawSel != 4 and downBaseContract and present.bitsT and present.scalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT) and args.intermediateSize % 256 == 0 and args.downOffset % 8 == 0 and dim(shapes.bitsT, 0) >= downIqBitsNeeded and dim(shapes.scalesT, 0) >= downSuperblocks',iqGridRows:"256 if iqRawSel == 3 else (512 if iqRawSel == 2 else (2048 if iqRawSel == 5 else (1024 if iqRawSel == 6 else 4096)))",iqGridBoundOk:'present.gridT and ranks.gridT == 1 and tensorDtypes.gridT == "uint32" and dim(shapes.gridT, 0) >= iqGridRows'},bindings:{hidden:{arg:"hiddenT",elementType:"f32"},attn_out:{arg:"attnT",elementType:"vec4<f32>"},q4_bits:{arg:"bitsT",elementType:"$bitsElem"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},intermediate_main:{arg:"intermediateT",name:"intermediate",elementType:"$intermediateVec4"}},variants:[{id:"oproj_q4",priority:0,when:["oprojQ4Contract","q4ResourcesFit","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{outputProjection:!0,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 32",oOffset:"args.oOffset",multiRow:"0",regRows:"1",scalesPk2:"0",workgroupSize:"subgroupCoveringWorkgroup",accumulate:"args.accumulate",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"'},passes:[{id:"main",name:"LlamaDecodeOProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","attn_out"],dispatch:{x:"args.hiddenSize"},subgroupCollectivesWidth:"portable"}]},{id:"down_q4",priority:0,when:["downQ4Contract","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",multiRow:"0",regRows:"1",workgroupSize:"subgroupCoveringWorkgroup",accumulate:"args.accumulate",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"',intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',downOffset:"args.downOffset",actRelu2:"1 if args.actRelu2 else 0"},passes:[{id:"main",name:"LlamaDecodeDownProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate_main"],dispatch:{x:"args.hiddenSize"},subgroupCollectivesWidth:"portable"}]},{cases:[{suffix:"mr2",priority:3,preference:["portableMr4Preferred","stageActivationsMeasured and args.hiddenSize % 16 == 0 and args.hiddenSize >= 4096"],divisor:8,regRows:"2"},{suffix:"mr4",priority:2,preference:null,divisor:16,regRows:"4"}],variants:[{id:"oproj_q4_{suffix}",priority:"{priority}",when:["oprojQ4Contract","args.hiddenSize % {divisor} == 0","q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"],demoteWhen:"{preference}",derive:{outputProjection:!0,useSubgroups:"wave32SubgroupsUsable",scaleScalar:'"u32" if (hasMin == 1 and tensorDtypes.scalesT == "float16") else dtypes.S',hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 32",oOffset:"args.oOffset",multiRow:"1",regRows:"{regRows}",wgClusters:"4",scalesPk2:'1 if (hasMin == 1 and tensorDtypes.scalesT == "float16") else 0',mRows:"tokenRows",stageActivations:"stageActivationsMeasured",workgroupSize:"subgroupCoveringWorkgroup",accumulate:"args.accumulate",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"'},passes:[{id:"main",name:"LlamaDecodeOProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","attn_out"],dispatch:{x:"ceil(args.hiddenSize / {divisor})"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{rows:2,priority:3,preference:["portableMr4Preferred","stageActivationsMeasured and args.hiddenSize % 16 == 0 and args.hiddenSize >= 4096"],divisor:8,regRows:"2"},{rows:4,priority:2,preference:null,divisor:16,regRows:"4"}],variants:[{id:"down_q4_mr{rows}",priority:"{priority}",when:["downQ4Contract","args.hiddenSize % {divisor} == 0","(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"],demoteWhen:"{preference}",derive:{outputProjection:!1,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",multiRow:"1",regRows:"{regRows}",mRows:"tokenRows",stageActivations:"stageActivationsMeasured",workgroupSize:"subgroupCoveringWorkgroup",accumulate:"args.accumulate",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"',intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',downOffset:"args.downOffset",actRelu2:"1 if args.actRelu2 else 0"},passes:[{id:"main",name:"LlamaDecodeDownProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate_main"],dispatch:{x:"ceil(args.hiddenSize / {divisor})"},subgroupCollectivesWidth:"portable"}]}]}]},assets:[["iq1-block-weights.e5f011a230.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% for r in range(regRows) %}let w{{ r }}=blk{{ r }}*2u;{{ iq1_regs(r ~ "", "q4_bits[w" ~ r ~ "]", "q4_bits[w" ~ r ~ " + 1u]", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endfor %}`],["iq3-block-weights.1f0e0af2dc.wgsl.jinja",`{% for r in range(regRows) %}let iw{{ r }}=blk{{ r }}*3u;{{ iq3xxs_regs(r ~ "", "q4_bits[iw" ~ r ~ "]", "q4_bits[iw" ~ r ~ " + 1u]", "q4_bits[iw" ~ r ~ " + 2u]", "f32(q4_scales[blk" ~ r ~ " >> 3u])") | trim }}
{% endfor %}`],["iq2-block-weights.f40d44d69b.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% for r in range(regRows) %}let sbw{{ r }}=(blk{{ r }}>>3u)*20u;let k{{ r }}=blk{{ r }}&7u;{{ iq2s_regs(r ~ "", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ "]", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ " + 1u]",
      "(q4_bits[sbw" ~ r ~ " + 16u + (k" ~ r ~ " >> 2u)] >> ((k" ~ r ~ " & 3u) * 8u)) & 0xFFu",
      "(q4_bits[sbw" ~ r ~ " + 18u + (k" ~ r ~ " >> 2u)] >> ((k" ~ r ~ " & 3u) * 8u)) & 0xFFu", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endfor %}`],["decode-q4-projection-multirow-tail.9be6ccfb6b.wgsl.jinja",`{% set hoistFormat = hoistFormat | default((
"iq3xxs"if(iq3xxs|default(false))else"iq2xxs"if((iq2xxs|default(false))and(regRows|default(1))>=4)else"iq2xs"if(iq2xs|default(false))else"iq2s"if(iq2s|default(false))else
  "iq1" if (iq1m | default(false)) else "") if (mRows > 1 and not (iqGridBound | default(false))) else "") %}{% set hoistBlockWeights = hoistBlockWeights | default(hoistFormat != "") %}{% set hoistTwoScale = hoistFormat == "iq1" or hoistFormat == "iq2xs" or hoistFormat == "iq2s" %}{% if hoistBlockWeights %}{% for r in range(regRows) %}let blk{{ r }}=base0+b+{{ r }}u*BPR;
{% endfor %}{% if hoistFormat == "iq1" %}{% include "iq1-block-weights.e5f011a230.wgsl.jinja" %}{% elif hoistFormat == "iq3xxs" %}{% include "iq3-block-weights.1f0e0af2dc.wgsl.jinja" %}{% else %}{% include "iq2-block-weights.f40d44d69b.wgsl.jinja" %}{% endif %}{% for m in range(mRows) %}{% set rowAcc = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% if m > 0 %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endif %}{% for r in range(regRows) %}{{ iq_dot_acc(r ~ "", rowAcc ~ r, "two" if hoistTwoScale else "one") | trim }}
{% endfor %}{% if m > 0 %}}
{% endif %}{% endfor %}{% else %}{% include "ops/_shared/q4-decode-row-block-accumulate.2710523b30.wgsl.jinja" %}{% for m in range(1, mRows) %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% set accPrefix = "accM" ~ m ~ "_" %}{% include "ops/_shared/q4-decode-row-block-accumulate.2710523b30.wgsl.jinja" %}{% set accPrefix = "acc" %}}
{% endfor %}{% endif %}}
{% if stageX | default(false) %}}
{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% set totalPrefix = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.ccd7057c03.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if (sgPath | default("")) == "general" %}{{ sg_cluster_store(sgClusters) }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let {{ "total" if m == 0 else "totalM" ~ m ~ "_" }}{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if (sgPath | default("")) == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% set rowTotal = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set rowBase = "" if m == 0 else m ~ "u * HIDDEN_SIZE + " %}{% for r in range(regRows) %}hidden[{{ rowBase }}dim0+{{ r }}u]={{ "" if (accumulate is defined and not accumulate) else "hidden[" ~ rowBase ~ "dim0 + " ~ r ~ "u] + " }}{{ rowTotal }}{{ r }};
{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}`],["decode-q4-projection-single-row-tail.70f6e5a1d5.wgsl.jinja",`{% include "ops/_shared/q4-decode-row-block-accumulate.2710523b30.wgsl.jinja" %}  }
{% if useSubgroups %}let total=sg_safe_sum(acc0,lid,SG_SAFE_WG,sgNative);
{% elif sgRuntime %}{{ sg_fold_workgroup([["acc0", "total"]]) }}
{% else %}partials[lid]=acc0;workgroupBarrier();
{% include "ops/_shared/tree-fold-inplace.629ddaa87a.wgsl.jinja" %}{{ wgsl_tree_fold(["partials"]) }}
let total=partials[0];
{% endif %}if(lid==0u){hidden[dim]={{ "" if (accumulate is defined and not accumulate) else "hidden[dim] + " }}total;} }`],["decode-residual-projection-q4.wgsl.jinja",`{% set mRows = mRows if mRows is defined else 1 %}{% set hoistFormat = (
"iq3xxs"if(iq3xxs|default(false))else"iq2xxs"if((iq2xxs|default(false))and(regRows|default(1))>=4)else"iq2xs"if(iq2xs|default(false))else"iq2s"if(iq2s|default(false))else
  "iq1" if (iq1m | default(false)) else "") if (mRows > 1 and not (iqGridBound | default(false))) else "" %}{% set hoistBlockWeights = hoistFormat != "" %}{% set xOpen = "attn_out[" if outputProjection else "act(vec4<f32>(intermediate[" %}{% set xClose = "]" if outputProjection else "]))" %}{% macro x_loads(base, note=false) %}{% if note and not outputProjection %}let hbase={{ base }};
{% else %}let hbase={{ base }};
{% endif %}{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endmacro %}{% set emitQuantDotHelpers = true %}{% set nibbleHelpers = not (q2k or q3k or q5k or q6k) %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2s | default(false)) or (((iq2xxs | default(false)) or (iq2xs | default(false)) or (iq3xxs | default(false))) and (iqGridBound | default(false))) or (not ((iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false))) and not lutId and (((quantBits != 8) and nibbleHelpers) or q2k or q3k or q5k or q6k)) %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if not multiRow or mRows > 1 %}const HIDDEN_SIZE:u32={{ hiddenSize }}u;
{% endif %}{% if outputProjection %}const BPR:u32={{ blocksPerRow }}u;const W_O_Q4:u32={{ oOffset }}u;
{% else %}const BPR:u32={{ blocksPerRow }}u;const W_DOWN_Q4:u32={{ downOffset }}u;
{% endif %}{% if not multiRow %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% set sgWorkgroupSize = (32 * wgClusters if outputProjection else 128) if multiRow else workgroupSize %}{% include "ops/_shared/subgroup-runtime-fold.f33ebc1f56.wgsl.jinja" %}{% set stageX = (stageActivations | default(false)) and multiRow and mRows == 1 and (useSubgroups or sgRuntime) and not ((iq1s | default(false)) or (iq1md | default(false))) %}{% set sgOwned = multiRow and useSubgroups and not stageX and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.87076f64db.wgsl.jinja" %}{% set sgClusters = (wgClusters if outputProjection else 4) ~ "u" %}{% set sgDim0 = "(wid.x * " ~ sgClusters ~ " + cluster) * " ~ regRows ~ "u" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = sgWorkgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if stageX %}const X_STAGE_BLOCKS:u32=64u;var<workgroup>x_sh:array<vec4<f32>,512>;fn x_slot(j:u32)->u32{return j^((j>>3u)&7u);}
{% endif %}{% if not useSubgroups and not sgRuntime %}{% if multiRow %}{% set partialsRowVector = true %}var<workgroup>partials4:array<vec{{ regRows }}<f32>,{{ sgWorkgroupSize }}>;
{% else %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% include "ops/_shared/q4-dequant-helpers.33f455d750.wgsl.jinja" %}{% if not outputProjection %}{% if actRelu2 %}fn act(v:vec4<f32>)->vec4<f32>{let r=max(v,vec4<f32>(0.0));return r*r;}
{% else %}fn act(v:vec4<f32>)->vec4<f32>{return v;}
{% endif %}{% endif %}{% if multiRow %}{% if outputProjection %}{% else %}{% endif %}@compute@workgroup_size({{ 32 * wgClusters if outputProjection else 128 }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}{{ ", @builtin(local_invocation_index) lut_tid: u32" if (lutStaged | default(false)) else "" }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% set laneStride = "activeCount" if sgPath == "general" else "32u" %}{% if sgOwned %}{{ sg_cluster_open(sgPath, sgClusters, (wgClusters if outputProjection else 4) * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*{{ wgClusters if outputProjection else 4 }}u+cluster)*{{ regRows }}u;let base0={{ "W_O_Q4" if outputProjection else "W_DOWN_Q4" }}+dim0*BPR;
{% for m in range(mRows) %}{% for r in range(regRows) %}var {{ "acc" ~ r if m == 0 else "accM" ~ m ~ "_" ~ r }}=0.0;
{% endfor %}{% endfor %}{% if stageX %}for(var c0=0u;c0<BPR;c0=c0+X_STAGE_BLOCKS){workgroupBarrier();for(var i=lid;i<X_STAGE_BLOCKS*8u;i=i+{{ 32 * wgClusters if outputProjection else 128 }}u){
{% if blocksPerRow % 64 != 0 %}if(c0*8u+i<BPR*8u){x_sh[x_slot(i)]={{ xOpen }}c0*8u+i{{ xClose }};}
{% else %}x_sh[x_slot(i)]={{ xOpen }}c0*8u+i{{ xClose }};
{% endif %}}workgroupBarrier();for(var b=c0+lane;b<min(c0+X_STAGE_BLOCKS,BPR);b=b+32u){let hl=(b-c0)*8u;
{% for i in range(8) %}let x{{ i }}=x_sh[x_slot(hl+{{ i }}u)];
{% endfor %}{% else %}for(var b=lane;b<BPR;b=b+{{ laneStride }}){ {{ x_loads("b * 8u") | trim }}
{% endif %}{% include "decode-q4-projection-multirow-tail.9be6ccfb6b.wgsl.jinja" %}{% endfor %}}
{% else %}{% if outputProjection %}{% else %}{% endif %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let dim=wid.x;let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(dim>=HIDDEN_SIZE){return;}let base0={{ "W_O_Q4" if outputProjection else "W_DOWN_Q4" }}+dim*BPR;var acc0=0.0;for(var b=lid;b<BPR;b=b+WORKGROUP_SIZE){ {{ x_loads("b * 8u", true) | trim }}
{% include "decode-q4-projection-single-row-tail.70f6e5a1d5.wgsl.jinja" %}{% endif %}`]],assumes:[["decode-residual-projection-q4.wgsl.jinja",{quantBits:4,q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q5k:0,lutId:0}]]}],["com.xenova.LlamaDecodeShortConv",{manifest:{domain:"com.xenova",name:"LlamaDecodeShortConv",sinceVersion:1,inputs:{weightsT:{onnx:"NormWeights",dtype:"W",rank:1},normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},bcxT:{onnx:"Bcx",dtype:"float32",rank:1,optional:!0},convStatesT:{onnx:"ConvStates",dtype:"float32",rank:1}},outputs:{convStatesT:{onnx:"ConvStates",dtype:"float32",rank:1,shape:"shapes.convStatesT"},convYT:{onnx:"ConvY",dtype:"float32",rank:1,shape:["args.hiddenSize"]}},args:{layer:{kind:"u32",required:!0},convLCache:{kind:"u32",onnx:"conv_l_cache",required:!0},convWeightOffset:{kind:"u32",onnx:"conv_weight_offset",required:!0},convInOffset:{kind:"u32",onnx:"conv_in_offset",required:!1},hiddenSize:{kind:"u32",onnx:"hidden_size",required:!0}},typeConstraints:{W:["float32","float16","uint32"],N:["float32","float16"]},derive:{weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',baseContract:'args.layer is defined and ranks.weightsT == 1 and ranks.convStatesT == 1 and ranks.convYT == 1 and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and tensorDtypes.convStatesT == "float32" and tensorDtypes.convYT == "float32" and f16Ok(tensorDtypes.weightsT) and args.hiddenSize > 0 and args.convLCache > 0 and args.convWeightOffset % 4 == 0 and dim(shapes.convYT, 0) == args.hiddenSize and dim(shapes.convStatesT, 0) >= (args.layer + 1) * args.hiddenSize * args.convLCache and weightElements >= args.convWeightOffset + args.hiddenSize * args.convLCache',weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec2<u32>" if tensorDtypes.weightsT == "uint32" else "vec4<f32>"',layer:"args.layer",hiddenSize:"args.hiddenSize",convLCache:"args.convLCache",convWeightOffset:"args.convWeightOffset",convWeightElem:'"f16" if tensorDtypes.weightsT == "float16" else "u32" if tensorDtypes.weightsT == "uint32" else "f32"'},variants:[{id:"preprojected",priority:0,when:["present.bcxT","not present.normedT","baseContract",'ranks.bcxT == 1 and tensorDtypes.bcxT == "float32" and dim(shapes.bcxT, 0) == 3 * args.hiddenSize'],derive:{workgroupSize:256},passes:[{id:"main",name:"LlamaDecodeConvDepthwise",shader:"decode-conv-depthwise.wgsl.jinja",bindings:[{arg:"bcxT",name:"conv_proj"},{arg:"weightsT",name:"conv_weights",elementType:"$convWeightElem"},{arg:"convStatesT",name:"conv_states"},{arg:"convYT",name:"conv_y"}],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]}]},assets:[["conv-state-update.34e5c06ac2.wgsl.jinja",`fn conv_weight(dim:u32,tap:u32)->f32{return f32(conv_weights[W_CONV+dim*CONV_L_CACHE+tap]);}
{% macro conv_state_update(bx, gate) %}let bx={{ bx }};var conv_out=bx*conv_weight(dim_out,CONV_L_CACHE-1u);for(var tap=0u;tap+1u<CONV_L_CACHE;tap=tap+1u){let next=conv_states[state_index(dim_out,tap+1u)];conv_states[state_index(dim_out,tap)]=next;conv_out=fma(next,conv_weight(dim_out,tap),conv_out);}conv_states[state_index(dim_out,CONV_L_CACHE-1u)]=bx;
conv_y[dim_out] = {{ gate }} * conv_out;{% endmacro %}`],["decode-conv-depthwise.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const HIDDEN_SIZE:u32={{ hiddenSize }}u;const CONV_L_CACHE:u32={{ convLCache }}u;const W_CONV:u32={{ convWeightOffset }}u;fn state_index(dim:u32,tap:u32)->u32{return(LAYER*HIDDEN_SIZE+dim)*CONV_L_CACHE+tap;}
{% include "conv-state-update.34e5c06ac2.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim_out=gid.x;if(dim_out>=HIDDEN_SIZE){return;}let proj_b=conv_proj[dim_out];let proj_c=conv_proj[HIDDEN_SIZE+dim_out];let proj_x=conv_proj[2u*HIDDEN_SIZE+dim_out];{{ conv_state_update("proj_b * proj_x", "proj_c") | indent(2, true) }} }`]],assumes:[["decode-conv-depthwise.wgsl.jinja",{convWeightElem:"f32"}]]}],["com.xenova.LlamaEmbed",{manifest:{domain:"com.xenova",name:"LlamaEmbed",sinceVersion:1,inputs:{inputT:{onnx:"InputToken",dtype:"uint32",rank:1},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},signsT:{onnx:"Signs",dtype:"float32",rank:1,optional:!0}},outputs:{hiddenT:{onnx:"Hidden",dtype:"Y",rank:"2 if args.seqLen is defined else 1",shape:"[args.seqLen, args.hiddenSize] if args.seqLen is defined else [args.hiddenSize]"}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},vocabSize:{kind:"u32",onnx:"vocab_size"},embedOffset:{kind:"u32",onnx:"embed_offset",required:!1},tokenOffset:{kind:"u32",onnx:"token_offset",required:!1},pastLen:{kind:"u32",onnx:"past_len",required:!1},cacheLen:{kind:"u32",onnx:"cache_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len",required:!1},format:{kind:"string",required:!1,oneOf:["q1_0","q8_rows","q8_0","q4_0","q4_k","q4_k_native_compact","q6_k","q5_k","q3_k","q2_k","lut2_32"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},rotationBlockSize:{kind:"u32",onnx:"rotation_block_size",required:!1}},typeConstraints:{W:["float32","float16","uint32"],S:["float32","float16","uint32"],Y:["float32","float16"]},tunables:{EMBED_OUT_VEC4S:{default:4}},derive:{weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',decodeMode:"args.seqLen is undefined",prefillMode:"args.seqLen is defined",denseMode:"present.weightsT and not present.bitsT",q1Mode:'present.bitsT and present.scalesT and args.format == "q1_0"',q8Mode:'present.bitsT and present.scalesT and (args.format == "q8_rows" or args.format == "q8_0")',q8BlockScales:'args.format == "q8_0"',q8ScaleCount:"args.vocabSize * (args.hiddenSize / 32 if q8BlockScales else 1)",rotationMode:"present.signsT or args.rotationBlockSize is defined",decodeShape:'decodeMode and ranks.inputT == 1 and ranks.hiddenT == 1 and tensorDtypes.inputT == "uint32" and tensorDtypes.hiddenT == "float32" and args.hiddenSize > 0 and args.vocabSize > 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.inputT, 0) > (args.tokenOffset if args.tokenOffset else 0)',decodeContract:"decodeShape and not rotationMode",rotationContract:'decodeShape and present.signsT and args.rotationBlockSize is defined and ranks.signsT == 1 and tensorDtypes.signsT == "float32" and numel(shapes.signsT) >= args.hiddenSize and args.rotationBlockSize in [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] and args.hiddenSize % args.rotationBlockSize == 0',prefillContract:'prefillMode and ranks.inputT == 1 and ranks.hiddenT == 2 and tensorDtypes.inputT == "uint32" and args.hiddenSize > 0 and args.seqLen > 0 and args.vocabSize > 0 and dim(shapes.hiddenT, 0) == args.seqLen and dim(shapes.hiddenT, 1) == args.hiddenSize and numel(shapes.inputT) >= args.seqLen and not rotationMode',prefillF16Ok:"f16Ok(dtypes.Y)",kq4Mode:'present.bitsT and present.scalesT and (args.format == "q4_0" or args.format == "q4_k")',kq4Blocks:"args.vocabSize * (args.hiddenSize / 32) if (args.vocabSize is defined and args.hiddenSize is defined) else 0",kq4Tables:'kq4Mode and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 32 == 0 and dim(shapes.bitsT, 0) >= kq4Blocks * 4 and dim(shapes.scalesT, 0) >= kq4Blocks * (2 if args.format == "q4_k" else 1)',lut2Tables:'present.bitsT and present.scalesT and args.format == "lut2_32" and args.lut is defined and args.lut in [8, 9] and not args.embedOffset and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.hiddenSize % 256 == 0 and dim(shapes.bitsT, 0) >= kq4Blocks * 2 and dim(shapes.scalesT, 0) >= (kq4Blocks / 8) * (3 if args.lut == 8 else 1)',kSupers:"kq4Blocks / 8",kBitsWords:'52 if args.format == "q6_k" else (44 if args.format == "q5_k" else (20 if args.format == "q2_k" else 28))',kScaleWords:'2 if (args.format == "q5_k" or args.format == "q2_k") else 1',embedOutVec4sWant:"max(1, min(tunables.EMBED_OUT_VEC4S, 8))",embedOutVec4s:"embedOutVec4sWant if args.hiddenSize % (4 * embedOutVec4sWant) == 0 else 1",hiddenSize:"args.hiddenSize",workgroupSize:64,wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",gatherRows:"args.seqLen if args.seqLen is defined else 1",gatherScaleBytes:'2 if tensorDtypes.scalesT == "float16" else 4',gatherRowBytes:'args.hiddenSize * (2 if tensorDtypes.weightsT in ["float16", "uint32"] else 4) if denseMode else ((args.hiddenSize / 128) * (16 + gatherScaleBytes) if q1Mode else (args.hiddenSize + gatherScaleBytes * (args.hiddenSize / 32 if q8BlockScales else 1) if q8Mode else ((args.hiddenSize / 32) * (16 + (2 if args.format == "q4_k" else 1) * gatherScaleBytes) if kq4Mode else (args.hiddenSize / 4 + (args.hiddenSize / 256) * (8 if args.lut == 8 else 4) if args.format == "lut2_32" else ((args.hiddenSize / 256) * 148 if args.format == "q4_k_native_compact" else (args.hiddenSize / 256) * (kBitsWords * 4 + kScaleWords * gatherScaleBytes))))))',gatherBytesMoved:'gatherRows * (4 + gatherRowBytes + args.hiddenSize * (2 if tensorDtypes.hiddenT == "float16" else 4)) + (args.hiddenSize * 4 if present.signsT else 0)'},bindings:{input_token:{arg:"inputT",elementType:"u32"},hidden:{arg:"hiddenT",elementType:"f32"},params:{arg:"paramsT",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0",required:!0,internal:!0},{name:"cache_len",type:"u32",value:"args.cacheLen if args.cacheLen else 0",required:!0,internal:!0},{name:"token_offset",type:"u32",value:"args.tokenOffset if args.tokenOffset else 0"}]},embed_q8:{arg:"bitsT",elementType:"u32"},embed_scales:{arg:"scalesT",elementType:"$scaleScalar"},tokens:{arg:"inputT",elementType:"u32"},hidden_main:{arg:"hiddenT",name:"hidden",elementType:"$outVec4"},params_seq_len:{name:"params",struct:[{name:"seq_len",type:"u32",value:"args.seqLen"}]},q6_bits:{arg:"bitsT",elementType:"u32"},q6_scales:{arg:"scalesT",elementType:"$scaleScalar"}},variants:[{id:"prefill_q8",priority:0,when:["prefillContract","q8Mode","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"prefillF16Ok and f16Ok(dtypes.S)","args.hiddenSize % (32 if q8BlockScales else 4) == 0","dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4)","dim(shapes.scalesT, 0) >= q8ScaleCount"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",cols4:"args.hiddenSize / 4",blockScales:"q8BlockScales",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedQ8",shader:"prefill-embed-q8.wgsl.jinja",bindings:["tokens","embed_q8","embed_scales","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"},profile:{bytesMoved:"gatherBytesMoved"}}]},{id:"q6k",priority:0,when:["decodeContract",'present.bitsT and present.scalesT and args.format == "q6_k"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= kSupers * kBitsWords","dim(shapes.scalesT, 0) >= kSupers * kScaleWords","f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",blocksPerRow:"args.hiddenSize / 32",isPrefill:!1},passes:[{id:"main",name:"LlamaEmbedQ6K",shader:"embed-q6k.wgsl.jinja",bindings:["input_token","q6_bits","q6_scales","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"},profile:{bytesMoved:"gatherBytesMoved"}}]},{id:"prefill_q6k",priority:0,when:["prefillContract",'present.bitsT and present.scalesT and args.format == "q6_k"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= kSupers * kBitsWords","dim(shapes.scalesT, 0) >= kSupers * kScaleWords","prefillF16Ok and f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",blocksPerRow:"args.hiddenSize / 32",isPrefill:!0,outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedQ6K",shader:"embed-q6k.wgsl.jinja",bindings:["tokens","q6_bits","q6_scales","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"},profile:{bytesMoved:"gatherBytesMoved"}}]}]},assets:[["prefill-embed-q8.wgsl.jinja",`{% include "ops/_shared/packed-4x8-compat.0c5ce8f6e4.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const HIDDEN_SIZE:u32={{ hiddenSize }}u;const COLS4:u32={{ cols4 }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let t=g/ROW_VEC4S;let span=g%ROW_VEC4S;let token=tokens[t];
{% if not blockScales %}let scale=f32(embed_scales[token]);
{% endif %}let base=token*COLS4+span*OUT_VEC4S;
{% for v in range(outVec4s) %}let lanes{{ v }}=unpack4xI8(embed_q8[base+{{ v }}u]);
{% if blockScales %}let scale{{ v }}=f32(embed_scales[(base+{{ v }}u)/8u]);
{% endif %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}(f32(lanes{{ v }}[{{ c }}])*scale{{ v if blockScales else "" }}){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}`],["embed-native-k-driver.dc3799a37c.wgsl.jinja",`{% if isPrefill %}const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let token_index=g/ROW_VEC4S;let span=g%ROW_VEC4S;let dim=span*4u*OUT_VEC4S;let block=tokens[token_index]*BLOCKS_PER_ROW+dim/32u;let lo=dim&31u;
{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}({{ nativeKValueFn }}(block,lo+{{ v * 4 + c }}u)){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}
{% else %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let block=input_token[params.token_offset]*BLOCKS_PER_ROW+dim/32u;hidden[dim]={{ nativeKValueFn }}(block,dim&31u);}
{% endif %}`],["embed-q6k.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;fn q6_value(block:u32,element_in_block:u32)->f32{let superblock=block>>3u;let j=block&7u;let plane=element_in_block>>4u;let e=element_in_block&15u;let word=q6_bits[superblock*52u+j*4u+(e>>2u)];let nibble=(word>>((e&3u)*8u+plane*4u))&15u;let hi_word=q6_bits[superblock*52u+32u+j*2u+plane];let hi2=(hi_word>>((e&3u)*8u+(e>>2u)*2u))&3u;let scale_word=q6_bits[superblock*52u+48u+(j>>1u)];let scale_byte=(scale_word>>(((j&1u)*2u+plane)*8u))&255u;let sub_scale=select(i32(scale_byte),i32(scale_byte)-256,scale_byte>=128u);let code=i32(nibble|(hi2<<4u))-32;let scale=f32(q6_scales[superblock])*f32(sub_scale);return f32(code)*scale;}
{% set nativeKValueFn = "q6_value" %}{% include "embed-native-k-driver.dc3799a37c.wgsl.jinja" %}`]],assumes:[["prefill-embed-q8.wgsl.jinja",{outScalar:"f32",outVec4:"vec4<f32>"}]]}],["com.xenova.LlamaPrefillConv",{manifest:{domain:"com.xenova",name:"LlamaPrefillConv",sinceVersion:1,inputs:{bcxT:{onnx:"Bcx",dtype:"A",rank:2},weightsT:{onnx:"Weights",dtype:"W",rank:1},convStatesT:{onnx:"ConvStates",dtype:"C",rank:1}},outputs:{convYT:{onnx:"ConvY",dtype:"A",rank:2,shape:["args.seqLen","args.hiddenSize"]},convStatesT:{onnx:"ConvStates",dtype:"C",rank:1,shape:["args.hiddenSize * args.convLCache"]},snapshotsT:{onnx:"ConvSnapshots",dtype:"C",rank:1,optional:!0}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},convLCache:{kind:"u32",onnx:"conv_l_cache"},convWeightOffset:{kind:"u32",onnx:"conv_weight_offset"},seqLen:{kind:"u32",onnx:"seq_len"},realLen:{kind:"u32",onnx:"real_len",required:!1},snapshotOffset:{kind:"u32",onnx:"snapshot_offset",required:!1},snapshotStride:{kind:"u32",onnx:"snapshot_stride",required:!1}},typeConstraints:{A:["float32","float16"],W:["float32","float16","uint32"],C:["float32"]},derive:{baseContract:'ranks.bcxT == 2 and ranks.weightsT == 1 and ranks.convStatesT == 1 and ranks.convYT == 2 and (tensorDtypes.bcxT == "float32" or tensorDtypes.bcxT == "float16") and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and tensorDtypes.convStatesT == "float32" and f16Ok(tensorDtypes.bcxT) and f16Ok(tensorDtypes.weightsT) and args.hiddenSize > 0 and args.seqLen > 0 and args.convLCache > 0 and dim(shapes.bcxT, 1) == 3 * args.hiddenSize and numel(shapes.bcxT) >= args.seqLen * 3 * args.hiddenSize and numel(shapes.convYT) >= args.seqLen * args.hiddenSize and numel(shapes.convStatesT) >= args.hiddenSize * args.convLCache and numel(shapes.weightsT) * (2 if tensorDtypes.weightsT == "uint32" else 1) >= args.convWeightOffset + args.hiddenSize * args.convLCache',activationScalar:"dtypes.A",weightScalar:"dtypes.W",hiddenSize:"args.hiddenSize",convLCache:"args.convLCache",threeHidden:"3 * args.hiddenSize",seqLen:"args.seqLen",convWeightOffset:"args.convWeightOffset",workgroupSize:"64"},when:["baseContract"],bindings:{bcx:{arg:"bcxT",elementType:"$activationScalar"},weights:{arg:"weightsT",elementType:"$weightScalar"},conv_states:{arg:"convStatesT",buffer:"read-only-storage",elementType:"f32"},conv_y:{arg:"convYT",elementType:"$activationScalar"},conv_states_f32:{arg:"convStatesT",name:"conv_states",elementType:"f32"},params:{struct:[{name:"past_len",type:"u32",value:"0",required:!0,internal:!0},{name:"cache_len",type:"u32",value:"0",required:!0,internal:!0},{name:"seq_len",type:"u32",value:"args.seqLen",required:!0,internal:!0},{name:"real_len",type:"u32",value:"args.realLen if args.realLen else args.seqLen"}]}},variants:[{id:"direct_small_kernel",priority:5,when:["args.convLCache <= 4","not present.snapshotsT"],passes:[{id:"conv_y",name:"LlamaPrefillConv.DirectY",shader:"prefill-conv-direct-y.wgsl.jinja",derive:{workgroupSize:"256"},bindings:["bcx","weights","conv_states","conv_y"],dispatch:{threads:"args.seqLen * args.hiddenSize",workgroupSize:"workgroupSize"}},{id:"state",name:"LlamaPrefillConv.DirectState",shader:"prefill-conv-direct-state.wgsl.jinja",bindings:["bcx","conv_states_f32","params"],dispatch:{workgroups:"args.hiddenSize"}}]},{id:"scalar",priority:0,when:["args.convLCache <= 64","not present.snapshotsT","4 * args.seqLen + 4 * args.convLCache <= device.limits.maxComputeWorkgroupStorageSize"],passes:[{id:"main",name:"LlamaPrefillConv",shader:"prefill-conv.wgsl.jinja",bindings:["bcx","weights","conv_states_f32","conv_y","params"],dispatch:{x:"args.hiddenSize"}}]}]},assets:[["prefill-conv-direct-y.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const CONV_L:u32={{ convLCache }}u;const BCX_STRIDE:u32={{ threeHidden }}u;const SEQ_LEN:u32={{ seqLen }}u;const W_CONV:u32={{ convWeightOffset }}u;const WG:u32={{ workgroupSize }}u;const COUNT:u32=SEQ_LEN*HIDDEN_SIZE;fn conv_w(dim:u32,tap:u32)->f32{return f32(weights[W_CONV+dim*CONV_L+tap]);}fn bx_at(j:i32,dim:u32)->f32{if(j>=0){let t=u32(j);let base=t*BCX_STRIDE+dim;return f32(bcx[base])*f32(bcx[base+2u*HIDDEN_SIZE]);}return conv_states[dim*CONV_L+u32(i32(CONV_L)+j)];}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let i=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*WG;if(i>=COUNT){return;}let dim=i%HIDDEN_SIZE;let t=i/HIDDEN_SIZE;var acc=0.0;for(var k=0u;k<CONV_L;k=k+1u){let j=i32(t)-i32(CONV_L-1u)+i32(k);acc=fma(conv_w(dim,k),bx_at(j,dim),acc);}let c=f32(bcx[t*BCX_STRIDE+HIDDEN_SIZE+dim]);conv_y[t*HIDDEN_SIZE+dim]={{ activationScalar }}(c*acc);}"],["bx-at.281c0edbf6.wgsl.jinja","fn bx_at(j:i32,dim:u32)->f32{if(j>=0){let t=u32(j);let base=t*BCX_STRIDE+dim;return f32(bcx[base])*f32(bcx[base+2u*HIDDEN_SIZE]);}return old_state[u32(i32(CONV_L)+j)];}"],["prefill-conv-direct-state.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const CONV_L:u32={{ convLCache }}u;const BCX_STRIDE:u32={{ threeHidden }}u;var<workgroup>old_state:array<f32,{{ convLCache }}>;
{% include "bx-at.281c0edbf6.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let dim=wid.x+wid.y*{{ DISPATCH_FOLD_WIDTH }}u;if(dim>=HIDDEN_SIZE){return;}let lid=lid3.x;if(lid<CONV_L){old_state[lid]=conv_states[dim*CONV_L+lid];}workgroupBarrier();if(lid<CONV_L){let j=i32(params.real_len)-i32(CONV_L)+i32(lid);conv_states[dim*CONV_L+lid]=bx_at(j,dim);} }`],["prefill-conv.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const CONV_L:u32={{ convLCache }}u;const BCX_STRIDE:u32={{ threeHidden }}u;const SEQ_LEN:u32={{ seqLen }}u;const W_CONV:u32={{ convWeightOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>bx_buf:array<f32,{{ seqLen }}>;var<workgroup>old_state:array<f32,{{ convLCache }}>;fn conv_w(dim:u32,tap:u32)->f32{return f32(weights[W_CONV+dim*CONV_L+tap]);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let dim=wid.x;if(dim>=HIDDEN_SIZE){return;}let lid=lid3.x;if(lid<CONV_L){old_state[lid]=conv_states[dim*CONV_L+lid];}workgroupBarrier();for(var t=lid;t<SEQ_LEN;t=t+WORKGROUP_SIZE){let b=f32(bcx[t*BCX_STRIDE+dim]);let x=f32(bcx[t*BCX_STRIDE+2u*HIDDEN_SIZE+dim]);bx_buf[t]=b*x;}workgroupBarrier();for(var t=lid;t<SEQ_LEN;t=t+WORKGROUP_SIZE){var acc=0.0;for(var k=0u;k<CONV_L;k=k+1u){let j=i32(t)-i32(CONV_L-1u)+i32(k);var inp=0.0;if(j>=0){inp=bx_buf[u32(j)];}else{inp=old_state[u32(i32(CONV_L)+j)];}acc=fma(conv_w(dim,k),inp,acc);}let c=f32(bcx[t*BCX_STRIDE+HIDDEN_SIZE+dim]);conv_y[t*HIDDEN_SIZE+dim]={{ activationScalar }}(c*acc);}if(lid<CONV_L){let j=i32(params.real_len)-i32(CONV_L)+i32(lid);var inp=0.0;if(j>=0){inp=bx_buf[u32(j)];}else{inp=old_state[u32(i32(CONV_L)+j)];}conv_states[dim*CONV_L+lid]=inp;} }"]],assumes:[["prefill-conv-direct-y.wgsl.jinja",{activationScalar:"f32",weightScalar:"f32"}],["prefill-conv.wgsl.jinja",{activationScalar:"f32",weightScalar:"f32"}]]}],["com.xenova.LlamaPrefillDequant",{manifest:{domain:"com.xenova",name:"LlamaPrefillDequant",sinceVersion:1,inputs:{bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1},denseT:{onnx:"Dense",dtype:"D",rank:1}},outputs:{denseT:{onnx:"Dense",dtype:"D",rank:1,shape:["args.denseElems"]}},args:{denseElems:{kind:"u32",onnx:"dense_elems"},blockOffset:{kind:"u32",onnx:"block_offset"},outRows:{kind:"u32",onnx:"out_rows"},inFeatures:{kind:"u32",onnx:"in_features"},dstElemOffset:{kind:"u32",onnx:"dst_elem_offset"},format:{kind:"string",oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q5_0","q2_k","q3_k","q6_k","q5_k","q1_0"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]}},typeConstraints:{S:["float32","float16","uint32"],D:["float32","float16"]},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',q4knc:'1 if fmt == "q4_k_native_compact" else 0',dqBlocks:"(args.blockOffset + args.outRows * (args.inFeatures / 32)) if (args.blockOffset is defined and args.outRows is defined and args.inFeatures is defined) else 0",scaleScalar:"dtypes.S",denseScalar:"dtypes.D",workgroupSize:64},when:["ranks.bitsT == 1","ranks.scalesT == 1","ranks.denseT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.denseT == "float32" or tensorDtypes.denseT == "float16")',"f16Ok(tensorDtypes.scalesT)","f16Ok(tensorDtypes.denseT)","args.outRows > 0","args.inFeatures > 0","args.denseElems >= args.dstElemOffset + args.outRows * args.inFeatures","numel(shapes.denseT) >= args.dstElemOffset + args.outRows * args.inFeatures"],bindings:{dense:{arg:"denseT",elementType:"$denseScalar"},params:{struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"out_rows",type:"u32",value:"args.outRows"},{name:"in_features",type:"u32",value:"args.inFeatures"},{name:"dst_elem_offset",type:"u32",value:"args.dstElemOffset"}]}},variants:[{id:"q4",priority:0,when:['present.bitsT and fmt != "q1_0"','(tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16"))',"(quantBits == 4 or quantBits == 8)","args.inFeatures % 32 == 0 and ((q4knc == 0 and lutc == 0) or args.blockOffset % 8 == 0)","numel(shapes.scalesT) >= (ceilDiv(dqBlocks, 8) * 5 if q4knc else ceilDiv(dqBlocks, 8) * lutcWords if lutc else dqBlocks / 4 if q2k else dqBlocks / 8 if q6k else dqBlocks / 4 if q5k else dqBlocks / 8 if q3k else dqBlocks * (2 if (hasMin or iq or lut16) else 1))","numel(shapes.bitsT) >= dqBlocks * (4 if q4knc else 2.5 if q2k else 6.5 if q6k else 5.5 if q5k else 3.5 if q3k else 5 if q5 else 2 if lut2 else quantBits)"],derive:{lutId:"args.lut if args.lut else 0"},passes:[{id:"main",name:"LlamaPrefillDequant",shader:"prefill-dequant.wgsl.jinja",bindings:[{arg:"bitsT",name:"q4_bits"},{arg:"scalesT",name:"q4_scales",elementType:"$scaleScalar"},"dense","params"],dispatch:{threads:"args.outRows * args.inFeatures / 4",workgroupSize:"workgroupSize"}}]}]},assets:[["prefill-dequant.wgsl.jinja",`{% set packed4x8CompatUnpackI8 = quantBits == 8 or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not (quantBits == 8 or (lut2 | default(false)) or lutId or q5 or q2k or q3k or q6k or q5k) %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let group=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total4=(params.out_rows*params.in_features)/4u;if(group>=total4){return;}let elem=group*4u;let r=elem/params.in_features;let c=elem%params.in_features;let bpr=params.in_features/32u;let blk=params.block_offset+r*bpr+c/32u;let eib=c%32u;
{% if quantBits == 8 %}let word=q4_bits[blk*8u+eib/4u];let scale=f32(q4_scales[blk]);let v=vec4<f32>(unpack4xI8(word))*scale;
{% elif q4knc %}let word=q4_bits[blk*4u+(eib%16u)/4u];let sbC=blk>>3u;let jC=blk&7u;let smBaseC=sbC*5u;let dmC=unpack2x16float(q4_scales[smBaseC]);let smPairC=q4_scales[smBaseC+1u+(jC>>1u)]>>((jC&1u)*12u);let dlC=dmC.x*f32(smPairC&63u);let mlC=dmC.y*f32((smPairC>>6u)&63u);var nibC:vec4<f32>;if(eib<16u){nibC=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu));}else{nibC=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu));}let v=nibC*dlC-vec4<f32>(mlC);
{% else %}let word=q4_bits[blk*4u+(eib%16u)/4u];let scale=f32(q4_scales[blk]);var nib:vec4<f32>;if(eib<16u){nib=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}else{nib=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}let v=nib*scale;
{% endif %}let o=params.dst_elem_offset+elem;dense[o]={{ denseScalar }}(v.x);dense[o+1u]={{ denseScalar }}(v.y);dense[o+2u]={{ denseScalar }}(v.z);dense[o+3u]={{ denseScalar }}(v.w);}`]],assumes:[["prefill-dequant.wgsl.jinja",{hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q6k:0,q5k:0,denseScalar:"f32",lutId:0}]]}],["com.xenova.LlamaPrefillMatmul",{manifest:{domain:"com.xenova",name:"LlamaPrefillMatmul",sinceVersion:1,inputs:{aT:{onnx:"A",dtype:"A",rank:2},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},realLenT:{onnx:"RealLen",dtype:"uint32",rank:1,optional:!0},yT:{onnx:"Y",dtype:"Y",rank:2},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0},bT:{dtype:"float32",rank:1,optional:!0},rT:{dtype:"float32",rank:2,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:2,shape:["args.M","args.outStride if args.outStride else args.outFeatures"]},partialsT:{onnx:"Partials",dtype:"float32",rank:1,optional:!0}},args:{M:{kind:"u32"},inFeatures:{kind:"u32",onnx:"in_features"},outFeatures:{kind:"u32",onnx:"out_features"},weightOffset:{kind:"u32",onnx:"weight_offset",required:!1},blockOffset:{kind:"u32",onnx:"block_offset",required:!1},outStride:{kind:"u32",onnx:"out_stride",required:!1},dstColStart:{kind:"u32",onnx:"dst_col_start",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","q5_0","q2_k","q3_k","q6_k","q5_k","q1_0","lut2_32","lut2_16","bf16","f16"]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},exact:{kind:"u32",onnx:"exact_f32_tiles",required:!1},activationQuant:{kind:"string",required:!1,oneOf:["int8"]},kSplits:{kind:"u32",onnx:"k_splits",required:!1},denseTailStart:{kind:"u32",required:!1,onnx:"dense_tail_start"},activation:{kind:"string",required:!1,oneOf:["none","tanh","erf"]}},typeConstraints:{A:["float32","float16"],W:["float32","float16"],S:["float32","float16","uint32"],Y:["float32","float16"]},tunables:{PORTABLE_Q4_ROW_REUSE:{default:1},PORTABLE_LUT4_MVEC:{default:4},SMALL_M_SPLIT_K:{default:1},SMALL_M_N128_SPLIT_K:{default:1},SMALL_M_SPLIT_K_VALUE:{default:4},SMALL_M_DEEP_SPLIT_K:{default:16},SMALL_M_N128_EXPAND_SPLIT_K:{default:4},SMALL_M_N128_NARROW_SPLIT_K:{default:16},DENSE_GEMV_MROWS:{default:1},DENSE_GEMV_CLUSTERS:{default:4},DENSE_GEMV_REG_ROWS:{default:2},LUT2_PREFILL_STATIC_SCHEDULE:{default:1},LUT2_PREFILL_K_SPLITS:{default:8},LUT2_PREFILL_POLICY_ROWS:{default:0},LUT2_PREFILL_REGISTER_ROWS:{default:1},LUT2_PREFILL_WIDE_COLUMNS:{default:1},LUT2_PREFILL_BATCHED_RESULTS:{default:1}},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',decodeGemvSmallM:"args.M >= 1 and args.M <= 4",wideRowTile:"args.M >= 64 and ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 128",fusedQ4RowReuseEligible:'args.M >= 128 and tunables.PORTABLE_Q4_ROW_REUSE and tensorDtypes.aT == "float16" and tensorDtypes.yT == "float16" and fmt == "q4_0" and not args.iqRaw',fusedTileWorkgroupTarget:"256 if fusedQ4RowReuseEligible and (variableSubgroup16To32 or tunables.PORTABLE_Q4_ROW_REUSE == 2) else 128",fusedWideRows:"args.M >= 128 and ceil(args.M / 128) * ceil(args.outFeatures / 32) >= fusedTileWorkgroupTarget",fusedNStarved:"ceil(args.outFeatures / 32) <= 96",fusedNarrowTile:"args.M <= 8 or (args.M <= 16 and fusedNStarved)",fusedRowReuse:"(4 if ceil(args.M / 64) * ceil(args.outFeatures / 32) >= fusedTileWorkgroupTarget and device.limits.maxComputeWorkgroupStorageSize >= (64 + 32) * 32 * 2 else (2 if ceil(args.M / 32) * ceil(args.outFeatures / 32) >= fusedTileWorkgroupTarget and device.limits.maxComputeWorkgroupStorageSize >= (32 + 32) * 32 * 2 else 1)) if fusedQ4RowReuseEligible and not fusedWideRows and not fusedNarrowTile else 1",fusedMPerThread:"4 if fusedWideRows else fusedRowReuse",fusedBm:"128 if fusedWideRows else (16 * fusedRowReuse if fusedRowReuse > 1 else (8 if fusedNarrowTile else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)))",fusedBn:"32 if fusedWideRows else (16 if fusedNarrowTile else 32)",fusedNTile:"4 if fusedWideRows else (16 if fusedNarrowTile else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8))",fusedAccs:"8 if fusedWideRows else (1 if fusedNarrowTile else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4))",q1Present:'present.bitsT and present.scalesT and fmt == "q1_0"',q4Present:'present.bitsT and present.scalesT and fmt != "q1_0" and fmt != "q4_k_native_compact" and fmt != "q5_k" and lutc == 0 and (args.iqRaw if args.iqRaw else 0) == 0',iq3xxsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 3",iq3sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 4",iq3xxsBlocks:"(args.blockOffset if iq3xxsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq3sSupers:"ceilDiv(iq3xxsBlocks, 8)",iq3xxsContract:'iq3xxsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq3xxsBlocks * 3 and numel(shapes.scalesT) >= iq3xxsBlocks / 8',iq3sContract:'iq3sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq3sSupers * 27 and numel(shapes.scalesT) >= iq3sSupers',q4knCompactPresent:'present.bitsT and present.scalesT and fmt == "q4_k_native_compact" and not args.denseTailStart',q5kPresent:'present.bitsT and present.scalesT and fmt == "q5_k"',q1Blocks:"(args.blockOffset if q1Present else 0) + args.outFeatures * (args.inFeatures / 128)",q1BaseContract:'q1Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and (tensorDtypes.yT in ["float32", "float16"]) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 128 == 0 and args.outFeatures > 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= q1Blocks * 4 and numel(shapes.scalesT) >= q1Blocks',q4Blocks:"(args.blockOffset if q4Present else 0) + args.outFeatures * (args.inFeatures / 32)",splitCount:"args.kSplits if args.kSplits else 1",q4BaseContract:'q4Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and (quantBits == 4 or quantBits == 8) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 32 == 0 and args.outFeatures > 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.scalesT) >= (q4Blocks / 4 if q2k else q4Blocks / 8 if (q6k or q3k) else q4Blocks * (2 if (hasMin or iq or lut16) else 1)) and numel(shapes.bitsT) >= q4Blocks * (2.5 if q2k else 3.5 if q3k else 6.5 if q6k else 5 if q5 else quantBits)',decodeGemvContract:'q4BaseContract and decodeGemvSmallM and splitCount == 1 and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and not q5 and numel(shapes.bitsT) % 4 == 0 and (not (q2k or q3k or q6k) or (args.blockOffset % 8 == 0 and args.inFeatures % 256 == 0))',q4knCompactBlocks:"(args.blockOffset if q4knCompactPresent else 0) + args.outFeatures * (args.inFeatures / 32)",q4knCompactBaseContract:'q4knCompactPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 4 == 0 and numel(shapes.bitsT) >= q4knCompactBlocks * 4 and numel(shapes.scalesT) >= (q4knCompactBlocks / 8) * 5',q4knCompactDecodeGemvContract:'q4knCompactBaseContract and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and decodeGemvSmallM and splitCount == 1 and not present.realLenT',lutcPresent:"present.bitsT and present.scalesT and lutc == 1 and lut2 == 0 and (args.iqRaw if args.iqRaw else 0) == 0",lutcBlocks:"(args.blockOffset if lutcPresent else 0) + args.outFeatures * (args.inFeatures / 32)",lutcBaseContract:'lutcPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and args.lut is defined and args.lut != 0 and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 4 == 0 and numel(shapes.bitsT) >= lutcBlocks * 4 and numel(shapes.scalesT) >= (lutcBlocks / 8) * lutcWords',lutcDecodeGemvContract:"lutcBaseContract and decodeGemvSmallM and splitCount == 1",lut2Present:"present.bitsT and present.scalesT and lut2 == 1 and (args.iqRaw if args.iqRaw else 0) == 0",lut2Blocks:"(args.blockOffset if lut2Present else 0) + args.outFeatures * (args.inFeatures / 32)",lut2Int8:'args.activationQuant is defined and args.activationQuant == "int8"',lut2ShapeContract:'lut2Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and args.lut is defined and args.lut != 0 and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 2 == 0 and numel(shapes.bitsT) >= lut2Blocks * 2 and numel(shapes.scalesT) >= (lut2Blocks / 8) * lutcWords',lut2BaseContract:"lut2ShapeContract and not lut2Int8",lut2DoubleKTilePreferred:'lut2Present and args.lut in [8, 9] and variableSubgroup16To32 and ((tensorDtypes.aT == "float16" and tensorDtypes.yT == "float16" and args.M >= 64) or (tensorDtypes.aT == "float32" and fusedWideRows)) and device.limits.maxComputeWorkgroupStorageSize >= (fusedBm + fusedBn) * 2 * 8 * 8',lut2DecodeGemvContract:"lut2BaseContract and decodeGemvSmallM and splitCount == 1",q5kBlocks:"(args.blockOffset if q5kPresent else 0) + args.outFeatures * (args.inFeatures / 32)",q5kBaseContract:'q5kPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.scalesT) >= q5kBlocks / 4 and numel(shapes.bitsT) >= q5kBlocks * 5.5',q5kDecodeGemvContract:'q5kBaseContract and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and decodeGemvSmallM and splitCount == 1 and numel(shapes.bitsT) % 4 == 0',lut2PolicyRows:"max(args.M, tunables.LUT2_PREFILL_POLICY_ROWS) if (lut2Present and args.lut in [8, 9]) else args.M",sgmatTallSubtile:"1 if (args.M > 16 and ceil(args.outFeatures / 64) * ceil(lut2PolicyRows / 32) >= 128) else 0",iq2sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 5",iq2xsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 6",iq2sBlocks:"(args.blockOffset if iq2sRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2xsBlocks:"(args.blockOffset if iq2xsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2sSupers:"ceilDiv(iq2sBlocks, 8)",iq2xsSupers:"ceilDiv(iq2xsBlocks, 8)",iq2sContract:'iq2sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2sSupers * 20 and numel(shapes.scalesT) >= iq2sSupers',iq2xsContract:'iq2xsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2xsSupers * 18 and numel(shapes.scalesT) >= iq2xsSupers',iq1Raw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 1",iq1Blocks:"(args.blockOffset if iq1Raw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1Contract:'iq1Raw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq1Blocks * 2 and numel(shapes.scalesT) >= iq1Blocks / 8',iq2xxsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 2",iq2xxsBlocks:"(args.blockOffset if iq2xxsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2xxsContract:'iq2xxsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2xxsBlocks * 2 and numel(shapes.scalesT) >= iq2xxsBlocks / 8',denseOutStride:"args.outStride if args.outStride else args.outFeatures",denseGemvClusters:"tunables.DENSE_GEMV_CLUSTERS",denseGemvRegRows:"tunables.DENSE_GEMV_REG_ROWS",denseGemvCols:"denseGemvClusters * denseGemvRegRows",denseGemvMRowsContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float16", "float32"]) and (tensorDtypes.weightsT in ["float16", "float32"]) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.yT) and (tensorDtypes.yT in ["float16", "float32"]) and args.M >= 1 and args.M <= 16 and args.inFeatures > 0 and args.inFeatures % 128 == 0 and args.outFeatures > 0 and denseGemvCols > 0 and args.outFeatures % denseGemvCols == 0 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',denseWholeBlockDest:"(args.dstColStart if args.dstColStart else 0) == 0 and denseOutStride == args.outFeatures",denseSgmatSmallMContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and tensorDtypes.aT == "float16" and tensorDtypes.weightsT == "float16" and tensorDtypes.yT == "float16" and args.M >= 1 and args.M <= 16 and args.inFeatures > 0 and args.inFeatures % 32 == 0 and args.outFeatures > 0 and args.outFeatures % 64 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',iq1sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 7",iq1sBlocks:"(args.blockOffset if iq1sRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1sContract:'iq1sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= (iq1sBlocks / 8) * 12 and numel(shapes.scalesT) >= iq1sBlocks / 8',iq1mdRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 8",iq1mdBlocks:"(args.blockOffset if iq1mdRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1mdContract:'iq1mdRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= (iq1mdBlocks / 8) * 14 and numel(shapes.scalesT) >= iq1mdBlocks / 8',iqGridMin:"4096 if (iq1Raw or iq1sRaw or iq1mdRaw) else (2048 if iq2sRaw else (1024 if iq2xsRaw else (512 if (iq3sRaw or iq2xxsRaw) else 256)))",inFeatures:"args.inFeatures",outFeatures:"args.outFeatures",aVec4:'"vec4<f16>" if tensorDtypes.aT == "float16" else "vec4<f32>"',wScalar:'"f16"',clusters:1,regRows:1,mRows:"args.M",useSubgroups:"wave32SubgroupsUsable",sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',nTile:"fusedNTile",nPerThread:1,aScalar:"dtypes.A",splitKReduceWorkgroupSize:256,lutId:0,scaleScalar:"dtypes.S",f16Tiles:"1",kBlocks:"1",bn:"fusedBn",bm:"fusedBm",accs:"fusedAccs",tallSubtile:"sgmatTallSubtile",blocksPerRow:"args.inFeatures / 32",q1BitsElement:'"u32"',lutM:"16 if (args.M >= 128 or (args.M >= 16 and args.inFeatures <= 5120 and args.outFeatures > 1024)) else (1 if args.outFeatures <= 1024 else (4 if args.M >= 4 else args.M))",lutN:"256",assumeBiasNegHalfScale:!0,scaleOnly:!0,lutScalar:'"f16"',mSize:"args.M",scalesPk2:0,denseVectorContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and tensorDtypes.aT == "float32" and tensorDtypes.weightsT == "float32" and tensorDtypes.yT == "float32" and args.M > 16 and args.inFeatures > 0 and args.inFeatures % 4 == 0 and args.outFeatures > 0 and args.outFeatures < 1024 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',denseGemvParallelContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float16", "float32"]) and (tensorDtypes.weightsT in ["float16", "float32"]) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.yT) and (tensorDtypes.yT in ["float16", "float32"]) and args.M == 1 and args.inFeatures >= 1024 and args.inFeatures % 4 == 0 and args.outFeatures > 0 and args.outFeatures <= 128 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',lut2PrefillSplits:"min(max(1, tunables.LUT2_PREFILL_K_SPLITS), args.inFeatures / 32)",lut2PrefillTileCount:"ceil(args.outFeatures / (128 if args.M <= 16 else 64)) * ceil(lut2PolicyRows / (16 if args.M <= 16 else 32))",halfWordContract:'present.bitsT and (args.format == "bf16" or args.format == "f16") and not present.scalesT and not present.weightsT and not present.realLenT and not present.gridT and not args.denseTailStart and not args.blockOffset and denseWholeBlockDest and ranks.aT == 2 and ranks.bitsT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and args.M > 0 and args.inFeatures > 0 and args.outFeatures > 0 and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outFeatures and numel(shapes.bitsT) * 2 >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',halfWordSgmatContract:"halfWordContract and (args.weightOffset if args.weightOffset else 0) % 8 == 0 and wave32Effective and args.M >= 64 and args.inFeatures % 16 == 0 and args.outFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup",halfWordSgmatShortContract:"halfWordContract and (args.weightOffset if args.weightOffset else 0) % 8 == 0 and wave32Effective and args.M < 64 and args.M % 16 == 0 and args.inFeatures % 16 == 0 and args.outFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup and device.limits.maxComputeWorkgroupStorageSize >= 16396",iqRawExactContract:"iq1Contract or iq2xxsContract or iq3xxsContract or iq3sContract or iq2sContract or iq2xsContract or iq1sContract or iq1mdContract",packedExactSgmatContract:'args.exact == 1 and (iqRawExactContract if (args.iqRaw if args.iqRaw else 0) != 0 else ((fmt in ["q4_0", "q8_0", "q4_k", "q6_k", "q3_k", "q2_k", "q4_k_native_compact", "q5_k", "iq", "lut32", "lut16", "lut32_compact", "lut16_compact", "lut2_32", "lut2_16"]) and (q4knCompactBaseContract if fmt == "q4_k_native_compact" else (q5kBaseContract if fmt == "q5_k" else (lut2BaseContract if lut2 else (lutcBaseContract if lutc else q4BaseContract)))) and (not (q6k or q3k or q2k) or (args.inFeatures % 256 == 0 and args.blockOffset % 8 == 0)))) and not present.weightsT and not present.realLenT and not present.gridT and not args.denseTailStart and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and wave32Effective and args.outFeatures % 64 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup and device.limits.maxComputeWorkgroupStorageSize >= 16396',q8ExactSgmatContract:'q4BaseContract and fmt == "q8_0" and args.exact == 1 and denseWholeBlockDest and not present.weightsT and not present.realLenT and not present.partialsT and not present.gridT and not args.denseTailStart and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and wave32Effective and args.M >= 64 and args.outFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup and ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 28',hasEpilogue:'present.bT or present.rT or (args.activation is defined and args.activation != "none")',epilogueOk:'(not present.bT or (tensorDtypes.bT == "float32" and ranks.bT == 1 and dim(shapes.bT, 0) == args.outFeatures)) and (not present.rT or (tensorDtypes.rT == "float32" and ranks.rT == 2 and sameShape(shapes.rT, shapes.yT)))',epilogueSupported:'tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and dim(shapes.yT, 0) == args.M and dim(shapes.yT, 1) == args.outFeatures and not args.dstColStart and (not args.outStride or args.outStride == args.outFeatures) and not present.realLenT and not present.partialsT and not args.denseTailStart and splitCount == 1 and (halfWordContract or (q4Present and args.format == "q8_0" and args.exact == 1))',halfWordTiledContract:"halfWordContract and (args.weightOffset if args.weightOffset else 0) % 8 == 0 and args.inFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension",halfWordTiledTall:"ceil(args.M / 32) * ceil(args.outFeatures / 64) >= 160",denseTiledContract:'present.weightsT and not present.bitsT and not present.scalesT and not present.realLenT and not args.denseTailStart and denseWholeBlockDest and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and tensorDtypes.aT == "float32" and tensorDtypes.weightsT == "float32" and tensorDtypes.yT == "float32" and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 4 == 0 and args.outFeatures > 0 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outFeatures and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension',denseTiledRegGrid:"ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 128 and args.outFeatures >= 1024"},bindings:{a_a_t:{arg:"aT",name:"a",elementType:"$aVec4"},weights:{arg:"weightsT",elementType:"$wScalar"},y_y_t:{arg:"yT",name:"y",elementType:"$yScalar"},partials:{arg:"partialsT",elementType:"f32"},partials_f32:{arg:"partialsT",name:"partials",buffer:"read-only-storage",elementType:"f32"},params_m:{name:"params",struct:[{name:"m",type:"u32",value:"args.M"}]},a:{arg:"aT",elementType:"$aScalar"},q4_bits:{arg:"bitsT",elementType:"u32"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},params_main:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"m",type:"u32",value:"args.M"}]},a_main:{arg:"aT",name:"a",elementType:"vec4<f32>"},y_f32:{arg:"yT",name:"y",elementType:"f32"},q4_metadata:{arg:"scalesT",elementType:"u32"},q4_bits_main:{arg:"bitsT",name:"q4_bits",elementType:"vec4<u32>"},params__uniform:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"}]},bf16_x:{arg:"aT",name:"x",elementType:"f32"},bf16_w:{arg:"bitsT",name:"w",elementType:"vec4<u32>"},bf16_y:{arg:"yT",name:"y",elementType:"f32"},bf16_params_m:{name:"params",struct:[{name:"M",type:"u32",value:"args.M"}]},q8_sgmat_scales:{arg:"scalesT",name:"scales",elementType:"$scaleScalar"},q8_sgmat_params:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset if args.blockOffset else 0"}]},q8_sgmat_params_m:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset if args.blockOffset else 0"},{name:"M",type:"u32",value:"args.M"}]},bf16_weights:{arg:"bitsT",name:"weights",elementType:"u32"},bf16_params:{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset if args.weightOffset else 0"},{name:"m",type:"u32",value:"args.M"}]},bf16_x4:{arg:"aT",name:"x",elementType:"vec4<f32>"},dense_w4:{arg:"weightsT",name:"w",elementType:"vec4<f32>"},packed_scales:{arg:"scalesT",name:"scales",elementType:"$scaleScalar"},packed_params:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"M",type:"u32",value:"args.M"}]},packed_params_main:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"}]}},variants:[{id:"dense_vector",priority:104,when:["not present.realLenT","denseVectorContract","ceil(args.M * args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',workgroupSize:64},passes:[{id:"main",name:"LlamaPrefillMatmulDenseVector",shader:"prefill-matmul-dense-vector.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},"y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"ceil(args.M * args.outFeatures / workgroupSize)"}}]},{id:"dense_gemv_mrows",priority:104,when:["tunables.DENSE_GEMV_MROWS == 1","not present.realLenT","denseGemvMRowsContract","args.outFeatures / denseGemvCols <= device.limits.maxComputeWorkgroupsPerDimension","32 * denseGemvClusters <= device.limits.maxComputeInvocationsPerWorkgroup","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (denseGemvClusters) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',clusters:"denseGemvClusters",regRows:"denseGemvRegRows"},passes:[{id:"main",name:"LlamaPrefillMatmulDenseGemvMRows",shader:"prefill-matmul-dense-gemv-mrows.wgsl.jinja",bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},"y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"args.outFeatures / denseGemvCols"},subgroupCollectivesWidth:"portable"}]},{id:"f16_tiled",priority:20,when:["halfWordTiledContract",'args.format == "f16"',"not hasEpilogue"],derive:{bn:64,tn:4,bk:32,bWordOffset:"(args.weightOffset if args.weightOffset else 0) / 8",bm:"32 if halfWordTiledTall else 16",tm:"8 if halfWordTiledTall else 4"},passes:[{id:"main",name:"LlamaPrefillMatmulF16Tiled",shader:"prefill-matmul-half-word-tiled.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x4","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil(args.M / (32 if halfWordTiledTall else 16))"}}]},{id:"dense_tiled",priority:20,when:["denseTiledContract","not denseTiledRegGrid","not hasEpilogue"],derive:{f32Weights:!0,bn:64,tn:4,bk:32,bWordOffset:"(args.weightOffset if args.weightOffset else 0) / 4",bm:"32 if halfWordTiledTall else 16",tm:"8 if halfWordTiledTall else 4"},passes:[{id:"main",name:"LlamaPrefillMatmulDenseTiled",shader:"prefill-matmul-half-word-tiled.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x4","dense_w4","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil(args.M / (32 if halfWordTiledTall else 16))"}}]},{id:"dense_tiled_splitk",priority:106,when:["denseTiledContract","not denseTiledRegGrid","splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures % (32 * args.kSplits) == 0","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","not hasEpilogue"],derive:{f32Weights:!0,bn:64,tn:4,bk:32,bWordOffset:"(args.weightOffset if args.weightOffset else 0) / 4",yScalar:'"f32"',bm:16,tm:4,kSplits:"args.kSplits",kPerSplit:"args.inFeatures / args.kSplits"},passes:[{id:"partials",name:"LlamaPrefillMatmulDenseTiledSplitK",shader:"prefill-matmul-half-word-tiled.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x4","dense_w4","partials","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil(args.M / 16)",z:"args.kSplits"}},{id:"reduce",name:"LlamaPrefillMatmulDenseTiledSplitKReduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",derive:{outStride:"args.outFeatures",dstColStart:0},bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"f16_tiled_reg",priority:10,when:["halfWordContract",'args.format == "f16"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","not hasEpilogue"],derive:{yScalar:'"f32"',f16Weights:!0},passes:[{id:"main",name:"LlamaPrefillMatmulF16TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"packed_subgroup_matrix_exact",priority:99,when:["packedExactSgmatContract","args.M % 64 == 0","floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',packedBStaging:!0,scaleScalar:'"u32" if (fmt == "q4_k_native_compact" or lutc) else ("f16" if tensorDtypes.scalesT == "float16" else "f32")',packedMatrixNTail:!1,packedMatrixKTail:!1,regionStore:"not denseWholeBlockDest",tileM:64,matrixBlocks32:!0,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedFormat:"fmt",packedLut:"args.lut if args.lut else 0",packedIqRaw:"args.iqRaw if args.iqRaw else 0",regionOutStride:"args.outStride",regionDstCol:"args.dstColStart"},passes:[{id:"main",name:"LlamaPrefillMatmulPackedExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","packed_scales","bf16_y","packed_params_main"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"packed_subgroup_matrix_exact_wide",priority:100,when:["packedExactSgmatContract","args.M % 128 == 0","floor(args.M / 128) * ceil(args.outFeatures / 64) >= 256","floor(args.M / 128) <= device.limits.maxComputeWorkgroupsPerDimension","device.limits.maxComputeInvocationsPerWorkgroup >= 256","device.limits.maxComputeWorkgroupSizeX >= 256","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',packedBStaging:!0,scaleScalar:'"u32" if (fmt == "q4_k_native_compact" or lutc) else ("f16" if tensorDtypes.scalesT == "float16" else "f32")',packedMatrixNTail:!1,packedMatrixKTail:!1,regionStore:"not denseWholeBlockDest",tileM:128,matrixBlocks32:!0,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:8,tileGroupCols:"ceil(args.outFeatures / 64)",packedFormat:"fmt",packedLut:"args.lut if args.lut else 0",packedIqRaw:"args.iqRaw if args.iqRaw else 0",regionOutStride:"args.outStride",regionDstCol:"args.dstColStart"},passes:[{id:"main",name:"LlamaPrefillMatmulPackedExactSgmatWide",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","packed_scales","bf16_y","packed_params_main"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 128)"},subgroupCollectivesWidth:"portable"}]},{id:"packed_subgroup_matrix_exact_splitk",priority:102,when:["packedExactSgmatContract","args.M % 64 == 0","floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension","splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","args.inFeatures % (32 * args.kSplits) == 0","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',packedBStaging:!0,scaleScalar:'"u32" if (fmt == "q4_k_native_compact" or lutc) else ("f16" if tensorDtypes.scalesT == "float16" else "f32")',packedMatrixNTail:!1,packedMatrixKTail:!1,kSplits:"args.kSplits",packedFormat:"fmt",packedLut:"args.lut if args.lut else 0",packedIqRaw:"args.iqRaw if args.iqRaw else 0"},passes:[{id:"partials",name:"LlamaPrefillMatmulPackedExactSgmatSplitK",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,matrixBlocks32:!0,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,splitK:"args.kSplits",kPerSplit:"args.inFeatures / args.kSplits",outputBuffer:'"partials"',outScalar:'"f32"'},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","packed_scales","partials","packed_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)",z:"args.kSplits"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulPackedExactSgmatSplitKReduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",derive:{outStride:"args.outStride",dstColStart:"args.dstColStart"},bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"packed_subgroup_matrix_exact_short",priority:100,when:["packedExactSgmatContract","args.M < 64","args.M % 16 == 0","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',packedBStaging:!0,scaleScalar:'"u32" if (fmt == "q4_k_native_compact" or lutc) else ("f16" if tensorDtypes.scalesT == "float16" else "f32")',packedMatrixNTail:!1,packedMatrixKTail:!1,regionStore:"not denseWholeBlockDest",tileM:"args.M",tileN:64,tileK:16,directAStorage:!0,doubleBufferedB:!0,directStore:!0,packedFormat:"fmt",packedLut:"args.lut if args.lut else 0",packedIqRaw:"args.iqRaw if args.iqRaw else 0",regionOutStride:"args.outStride",regionDstCol:"args.dstColStart"},passes:[{id:"main",name:"LlamaPrefillMatmulPackedExactSgmatShort",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","packed_scales","bf16_y","packed_params_main"],dispatch:{x:"ceil(args.outFeatures / 64)",y:1},subgroupCollectivesWidth:"portable"}]},{id:"packed_subgroup_matrix_exact_short_splitk",priority:102,when:["packedExactSgmatContract","args.M < 64","args.M % 16 == 0","splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","args.inFeatures % (16 * args.kSplits) == 0","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',packedBStaging:!0,scaleScalar:'"u32" if (fmt == "q4_k_native_compact" or lutc) else ("f16" if tensorDtypes.scalesT == "float16" else "f32")',packedMatrixNTail:!1,packedMatrixKTail:!1,kSplits:"args.kSplits",tileM:"args.M",packedFormat:"fmt",packedLut:"args.lut if args.lut else 0",packedIqRaw:"args.iqRaw if args.iqRaw else 0"},passes:[{id:"partials",name:"LlamaPrefillMatmulPackedExactSgmatShortSplitK",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileN:64,tileK:16,directAStorage:!0,doubleBufferedB:!0,directStore:!0,splitK:"args.kSplits",kPerSplit:"args.inFeatures / args.kSplits",outputBuffer:'"partials"',outScalar:'"f32"'},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","packed_scales","partials","packed_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:1,z:"args.kSplits"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulPackedExactSgmatSplitKReduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",derive:{outStride:"args.outStride",dstColStart:"args.dstColStart"},bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"q8_subgroup_matrix_exact_m_tail",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"scalar",priority:0,when:["present.weightsT","denseWholeBlockDest","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'(tensorDtypes.weightsT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.yT)","args.M > 0","args.inFeatures > 0","args.outFeatures > 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * args.outFeatures","not args.denseTailStart","not hasEpilogue"],derive:{wScalar:"dtypes.W",yScalar:"dtypes.Y",workgroupSize:64},passes:[{id:"main",name:"LlamaPrefillMatmul",shader:"prefill-matmul.wgsl.jinja",bindings:["a","weights","y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"},{name:"m",type:"u32",value:"args.M"},{name:"in_features",type:"u32",value:"args.inFeatures"},{name:"out_features",type:"u32",value:"args.outFeatures"}]}],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"workgroupSize"}}]},{id:"tiled_exact",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}}]},{cases:[{id:"q8_tiled_exact_large_m",fmtWhen:'fmt == "q8_0"',minTiles:24,name:"Q8LargeM"},{id:"tiled_exact_large_m",fmtWhen:'fmt != "q8_0"',minTiles:48,name:"Q4ExactLargeM"}],variants:[{id:"{id}",priority:1,when:["q4BaseContract","{fmtWhen}",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= {minTiles}","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}}]}]},{id:"tiled_exact_splitk",priority:10,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue","1 == 1 or (32 + 32) * 2 * 8 * 16 <= device.limits.maxComputeWorkgroupStorageSize"],demoteWhen:['1 > 1 and (fmt != "q4_0" or args.M < 32 or args.M > 2 * 32 or ceil(args.M / 32) * ceil(args.outFeatures / 32) * splitCount < 4 * device.adapterInfo.subgroupMaxSize)'],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"2",bn:"32",bm:"32",nTile:"4",mPerThread:"1",accs:"8",kSplits:"args.kSplits",splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits)",yScalar:'"f32"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","params_main","partials"],profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},dispatch:{x:"min(ceil(args.outFeatures / bn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / bm), device.limits.maxComputeWorkgroupsPerDimension)",z:"args.kSplits"}},{id:"reduce",name:"LlamaPrefillMatmulQ4Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"tiled_exact_mrows2_splitk",priority:11,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue","variableSubgroup16To32","2 == 1 or (32 + 64) * 1 * 8 * 16 <= device.limits.maxComputeWorkgroupStorageSize"],demoteWhen:['2 > 1 and (fmt != "q4_0" or args.M < 32 or args.M > 2 * 32 or ceil(args.M / 32) * ceil(args.outFeatures / 64) * splitCount < 4 * device.adapterInfo.subgroupMaxSize)'],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"32",nTile:"8",mPerThread:"2",accs:"8",kSplits:"args.kSplits",splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits)",yScalar:'"f32"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","params_main","partials"],profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},dispatch:{x:"min(ceil(args.outFeatures / bn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / bm), device.limits.maxComputeWorkgroupsPerDimension)",z:"args.kSplits"}},{id:"reduce",name:"LlamaPrefillMatmulQ4Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"q4kn_compact_tiled_exact_splitk",priority:10,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","args.inFeatures % (64 * args.kSplits) == 0","q4knCompactBaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:0,lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"2",bn:"32",bm:"32",nTile:"4",accs:"8",kSplits:"args.kSplits",splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits)",yScalar:'"f32"',quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeCompactSplitK",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_metadata","params_main","partials"],profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},dispatch:{x:"min(ceil(args.outFeatures / 32), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 32), device.limits.maxComputeWorkgroupsPerDimension)",z:"args.kSplits"},derive:{q5k:!1,q4knCompact:!0}},{id:"reduce",name:"LlamaPrefillMatmulQ4KNativeCompactSplitKReduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"q4kn_compact_tiled_exact",priority:0,when:["q4knCompactBaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:0,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeCompact",shader:"prefill-matmul-q4.wgsl.jinja",derive:{q5k:!1,q4knCompact:!0},bindings:["a_main","q4_bits","q4_metadata","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4kn_compact_tiled_exact_large_m",priority:1,when:["q4knCompactBaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 48","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:0,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeCompactLargeM",shader:"prefill-matmul-q4.wgsl.jinja",derive:{q5k:!1,q4knCompact:!0},bindings:["a_main","q4_bits","q4_metadata","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4_decode_gemv",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"}]},{cases:[{suffix:"mr2",rows:2,divisor:8,priority:110},{suffix:"mr4",rows:4,divisor:16,priority:109}],variants:[{id:"q4_decode_gemv_{suffix}",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * 4 <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * 4 <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:4,regRows:"{rows}",outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"}]}]}]},assets:[["prefill-matmul-dense-vector.wgsl.jinja","{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){if(gid.x>={{ mSize * outFeatures }}u){return;}let row=gid.x/{{ outFeatures }}u;let col=gid.x% {{ outFeatures }}u;let ab=row*{{ (inFeatures / 4) | int }}u;let wb=params.weight_offset/4u+col*{{ (inFeatures / 4) | int }}u;var acc=0.0;for(var k=0u;k<{{ (inFeatures / 4) | int }}u;k++){let av=vec4<f32>(a[ab+k]);let wv=vec4<f32>(weights[wb+k]);acc=acc+av.x*wv.x;acc=acc+av.y*wv.y;acc=acc+av.z*wv.z;acc=acc+av.w*wv.w;}y[row*{{ outStride }}u+{{ dstColStart }}u+col]={{ yScalar }}(acc);}"],["prefill-matmul-dense-gemv-mrows.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const K_V4:u32={{ inFeatures }}u/4u;const DST_COL:u32={{ dstColStart }}u;const OUT_STRIDE:u32={{ outStride }}u;const CLUSTERS:u32={{ clusters }}u;const REG_ROWS:u32={{ regRows }}u;
{% include "ops/_shared/subgroup-runtime-fold.3a89b81fa3.wgsl.jinja" %}{% set sgOwned = useSubgroups and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.87076f64db.wgsl.jinja" %}{% set sgDim0 = "(wid.x * CLUSTERS + cluster) * REG_ROWS" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 32 * clusters %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}var<workgroup>partials4:array<f32,{{ 32 * clusters }}>;
{% endif %}@compute@workgroup_size({{ 32 * clusters }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% if sgOwned %}{{ sg_cluster_open(sgPath, "CLUSTERS", clusters * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*CLUSTERS+cluster)*REG_ROWS;let wbase=params.weight_offset/4u+dim0*K_V4;
{% for m in range(mRows) %}{% for r in range(regRows) %}var acc{{ m }}_{{ r }}=0.0;
{% endfor %}{% endfor %}for(var v=lane;v<K_V4;v=v+{{ "activeCount" if sgPath == "general" else "32u" }}){
{% for r in range(regRows) %}let w{{ r }}=vec4<f32>(weights[wbase+{{ r }}u*K_V4+v]);
{% endfor %}{% for m in range(mRows) %}{let x=vec4<f32>(a[{{ m }}u*K_V4+v]);
{% for r in range(regRows) %}acc{{ m }}_{{ r }}=acc{{ m }}_{{ r }}+dot(w{{ r }},x);
{% endfor %}}
{% endfor %}}
{% if not useSubgroups and not sgRuntime %}{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" ~ m ~ "_" %}{% set totalPrefix = "total" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.e8bf64d477.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if sgPath == "general" %}{{ sg_cluster_store("CLUSTERS") }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let total{{ m }}_{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if sgPath == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% for r in range(regRows) %}y[{{ m }}u*OUT_STRIDE+DST_COL+dim0+{{ r }}u]={{ yScalar }}(total{{ m }}_{{ r }});
{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}{% endfor %}}`],["prefill-matmul-half-word-tiled.wgsl.jinja",`{% set f32w = f32Weights | default(false) %}{% set lanesN = 16 %}{% set lanesM = (bm / tm) | int %}{% set lanes = lanesN * lanesM %}{% set aLoads = (bm * bk / 4 / lanes) | int %}{% set bLoads = (bn * bk / (4 if f32w else 8) / lanes) | int %}{% set kSplit = (kSplits | default(1)) > 1 %}{% set kTail = not kSplit and inFeatures % bk != 0 %}{% if f32w %}{% else %}{% include "ops/_shared/f16-bits-to-f32.09629f6275.wgsl.jinja" %}fn widen_f16_x4(words:vec2<u32>)->vec4<f32>{return vec4<f32>(f16_bits_to_f32(words.x&0xffffu),f16_bits_to_f32(words.x>>16u),f16_bits_to_f32(words.y&0xffffu),f16_bits_to_f32(words.y>>16u));}
{% endif %}{{ env.wgsl.resourceDeclarations }}
{% if f32w %}{% else %}{% endif %}{% if f32w %}{% else %}{% endif %}{% if kSplit %}{% endif %}const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const K_V4:u32=K/4u;
{% if not f32w %}const K_V8:u32=K/8u;
{% endif %}const W_BASE:u32={{ bWordOffset }}u;const BM:u32={{ bm }}u;const BN:u32={{ bn }}u;const BK:u32={{ bk }}u;const BK_V4:u32=BK/4u;
{% if not f32w %}const BK_V8:u32=BK/8u;
{% endif %}const LANES_N:u32={{ lanesN }}u;
{% if kSplit %}const KPS:u32={{ kPerSplit }}u;
{% endif %}var<workgroup>tileA:array<vec4<f32>,BK_V4*BM>;var<workgroup>tileB:array<vec4<f32>,BK_V4*BN>;@compute@workgroup_size({{ lanesN }},{{ lanesM }},1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let li=lid.y*LANES_N+lid.x;let mBase=wg.y*BM;let nBase=wg.x*BN;let aRow=lid.y*{{ tm }}u;
{% for i in range(tm) %}{% for j in range(tn) %}var acc{{ i }}_{{ j }}:f32=0.0;
{% endfor %}{% endfor %}{% if kSplit %}let kEnd=(wg.z+1u)*KPS;for(var kBase:u32=wg.z*KPS;kBase<kEnd;kBase=kBase+BK){
{% else %}for(var kBase:u32=0u;kBase<K;kBase=kBase+BK){
{% endif %}{% for s in range(aLoads) %}{let idx=li+{{ s * lanes }}u;let m=idx/BK_V4;let kv=idx%BK_V4;var v=vec4<f32>(0.0);
      if (mBase + m < params.M{% if kTail %} && kBase + kv * 4u < K{% endif %}) {
v=x[(mBase+m)*K_V4+kBase/4u+kv];}tileA[kv*BM+m]=v;}
{% endfor %}{% for s in range(bLoads) %}{% if f32w %}{let idx=li+{{ s * lanes }}u;let n=idx/BK_V4;let kv=idx%BK_V4;var v=vec4<f32>(0.0);
      if (nBase + n < N{% if kTail %} && kBase + kv * 4u < K{% endif %}) {
v=w[W_BASE+(nBase+n)*K_V4+kBase/4u+kv];}tileB[kv*BN+n]=v;}
{% else %}{let idx=li+{{ s * lanes }}u;let n=idx/BK_V8;let k8=idx%BK_V8;var words=vec4<u32>(0u);
      if (nBase + n < N{% if kTail %} && kBase + k8 * 8u < K{% endif %}) {
words=w[W_BASE+(nBase+n)*K_V8+kBase/8u+k8];}tileB[(2u*k8)*BN+n]=widen_f16_x4(words.xy);tileB[(2u*k8+1u)*BN+n]=widen_f16_x4(words.zw);}
{% endif %}{% endfor %}workgroupBarrier();for(var kv:u32=0u;kv<BK_V4;kv=kv+1u){
{% for i in range(tm) %}let a{{ i }}=tileA[kv*BM+aRow+{{ i }}u];
{% endfor %}{% for j in range(tn) %}let b{{ j }}=tileB[kv*BN+lid.x+{{ j }}u*LANES_N];
{% endfor %}{% for i in range(tm) %}{% for j in range(tn) %}acc{{ i }}_{{ j }}=fma(a{{ i }}.w,b{{ j }}.w,fma(a{{ i }}.z,b{{ j }}.z,fma(a{{ i }}.y,b{{ j }}.y,fma(a{{ i }}.x,b{{ j }}.x,acc{{ i }}_{{ j }}))));
{% endfor %}{% endfor %}}workgroupBarrier();}
{% set out = "partials[(wg.z * params.M + m) * N + n]" if kSplit else "y[m * N + n]" %}{% for i in range(tm) %}{let m=mBase+aRow+{{ i }}u;if(m<params.M){
{% for j in range(tn) %}{let n=nBase+lid.x+{{ j }}u*LANES_N;if(n<N){ {{ out }}=acc{{ i }}_{{ j }};} }
{% endfor %}} }
{% endfor %}}`],["prefill-matmul-q4-reduce.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const N:u32={{ outFeatures }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const K_SPLITS:u32={{ kSplits }}u;@compute@workgroup_size({{ splitKReduceWorkgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(splitKReduceWorkgroupSize, "flat", "params.m * N") }} let m=flat/N;let n=flat%N;var acc=0.0;for(var z=0u;z<K_SPLITS;z=z+1u){acc=acc+partials[(z*params.m+m)*N+n];}y[m*OUT_STRIDE+DST_COL+n]={{ yScalar }}(acc);}`],["prefill-matmul-tiled-reg.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const BK:u32=16u;const BM:u32=64u;const BN:u32=64u;const TM:u32=4u;const TN:u32=4u;const K_VECS:u32=BK/4u;var<workgroup>tileA:array<array<vec4<f32>,K_VECS>,BM>;var<workgroup>tileB:array<array<vec4<f32>,K_VECS>,BN>;
{% include "ops/_shared/f16-bits-to-f32.09629f6275.wgsl.jinja" %}fn f16Weight(index:u32)->f32{let word=weights[index>>1u];return f16_bits_to_f32(select(word,word>>16u,(index&1u)==1u));}@compute@workgroup_size(16,16,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let li=lid.y*16u+lid.x;let woff=params.weight_offset;let n_tiles=(N+BN-1u)/BN;let m_tiles=(params.m+BM-1u)/BM;for(var nt:u32=wg.x;nt<n_tiles;nt=nt+nwg.x){let nBase=nt*BN;for(var mt:u32=wg.y;mt<m_tiles;mt=mt+nwg.y){let mBase=mt*BM;var acc:array<f32,TM*TN>;for(var i:u32=0u;i<TM*TN;i=i+1u){acc[i]=0.0;}let numTiles=(K+BK-1u)/BK;for(var kt:u32=0u;kt<numTiles;kt=kt+1u){let kBase=kt*BK;for(var idx:u32=li;idx<BM*K_VECS;idx=idx+256u){let ar=idx/K_VECS;let ac4=idx%K_VECS;let am=mBase+ar;let ak=kBase+ac4*4u;var aWord=vec4<f32>(0.0);if(am<params.m){
{% for component in range(4) %}if(ak+{{ component }}u<K){aWord[{{ component }}u]=f32(a[am*K+ak+{{ component }}u]);}
{% endfor %}}tileA[ar][ac4]=aWord;}for(var idx:u32=li;idx<BN*K_VECS;idx=idx+256u){let bc=idx/K_VECS;let br4=idx%K_VECS;let bn=nBase+bc;let bk=kBase+br4*4u;var bWord=vec4<f32>(0.0);if(bn<N){
{% for component in range(4) %}if(bk+{{ component }}u<K){bWord[{{ component }}u]=f16Weight(woff+bn*K+bk+{{ component }}u);}
{% endfor %}}tileB[bc][br4]=bWord;}workgroupBarrier();
{% set regT = "f32" %}{% filter indent(8, true) %}{% include "ops/_shared/matmul-reg-microtile-fma.1983670e1d.wgsl.jinja" %}{% endfilter %}workgroupBarrier();}let m0=mBase+lid.y*TM;let n0=nBase+lid.x*TN;for(var ti:u32=0u;ti<TM;ti=ti+1u){let m=m0+ti;if(m>=params.m){continue;}for(var tj:u32=0u;tj<TN;tj=tj+1u){let n=n0+tj;if(n<N){y[m*N+n]={{ yScalar }}(acc[ti*TN+tj]);} } } } } }`],["ops/_shared/gemm-subgroup-matrix.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.de53490305.wgsl.jinja" %}{% set usePackedBStaging = packedBStaging | default(false) %}{% set packedFormat = packedFormat | default("") %}{% set packedIqRaw = packedIqRaw | default(0) %}{% set packed4x8CompatUnpackI8 = usePackedBStaging and (packedFormat in ["q8_0", "q6_k", "q3_k", "iq"] or packedIqRaw in [1, 7, 8]) %}{% set packed4x8CompatUnpackU8 = usePackedBStaging and (packedIqRaw in [2, 5, 6] or (not packedIqRaw and packedFormat not in ["q8_0", "iq", "lut32", "lut16", "lut32_compact", "lut16_compact", "lut2_32", "lut2_16"])) %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if usePackedBStaging %}{% if packedIqRaw %}{% set iq1m = packedIqRaw in [1, 7, 8] %}{% set iq2xxs = packedIqRaw == 2 %}{% set iq3xxs = packedIqRaw == 3 %}{% set iq3s = packedIqRaw == 4 %}{% set iq2s = packedIqRaw == 5 %}{% set iq2xs = packedIqRaw == 6 %}{% include "ops/_shared/iq-grid.b22b84ae14.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.6dec8810cb.wgsl.jinja" %}{% endif %}{% set lut2 = packedFormat in ["lut2_32", "lut2_16"] %}{% set lutc = lut2 or packedFormat in ["lut32_compact", "lut16_compact"] %}{% set lutc16 = packedFormat in ["lut16_compact", "lut2_16"] %}{% set lutId = packedLut | default(0) %}{% if lutId %}{% set emitQuantValueHelpers = not lut2 %}{% include "ops/_shared/lut2-block-scale.5bf39fd59c.wgsl.jinja" %}{% include "ops/_shared/q4nl-dequant-helpers.dd92a6a387.wgsl.jinja" %}{% endif %}{% endif %}{% set operandScalar = "f32" %}{% set accScalar = "f32" %}{% set SPLIT_K = splitK if splitK is defined else 1 %}{% set kPerSplit = kPerSplit | default(0) %}{% set OUT = outputBuffer if outputBuffer is defined else "y" %}{% set OUT_SCALAR = outScalar if outScalar is defined else yScalar %}{% set tileMValue = tileM if tileM is defined else 32 %}{% set tileNValue = 64 %}{% set tileKValue = tileK if tileK is defined else 32 %}{% set gemmEpi = "none" %}{% set epiOpen = "" %}{% set epiClose = "" %}{% set packedNTail = packedMatrixNTail is defined and packedMatrixNTail %}{% set packedKTail = false %}{% set useDirectStore = directStore is defined and directStore and gemmEpi == "none" and not hasBias and not (hasResidual | default(false)) %}{% set useDirectAStorage = directAStorage is defined and directAStorage %}{% set useDoubleBufferedB = doubleBufferedB is defined and doubleBufferedB %}{% set useBalancedAccumulation = false %}{% set columnTileBaseValue = 0 %}{% set rowTileBaseValue = 0 %}{% set rowTileBaseRuntimeValue = rowTileBaseRuntime | default("") %}{% set runtimeRowTileBase = rowTileBaseRuntimeValue != "" %}{% set tileGroup = tileGroupRows if (tileGroupRows is defined and SPLIT_K == 1) else 1 %}{% set tileGroupColsValue = tileGroupCols | default(0) %}{% if matrixBlocks32 is defined and matrixBlocks32 %}{% set subgroupRows = (tileMValue / 32)|int %}{% set subgroupCols = 2 %}{% set subgroupCount = subgroupRows * subgroupCols %}{% set subtileRows = 32 %}{% set subtileCols = 32 %}{% elif tileMValue == 16 %}{% set subgroupCount = 4 %}{% set subgroupRows = 1 %}{% set subgroupCols = 4 %}{% set subtileCols = 16 %}{% set subtileRows = 16 %}{% elif tileMValue == 48 %}{% set subgroupCount = 4 %}{% set subgroupRows = 2 %}{% set subgroupCols = 2 %}{% set subtileCols = 32 %}{% set subtileRows = 24 %}{% elif tileMValue > 32 %}{% set subgroupCount = 4 %}{% set subgroupRows = 4 %}{% set subgroupCols = 1 %}{% set subtileCols = 64 %}{% set subtileRows = (tileMValue / 4)|int %}{% else %}{% set subgroupCount = 4 %}{% set subgroupRows = 2 %}{% set subgroupCols = 2 %}{% set subtileCols = 32 %}{% set subtileRows = 16 %}{% endif %}{% set aTilesPerSubgroup = (subtileRows / 8)|int %}{% set bTilesPerSubgroup = (subtileCols / 8)|int %}{% set workgroupThreads = 32 * subgroupCount %}{% set scratchBanks = subgroupCount * aTilesPerSubgroup * bTilesPerSubgroup %}{% set reuseB = false %}{% macro matrixAccumulators(matrixSoftware) %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}var matC{{ m }}{{ n }}:{{ matrixType(matrixSoftware, "result", accScalar) }};
{% endfor %}{% endfor %}{% endmacro %}{% macro storeMatrixAccumulators(matrixSoftware) %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[bank + " ~ (m * bTilesPerSubgroup + n) ~ "u]", "0u", "matC" ~ (m) ~ (n), "8u") }}
{% endfor %}{% endfor %}{% endmacro %}const IN_F:u32={{ inFeatures }}u;
{% set useRegionStore = (regionStore | default(false)) and SPLIT_K == 1 %}{% set OUT_ROW = "OUT_STRIDE" if useRegionStore else "OUT_F" %}{% if not useRegionStore or packedNTail %}const OUT_F:u32={{ outFeatures }}u;
{% endif %}{% if useRegionStore %}const OUT_STRIDE:u32={{ regionOutStride | default(0) }}u;const DST_COL:u32={{ regionDstCol | default(0) }}u;
{% endif %}{% set kLoopEndValue = "IN_F" %}const TILE_COLS:u32={{ tileNValue }}u;const TILE_ROWS:u32={{ tileMValue }}u;const TILE_K:u32={{ tileKValue }}u;const SUB_COLS:u32={{ subtileCols }}u;const SUB_ROWS:u32={{ subtileRows }}u;const COLUMN_TILE_BASE:u32={{ columnTileBaseValue }}u;
{% if not runtimeRowTileBase %}const ROW_TILE_BASE:u32={{ rowTileBaseValue }}u;
{% endif %}{% if SPLIT_K > 1 %}const K_PER_SPLIT:u32={{ kPerSplit }}u;
{% endif %}{% if not useDirectAStorage %}var<workgroup>tile_A:array<{{ operandScalar }},{{ tileMValue }}*{{ tileKValue }}>;
{% endif %}var<workgroup>tile_B:array<{{ operandScalar }},{{ (2 if useDoubleBufferedB else 1) * tileNValue }}*{{ tileKValue }}>;
{% if not useDirectStore and not reuseB %}var<workgroup>scratch:array<array<{{ accScalar }},64>,{{ scratchBanks }}>;
{% endif %}{% if not useDirectAStorage %}fn loadSHMA(tile_base:u32,k_idx:u32,row:u32,c_idx:u32){let col:u32=c_idx*8u;for(var row_offset:u32=0u;row_offset<TILE_ROWS;row_offset+={{ (workgroupThreads * 8 / tileKValue)|int }}u){let r:u32=row+row_offset;let a_global:u32=min(tile_base+r,params.M-1u);for(var col_offset:u32=0u;col_offset<8u;col_offset++){let k:u32=k_idx+col+col_offset;tile_A[r*TILE_K+col+col_offset]=x[a_global*IN_F+k];} } }
{% endif %}fn loadSHMB(tile_base:u32,k_idx:u32,row:u32,c_idx:u32,bank_offset:u32){
{% if q8BStaging is defined and q8BStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){
{% if packedNTail or packedKTail %}if(tile_base+b_row<OUT_F&&k_idx+col<IN_F){
{% endif %}let block:u32=params.block_offset+(tile_base+b_row)*(IN_F/32u)+k_idx/32u;let scale:f32=f32(scales[block]);let v=w[block*2u+c_idx];let dst:u32=bank_offset+b_row*TILE_K+col;for(var i:u32=0u;i<4u;i++){let word=bitcast<i32>(v[i]);tile_B[dst+i*4u]=f32(extractBits(word,0u,8u))*scale;tile_B[dst+i*4u+1u]=f32(extractBits(word,8u,8u))*scale;tile_B[dst+i*4u+2u]=f32(extractBits(word,16u,8u))*scale;tile_B[dst+i*4u+3u]=f32(extractBits(word,24u,8u))*scale;}
{% if packedNTail or packedKTail %}}else{for(var i=0u;i<16u;i++){tile_B[bank_offset+b_row*TILE_K+col+i]=0.0;} }
{% endif %}} }
{% elif usePackedBStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){let k:u32=k_idx+col;let blk:u32=params.block_offset+(tile_base+b_row)*(IN_F/32u)+k/32u;let blk_half:u32=(k/16u)&1u;
{% include "ops/_shared/packed-half-block-f32.baa86c90c3.wgsl.jinja" %}      let dst: u32 = bank_offset + b_row * TILE_K + col;
{% for i in range(4) %}tile_B[dst+{{ 4 * i }}u]=hv{{ i }}.x;tile_B[dst+{{ 4 * i + 1 }}u]=hv{{ i }}.y;tile_B[dst+{{ 4 * i + 2 }}u]=hv{{ i }}.z;tile_B[dst+{{ 4 * i + 3 }}u]=hv{{ i }}.w;
{% endfor %}} }
{% else %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){let w_global:u32=tile_base+b_row;for(var i:u32=0u;i<16u;i++){let k:u32=k_idx+col+i;tile_B[bank_offset+b_row*TILE_K+col+i]=w[w_global*IN_F+k];} } }
{% endif %}}
{% if not useDirectStore %}{% set needsColBase = hasBias or (tailSafe is defined and tailSafe) or packedNTail %}{% set BIAS_1 = "" %}{% set BIAS_2 = "" %}{% set ACC_1 = "scratch[src_slot][row * 8u + col]" %}{% set ACC_2 = "scratch[src_slot][row * 8u + col2]" %}fn storeOutput(offset: u32{% if needsColBase %}, col_base: u32{% endif %}, row: u32, col: u32, src_slot: u32, row_limit: i32) {
if(row_limit>0&&row<u32(row_limit)){let col2:u32=col+1u;
{% if (tailSafe is defined and tailSafe) or packedNTail %}if(col_base+col<OUT_F){ {{ OUT }}[offset+row*OUT_F+col]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_1 }}{{ BIAS_1 }}){{ " + residual[offset + row * OUT_F + col]" if hasResidual | default(false) else "" }}{{ epiClose }});}if(col_base+col2<OUT_F){ {{ OUT }}[offset+row*OUT_F+col2]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_2 }}{{ BIAS_2 }}){{ " + residual[offset + row * OUT_F + col2]" if hasResidual | default(false) else "" }}{{ epiClose }});}
{% else %}{{ OUT }}[offset+row*OUT_F+col]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_1 }}{{ BIAS_1 }}){{ " + residual[offset + row * OUT_F + col]" if hasResidual | default(false) else "" }}{{ epiClose }});{{ OUT }}[offset+row*OUT_F+col2]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_2 }}{{ BIAS_2 }}){{ " + residual[offset + row * OUT_F + col2]" if hasResidual | default(false) else "" }}{{ epiClose }});
{% endif %}} }
{% endif %}{% include "ops/_shared/subgroup-matrix-topology.5c216ddcce.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.cedda9952e.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}{% if runtimeRowTileBase %}let ROW_TILE_BASE:u32={{ rowTileBaseRuntimeValue }};
{% endif %}{% if SPLIT_K > 1 %}let a_global_base:u32=(ROW_TILE_BASE+workgroup_id.y)*TILE_ROWS;let split_id:u32=workgroup_id.z;
{% elif tileGroup > 1 %}let tile_linear:u32=workgroup_id.x+workgroup_id.y*{{ tileGroupColsValue }}u;let tile_group:u32=tile_linear/{{ tileGroup * tileGroupColsValue }}u;let group_first_row:u32=tile_group*{{ tileGroup }}u;let group_rows:u32=min(nwg.y-group_first_row,{{ tileGroup }}u);let in_group:u32=tile_linear-tile_group*{{ tileGroup * tileGroupColsValue }}u;let a_global_base:u32=(ROW_TILE_BASE+group_first_row+in_group%group_rows)*TILE_ROWS;
{% else %}let a_global_base:u32=(ROW_TILE_BASE+workgroup_id.y+workgroup_id.z*nwg.y)*TILE_ROWS;
{% endif %}{% if tileGroup > 1 %}let w_global_base:u32=(COLUMN_TILE_BASE+in_group/group_rows)*TILE_COLS;
{% else %}let w_global_base:u32=(COLUMN_TILE_BASE+workgroup_id.x)*TILE_COLS;
{% endif %}let subtile_id:u32=matrixRole;
{% if subgroupCols == 1 %}let subtile_idx:u32=0u;let subtile_idy:u32=subtile_id;
{% else %}let subtile_idx:u32=subtile_id/{{ subgroupRows }}u;let subtile_idy:u32=subtile_id% {{ subgroupRows }}u;
{% endif %}let base_A:u32=subtile_idy*SUB_ROWS;let base_B:u32=subtile_idx*SUB_COLS;{{ matrixAccumulators(matrixSoftware) }}
{% if useDoubleBufferedB %}var b_bank:u32=0u;
{% if SPLIT_K > 1 %}let k_begin:u32=split_id*K_PER_SPLIT;let k_end:u32=min(k_begin+K_PER_SPLIT,{{ kLoopEndValue }});
{% endif %}{% set kFirst = "k_begin" if SPLIT_K > 1 else "0u" %}{% set kLast = "k_end" if SPLIT_K > 1 else kLoopEndValue %}loadSHMB(w_global_base,{{ kFirst }},local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,0u);workgroupBarrier();for(var kidx:u32={{ kFirst }};kidx<{{ kLast }};kidx+=TILE_K){let b_bank_offset=b_bank*TILE_COLS*TILE_K;for(var step:u32=0u;step<{{ "min(TILE_K, IN_F - kidx)" if packedKTail else "TILE_K" }};step+={{ 32 if useBalancedAccumulation else 8 }}u){
{% include "ops/_shared/subgroup-matrix-dynamic-step.501544938c.wgsl.jinja" %}    }
let next_kidx=kidx+TILE_K;if(next_kidx<{{ kLast }}){let next_bank=1u-b_bank;loadSHMB(w_global_base,next_kidx,local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,next_bank*TILE_COLS*TILE_K);workgroupBarrier();b_bank=next_bank;} }
{% else %}{% if SPLIT_K > 1 %}let k_begin:u32=split_id*K_PER_SPLIT;let k_end:u32=min(k_begin+K_PER_SPLIT,{{ kLoopEndValue }});for(var kidx:u32=k_begin;kidx<k_end;kidx+=TILE_K){
{% else %}for(var kidx:u32=0u;kidx<{{ kLoopEndValue }};kidx+=TILE_K){
{% endif %}{% if not useDirectAStorage %}loadSHMA(a_global_base,kidx,local_idx/{{ (tileKValue / 8)|int }}u,local_idx% {{ (tileKValue / 8)|int }}u);
{% endif %}loadSHMB(w_global_base,kidx,local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,0u);workgroupBarrier();for(var step:u32=0u;step<{{ "min(TILE_K, IN_F - kidx)" if packedKTail else "TILE_K" }};step+=8u){
{% include "ops/_shared/subgroup-matrix-dynamic-step.501544938c.wgsl.jinja" %}    }
workgroupBarrier();}
{% endif %}  let matrix_c_offset: u32 = {% if SPLIT_K > 1 %}split_id * params.M * OUT_F + {% endif %}(a_global_base + base_A) * {{ OUT_ROW }} + {{ "DST_COL + " if useRegionStore else "" }}w_global_base + base_B;
{% if useDirectStore %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{% if packedNTail %}if(w_global_base+base_B+{{ n * 8 }}u<OUT_F){
{% endif %}{{ matrixStore(matrixSoftware, "row_major", "&" ~ (OUT), "matrix_c_offset + " ~ (m * 8) ~ "u * " ~ OUT_ROW ~ " + " ~ (n * 8) ~ "u", "matC" ~ (m) ~ (n), OUT_ROW) }}
{% if packedNTail %}}
{% endif %}{% endfor %}{% endfor %}{% else %}let row:u32=sg_id/4u;let col:u32=(sg_id%4u)*2u;let bank:u32=subtile_id*{{ aTilesPerSubgroup * bTilesPerSubgroup }}u;{{ storeMatrixAccumulators(matrixSoftware) }} workgroupBarrier();
{% for m in range(aTilesPerSubgroup) %}let row_limit_{{ m }}:i32=i32(params.M)-i32(a_global_base+base_A+{{ m * 8 }}u);
{% for n in range(bTilesPerSubgroup) %}storeOutput(
    matrix_c_offset + {{ m * 8 }}u * OUT_F + {{ n * 8 }}u{% if needsColBase %},
w_global_base+base_B+{{ n * 8 }}u,
{% else %},
{% endif %}row,col,bank+{{ m * bTilesPerSubgroup + n }}u,row_limit_{{ m }}{{ ", pairTotals[" ~ (m * bTilesPerSubgroup + n) ~ "]" if useBalancedAccumulation else "" }});
{% endfor %}{% endfor %}{% endif %}{% endmacro %}@compute@workgroup_size({{ workgroupThreads }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)workgroup_id:vec3<u32>,
{% if SPLIT_K == 1 %}@builtin(num_workgroups)nwg:vec3<u32>,
{% endif %}  @builtin(local_invocation_index) local_idx: u32{% if not useDirectStore %},
@builtin(subgroup_invocation_id)sg_id:u32,
{% else %},
{% endif %}@builtin(subgroup_size)sg_size:u32){let matrixRole=claimMatrixSubgroupRole(sg_size,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,{{ workgroupThreads }}u/32u);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;
{% if not useDirectStore %}let sg_id=local_idx%32u;
{% endif %}{{ matrixKernelBody(true) }} } }`],["prefill-matmul.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.812cc3cd16.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize, "g", "") }} let total=params.m*params.out_features;if(g>=total){return;}let row=g/params.out_features;let col=g%params.out_features;let a_base=row*params.in_features;let w_base=params.weight_offset+col*params.in_features;var acc=0.0;for(var k=0u;k<params.in_features;k=k+1u){acc=acc+f32(a[a_base+k])*f32(weights[w_base+k]);}y[g]={{ yScalar }}(acc);}`],["prefill-matmul-q4.wgsl.jinja",`{% set mPerThread = mPerThread if mPerThread is defined else 1 %}{% set rowVar = "m_row" if mPerThread > 1 else "m_local" %}{% set mBound = "params.m" %}{% set packed4x8CompatUnpackI8 = quantBits == 8 or q3k or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or ((((iq3xxs | default(false)) or (iq3s | default(false))) and (iqGridBound | default(false)))) or (q4knCompact | default(false)) or not (quantBits == 8 or (lut2 | default(false)) or lutId or q5 or q2k or q3k or q6k or q5k or (q4knCompact | default(false)) or (iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false)) or (iq3s | default(false))) %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;
{% if not kSplits %}const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;
{% endif %}const BPR:u32=K/32u;const K_V4:u32=K/4u;const BM:u32={{ bm }}u;const BN:u32={{ bn }}u;const K_BLOCKS:u32={{ kBlocks }}u;const BK_V4:u32=K_BLOCKS*8u;const N_TILE:u32={{ nTile }}u;const WG:u32=128u;
{% if mPerThread > 1 %}const M_PER_THREAD:u32={{ mPerThread }}u;
{% endif %}{% if kSplits %}const SPLIT_BLOCKS:u32={{ splitBlocks }}u;
{% endif %}alias TileVec=vec4<f32>;var<workgroup>aTile:array<TileVec,{{ bm * kBlocks * 8 }}>;var<workgroup>bTile:array<TileVec,{{ bn * kBlocks * 8 }}>;@compute@workgroup_size(WG,1,1){{ " @subgroup_size(" ~ pinnedSubgroupSize ~ ")" if (pinnedSubgroupSize | default(0)) else "" }} fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid.x;let m_local=tid/N_TILE;let n_local=tid-m_local*N_TILE;
{% if mPerThread > 1 %}let m_row=m_local*M_PER_THREAD;
{% endif %}let n_tiles=(N+BN-1u)/BN;let m_tiles=({{ mBound }}+BM-1u)/BM;for(var wgx:u32=wg.x;wgx<n_tiles;wgx=wgx+nwg.x){let nBase=wgx*BN;for(var wgy:u32=wg.y;wgy<m_tiles;wgy=wgy+nwg.y){let mBase=wgy*BM;let m=mBase+{{ rowVar }};
{% for nn in range(mPerThread * accs) %}var acc{{ nn }}:f32=0.0;
{% endfor %}{% if kSplits %}let ktEnd=min((wg.z+1u)*SPLIT_BLOCKS,BPR);for(var kt:u32=wg.z*SPLIT_BLOCKS;kt<ktEnd;kt=kt+K_BLOCKS){
{% else %}let ktEnd=BPR;for(var kt:u32=0u;kt<ktEnd;kt=kt+K_BLOCKS){
{% endif %}let kv0=kt*8u;for(var i:u32=tid;i<{{ bm * kBlocks * 8 }}u;i=i+WG){let tm=i/BK_V4;let kv=i-tm*BK_V4;let gm=mBase+tm;
{% if kBlocks == 1 %}if(gm<{{ mBound }}){
{% else %}if(gm<{{ mBound }}&&kt+kv/8u<ktEnd){
{% endif %}aTile[i]=TileVec(a[gm*K_V4+kv0+kv]);}else{aTile[i]=TileVec(0.0);} }for(var i:u32=tid;i<{{ bn * kBlocks * 8 }}u;i=i+WG){let tn=i/BK_V4;let kv=i-tn*BK_V4;
{% if kBlocks == 1 %}let eib=kv*4u;
{% else %}let blockInTile=kv/8u;let eib=(kv%8u)*4u;
{% endif %}let r=nBase+tn;var v=vec4<f32>(0.0);
{% if kBlocks == 1 %}if(r<N){let blk=params.block_offset+r*BPR+kt;
{% else %}if(r<N&&kt+blockInTile<ktEnd){let blk=params.block_offset+r*BPR+kt+blockInTile;
{% endif %}{% if quantBits == 8 %}let word=q4_bits[blk*8u+eib/4u];
{% if iq %}let scale=f32(q4_scales[blk*2u+select(0u,1u,eib>=16u)]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}v=vec4<f32>(unpack4xI8(word))*scale;
{% elif q5 %}let g5=(blk>>2u)*20u;let l5=blk&3u;let word=q4_bits[g5+l5*4u+(eib%16u)/4u];let qh=q4_bits[g5+16u+l5];let scale=f32(q4_scales[blk]);var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));v=(vec4<f32>(q5n)-16.0)*scale;
{% elif q4knCompact is defined and q4knCompact %}let sb4nc=blk>>3u;let j4nc=blk&7u;let smBase4nc=sb4nc*5u;let dm4nc=unpack2x16float(q4_metadata[smBase4nc]);let smCarrier4nc=q4_metadata[smBase4nc+1u+(j4nc>>1u)];let smPair4nc=smCarrier4nc>>((j4nc&1u)*12u);let sc4nc=smPair4nc&63u;let mn4nc=(smPair4nc>>6u)&63u;let word=q4_bits[blk*4u+(eib%16u)/4u];var nib4nc:vec4<u32>;if(eib<16u){nib4nc=unpack4xU8(word&0x0F0F0F0Fu);}else{nib4nc=unpack4xU8((word>>4u)&0x0F0F0F0Fu);}let dl4nc=dm4nc.x*f32(sc4nc);let ml4nc=dm4nc.y*f32(mn4nc);let product4nc=dl4nc*vec4<f32>(nib4nc);v=product4nc-vec4<f32>(ml4nc);
{% elif q5k is defined and q5k %}let sb5=blk>>3u;let j5=blk&7u;let g5k=sb5*44u;let word=q4_bits[g5k+4u*j5+(eib%16u)/4u];let qh=q4_bits[g5k+32u+j5];let sc5=f32((q4_bits[g5k+40u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let mn5=f32((q4_bits[g5k+42u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let scEff5=f32(q4_scales[sb5*2u])*sc5;let mEff5=f32(q4_scales[sb5*2u+1u])*mn5;var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));v=vec4<f32>(q5n)*scEff5-vec4<f32>(mEff5);
{% elif q2k %}let sb=blk>>3u;let j2=blk&7u;let g2=sb*20u;let cw=q4_bits[g2+2u*j2+select(0u,1u,eib>=16u)];let scb=(q4_bits[g2+16u+(j2>>1u)]>>((j2&1u)*16u+select(0u,8u,eib>=16u)))&0xFFu;let d2=f32(q4_scales[sb*2u]);let dm2=f32(q4_scales[sb*2u+1u]);let w2=(eib%16u)/4u;let q2=(vec4<u32>(cw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w2)))&vec4<u32>(3u);v=vec4<f32>(q2)*(d2*f32(scb&15u))-vec4<f32>(dm2*f32(scb>>4u));
{% elif q3k %}let sb=blk>>3u;let j3=blk&7u;let g3=sb*28u;let cw3=q4_bits[g3+2u*j3+select(0u,1u,eib>=16u)];let hw3=q4_bits[g3+16u+j3];let sc83=unpack4xI8(q4_bits[g3+24u+(j3>>1u)])[(j3&1u)*2u+select(0u,1u,eib>=16u)];let scale3=f32(q4_scales[sb])*f32(sc83);let w3=(eib%16u)/4u;let q3n=(vec4<u32>(cw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w3)))&vec4<u32>(3u);let h3=(vec4<u32>(hw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(select(0u,4u,eib>=16u)+w3)))&vec4<u32>(1u);v=(vec4<f32>(q3n|(h3<<vec4<u32>(2u)))-4.0)*scale3;
{% elif q6k %}let sb=blk>>3u;let j6=blk&7u;let g6=sb*52u;let w6=(eib%16u)/4u;let word=q4_bits[g6+4u*j6+w6];let hw=q4_bits[g6+32u+2u*j6+select(0u,1u,eib>=16u)];let sc8=unpack4xI8(q4_bits[g6+48u+(j6>>1u)])[(j6&1u)*2u+select(0u,1u,eib>=16u)];let scale=f32(q4_scales[sb])*f32(sc8);var q6n:vec4<u32>;if(eib<16u){q6n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q6n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}let hi2=(vec4<u32>(hw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w6)))&vec4<u32>(3u);v=(vec4<f32>(q6n|(hi2<<vec4<u32>(4u)))-32.0)*scale;
{% else %}let word=q4_bits[blk*4u+(eib%16u)/4u];
{% if hasMin %}let scale=f32(q4_scales[blk*2u]);let bias=f32(q4_scales[blk*2u+1u]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}var nib:vec4<f32>;if(eib<16u){nib=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}else{nib=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}
{% if hasMin %}v=nib*scale+bias;
{% else %}v=nib*scale;
{% endif %}{% endif %}}bTile[i]=TileVec(v);}workgroupBarrier();
{% if mPerThread > 1 %}if(m<{{ mBound }}){for(var kv:u32=0u;kv<BK_V4;kv=kv+1u){
{% for mm in range(mPerThread) %}let a_vec{{ mm }}=vec4<f32>(aTile[({{ rowVar }}+{{ mm }}u)*BK_V4+kv]);
{% endfor %}{% for nn in range(accs) %}let b_vec{{ nn }}=vec4<f32>(bTile[(n_local+{{ nn * nTile }}u)*BK_V4+kv]);
{% endfor %}{% for mm in range(mPerThread) %}{% for nn in range(accs) %}acc{{ mm * accs + nn }}=fma(a_vec{{ mm }}.w,b_vec{{ nn }}.w,fma(a_vec{{ mm }}.z,b_vec{{ nn }}.z,fma(a_vec{{ mm }}.y,b_vec{{ nn }}.y,fma(a_vec{{ mm }}.x,b_vec{{ nn }}.x,acc{{ mm * accs + nn }}))));
{% endfor %}{% endfor %}} }
{% else %}if(m<{{ mBound }}){for(var kv:u32=0u;kv<BK_V4;kv=kv+1u){let a_vec=vec4<f32>(aTile[m_local*BK_V4+kv]);
{% for nn in range(accs) %}let b_vec{{ nn }}=vec4<f32>(bTile[(n_local+{{ nn * nTile }}u)*BK_V4+kv]);acc{{ nn }}=fma(a_vec.w,b_vec{{ nn }}.w,fma(a_vec.z,b_vec{{ nn }}.z,fma(a_vec.y,b_vec{{ nn }}.y,fma(a_vec.x,b_vec{{ nn }}.x,acc{{ nn }}))));
{% endfor %}} }
{% endif %}workgroupBarrier();}
{% for mm in range(mPerThread) %}{% set mRow = "m" if mPerThread == 1 else "(m + " ~ mm ~ "u)" %}{% set aBase = mm * accs %}if({{ mRow }}<{{ mBound }}){
{% if kSplits %}{% for nn in range(accs) %}if(nBase+n_local+{{ nn * nTile }}u<N){partials[(wg.z*params.m+{{ mRow }})*N+nBase+n_local+{{ nn * nTile }}u]=acc{{ aBase + nn }};}
{% endfor %}{% else %}{% for nn in range(accs) %}if(nBase+n_local+{{ nn * nTile }}u<N){
          y[{{ mRow }} * OUT_STRIDE + DST_COL + nBase + n_local + {{ nn * nTile }}u] = {% if yScalar == "f16" %}f16(acc{{ aBase + nn }}){% else %}acc{{ aBase + nn }}{% endif %};
}
{% endfor %}{% endif %}}
{% endfor %}} } }`],["prefill-matmul-q4-decode-gemv.wgsl.jinja",`{% set emitQuantDotHelpers = true %}{% set nibbleHelpers = not (q5 or q2k or q3k or q5k or q6k) %}{% set aF16 = aF16 if aF16 is defined else false %}{% set yF16 = yF16 if yF16 is defined else false %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not lutId and quantBits != 8 %}{% include "ops/_shared/packed-4x8-compat.4dcfc72c71.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const BPR:u32={{ blocksPerRow }}u;const DST_COL:u32={{ dstColStart }}u;const CLUSTERS:u32={{ clusters }}u;const REG_ROWS:u32={{ regRows }}u;
{% if mRows > 1 %}const OUT_STRIDE:u32={{ outStride }}u;const A_ROW_V4:u32=BPR*8u;
{% endif %}{% include "ops/_shared/subgroup-runtime-fold.3a89b81fa3.wgsl.jinja" %}{% set sgOwned = useSubgroups and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.87076f64db.wgsl.jinja" %}{% set sgDim0 = "(wid.x * CLUSTERS + cluster) * REG_ROWS" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 32 * clusters %}{% include "ops/_shared/subgroup-runtime-proof.b3a3fde565.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}var<workgroup>partials4:array<f32,{{ 32 * clusters }}>;
{% endif %}{% include "ops/_shared/q4-dequant-helpers.a65f490371.wgsl.jinja" %}@compute@workgroup_size({{ 32 * clusters }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% if sgOwned %}{{ sg_cluster_open(sgPath, "CLUSTERS", clusters * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*CLUSTERS+cluster)*REG_ROWS;let base0=params.block_offset+dim0*BPR;
{% for r in range(regRows) %}var acc{{ r }}=0.0;
{% endfor %}{% for m in range(1, mRows) %}{% for r in range(regRows) %}var accM{{ m }}_{{ r }}=0.0;
{% endfor %}{% endfor %}for(var b=lane;b<BPR;b=b+{{ "activeCount" if sgPath == "general" else "32u" }}){let hbase=b*8u;let x0={{ "vec4<f32>(a[hbase])" if aF16 else "a[hbase]" }};let x1={{ "vec4<f32>(a[hbase + 1u])" if aF16 else "a[hbase + 1u]" }};let x2={{ "vec4<f32>(a[hbase + 2u])" if aF16 else "a[hbase + 2u]" }};let x3={{ "vec4<f32>(a[hbase + 3u])" if aF16 else "a[hbase + 3u]" }};let x4={{ "vec4<f32>(a[hbase + 4u])" if aF16 else "a[hbase + 4u]" }};let x5={{ "vec4<f32>(a[hbase + 5u])" if aF16 else "a[hbase + 5u]" }};let x6={{ "vec4<f32>(a[hbase + 6u])" if aF16 else "a[hbase + 6u]" }};let x7={{ "vec4<f32>(a[hbase + 7u])" if aF16 else "a[hbase + 7u]" }};
{% include "ops/_shared/q4-decode-row-block-accumulate.98693154e0.wgsl.jinja" %}{% for m in range(1, mRows) %}{let hbase{{ m }}={{ m }}u*A_ROW_V4+hbase;let x0={{ "vec4<f32>(a[hbase" ~ m ~ "])" if aF16 else "a[hbase" ~ m ~ "]" }};let x1={{ "vec4<f32>(a[hbase" ~ m ~ " + 1u])" if aF16 else "a[hbase" ~ m ~ " + 1u]" }};let x2={{ "vec4<f32>(a[hbase" ~ m ~ " + 2u])" if aF16 else "a[hbase" ~ m ~ " + 2u]" }};let x3={{ "vec4<f32>(a[hbase" ~ m ~ " + 3u])" if aF16 else "a[hbase" ~ m ~ " + 3u]" }};let x4={{ "vec4<f32>(a[hbase" ~ m ~ " + 4u])" if aF16 else "a[hbase" ~ m ~ " + 4u]" }};let x5={{ "vec4<f32>(a[hbase" ~ m ~ " + 5u])" if aF16 else "a[hbase" ~ m ~ " + 5u]" }};let x6={{ "vec4<f32>(a[hbase" ~ m ~ " + 6u])" if aF16 else "a[hbase" ~ m ~ " + 6u]" }};let x7={{ "vec4<f32>(a[hbase" ~ m ~ " + 7u])" if aF16 else "a[hbase" ~ m ~ " + 7u]" }};
{% set accPrefix = "accM" ~ m ~ "_" %}{% include "ops/_shared/q4-decode-row-block-accumulate.98693154e0.wgsl.jinja" %}{% set accPrefix = "acc" %}}
{% endfor %}}
{% if not useSubgroups and not sgRuntime and mRows > 1 %}{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% set totalPrefix = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.e8bf64d477.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if sgPath == "general" %}{{ sg_cluster_store("CLUSTERS") }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let {{ "total" if m == 0 else "totalM" ~ m ~ "_" }}{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if sgPath == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% set rowTotal = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set rowBase = "" if m == 0 else m ~ "u * OUT_STRIDE + " %}{% for r in range(regRows) %}{% if yF16 %}y[{{ rowBase }}DST_COL+dim0+{{ r }}u]=f16({{ rowTotal }}{{ r }});
{% else %}y[{{ rowBase }}DST_COL+dim0+{{ r }}u]={{ rowTotal }}{{ r }};
{% endif %}{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}{% endfor %}}`]],assumes:[["prefill-matmul-dense-vector.wgsl.jinja",{yScalar:"f32"}],["prefill-matmul-dense-gemv-mrows.wgsl.jinja",{clusters:4,regRows:2,yScalar:"f32"}],["ops/_shared/gemm-subgroup-matrix.wgsl.jinja",{lutc16:0}],["prefill-matmul-q4.wgsl.jinja",{lut2:0,lutc:0,lutc16:0,lutId:0}],["prefill-matmul-q4-decode-gemv.wgsl.jinja",{q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q5k:0,lutId:0}]]}],["com.xenova.LlamaPrefillMoeDown",{manifest:{domain:"com.xenova",name:"LlamaPrefillMoeDown",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:1,optional:!0},intermediateT:{onnx:"Intermediate",dtype:"I",rank:"2 if args.seqLen is defined else 1"},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},routeIdxT:{onnx:"RouteIdx",dtype:"uint32",rank:1},routeWT:{onnx:"RouteW",dtype:"float32",rank:1},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0},sharedBitsT:{onnx:"SharedBits",dtype:"uint32",rank:1,optional:!0},sharedScalesT:{onnx:"SharedScales",dtype:"SS",rank:1,optional:!0}},outputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:1,optional:!0,shape:["args.hiddenSize"]},yT:{onnx:"Y",dtype:"Y",rank:2,optional:!0,shape:["args.seqLen","args.hiddenSize"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},moeIntermediateSize:{kind:"u32",onnx:"moe_intermediate_size"},seqLen:{kind:"u32",onnx:"sequence_length",required:!1},numExperts:{kind:"u32",onnx:"expert_count"},topK:{kind:"u32",onnx:"experts_per_token"},downOffset:{kind:"u32",onnx:"down_exps_offset"},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q2_k","q5_k","q6_k","q8_0","q8_0_pair","iq","lut32","lut16","lut32_compact","lut16_compact","iq4_xs"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1,oneOf:[0,1,2,3,4,5,6,7,8]},sharedIntermediateSize:{kind:"u32",onnx:"shared_expert_intermediate_size",required:!1},sharedDownOffset:{kind:"u32",onnx:"shared_down_offset",required:!1},sharedFormat:{kind:"string",onnx:"shared_format",required:!1,oneOf:["q4_0","q4_k_native_compact","q5_k","q6_k","q8_0","lut32_compact"]},sharedLut:{kind:"u32",onnx:"shared_dequant_lut_id",required:!1,oneOf:[0,1,2,3,4,5,6,7,8,9]}},typeConstraints:{I:["float32","float16"],W:["float32","float16","uint32"],Y:["float32","float16"],S:["float32","float16","uint32"],SS:["float32","float16","uint32"]},tunables:{H_TILE:{default:4},H_TILE_DECODE:{default:4}},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',iqRawFormat:"args.iqRaw == 1 or args.iqRaw == 2 or args.iqRaw == 3",iqRawBitsPerBlock:"3 if args.iqRaw == 3 else 2",iqRawGridWords:"4096 if args.iqRaw == 1 else (512 if args.iqRaw == 2 else 256)",iqRawSuperFormat:"args.iqRaw == 4 or args.iqRaw == 5 or args.iqRaw == 6",iqRawSuperWords:"27 if args.iqRaw == 4 else (20 if args.iqRaw == 5 else 18)",iqRawSuperGridWords:"512 if args.iqRaw == 4 else (2048 if args.iqRaw == 5 else 1024)",iq4xs:'1 if fmt == "iq4_xs" else 0',sharedExpert:"1 if present.sharedBitsT else 0",sharedFmt:'args.sharedFormat if args.sharedFormat else ""',sharedQuantBits:'8 if sharedFmt == "q8_0" else 4',sharedQ5k:'1 if sharedFmt == "q5_k" else 0',sharedQ6k:'1 if sharedFmt == "q6_k" else 0',sharedQ4knc:'1 if sharedFmt == "q4_k_native_compact" else 0',sharedLutc:'1 if sharedFmt == "lut32_compact" else 0',sharedLutId:"args.sharedLut if (sharedLutc and args.sharedLut) else 0",sharedSuper:"sharedQ5k or sharedQ6k or sharedQ4knc or sharedLutc",sharedU32Metadata:"sharedLutc or sharedQ4knc",sharedScaleScalar:'"u32" if (sharedU32Metadata or not sharedExpert) else dtypes.SS',sharedFormatOk:'(sharedFmt == "q4_0" or sharedFmt == "q8_0" or sharedSuper) and ((args.sharedLut is defined and args.sharedLut != 0) if sharedLutc else args.sharedLut is undefined)',sharedTensorsOk:'ranks.sharedBitsT == 1 and ranks.sharedScalesT == 1 and tensorDtypes.sharedBitsT == "uint32" and (tensorDtypes.sharedScalesT == "uint32" if sharedU32Metadata else (tensorDtypes.sharedScalesT == "float32" or tensorDtypes.sharedScalesT == "float16")) and f16Ok(dtypes.SS) and dim(shapes.sharedBitsT, 0) % 4 == 0',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',rowCount:"args.seqLen if args.seqLen is defined else 1",decodeMode:"present.hiddenT and not present.yT and args.seqLen is undefined",prefillMode:"not present.hiddenT and present.yT and args.seqLen is defined",slotCount:"rowCount * args.topK",downHTileWant:"max(1, tunables.H_TILE)",downHTile:"downHTileWant if args.hiddenSize % downHTileWant == 0 else 1",decodeHTileWant:"max(1, tunables.H_TILE_DECODE if quantBits == 4 else tunables.H_TILE_DECODE / 2)",decodeHTile:"decodeHTileWant if args.hiddenSize % decodeHTileWant == 0 else 1",denseWeightEnd:"args.downOffset + args.numExperts * args.hiddenSize * args.moeIntermediateSize",packedWeightRowsEnd:"args.downOffset + args.numExperts * args.hiddenSize * (args.moeIntermediateSize / 32)",lutcScaleWords:"(packedWeightRowsEnd / 8) * lutcWords",lutcPackOk:"(not lutc) or (args.lut is defined and args.lut != 0 and args.moeIntermediateSize % 256 == 0 and args.downOffset % 8 == 0 and packedWeightRowsEnd % 8 == 0)",q4knc:'1 if fmt == "q4_k_native_compact" else 0',kSuper:"q4knc or q6k or q5k or q2k or iq4xs",downSuperblocks:"ceilDiv(packedWeightRowsEnd, 8)",kSuperPackOk:"(not kSuper) or ((not iq4xs or (args.lut is defined and args.lut == 1)) and args.moeIntermediateSize % 256 == 0 and args.downOffset % 8 == 0 and packedWeightRowsEnd % 8 == 0 and dim(shapes.bitsT, 0) % 4 == 0)",u32Metadata:"lutc or q4knc or iq4xs",packedBitsNeeded:"downSuperblocks * 52 if q6k else (downSuperblocks * 44 if q5k else (downSuperblocks * 20 if q2k else packedWeightRowsEnd * quantBits))",packedScalesNeeded:"downSuperblocks * 5 if q4knc else lutcScaleWords if lutc else downSuperblocks if q6k else downSuperblocks * 2 if (q5k or q2k or iq4xs) else packedWeightRowsEnd * (2 if (hasMin or iq or lut16) else 1)",dispatchFits:"args.hiddenSize <= device.limits.maxComputeWorkgroupsPerDimension and rowCount <= device.limits.maxComputeWorkgroupsPerDimension",denseSource:"present.weightsT and not present.bitsT and not present.scalesT and args.format is undefined and args.lut is undefined and args.iqRaw is undefined",packedSource:"not present.weightsT and present.bitsT and present.scalesT",regularPacked:"args.iqRaw is undefined or args.iqRaw == 0",commonContract:'ranks.routeIdxT == 1 and ranks.routeWT == 1 and tensorDtypes.routeIdxT == "uint32" and tensorDtypes.routeWT == "float32" and (tensorDtypes.intermediateT == "float32" or tensorDtypes.intermediateT == "float16") and args.hiddenSize > 0 and args.moeIntermediateSize > 0 and args.numExperts >= 1 and args.topK >= 1 and args.topK <= 32 and dispatchFits',decodeCoreContract:'commonContract and decodeMode and ranks.hiddenT == 1 and ranks.intermediateT == 1 and tensorDtypes.hiddenT == "float32" and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.intermediateT, 0) >= args.topK * args.moeIntermediateSize and dim(shapes.routeIdxT, 0) >= args.topK and dim(shapes.routeWT, 0) >= args.topK',sharedAbsent:"not present.sharedBitsT and not present.sharedScalesT and args.sharedIntermediateSize is undefined and args.sharedDownOffset is undefined and args.sharedFormat is undefined and args.sharedLut is undefined",decodeBaseContract:"decodeCoreContract and sharedAbsent",sharedRowsEnd:"args.sharedDownOffset + args.hiddenSize * (args.sharedIntermediateSize / 32) if sharedExpert else 0",sharedSuperblocks:"ceilDiv(sharedRowsEnd, 8)",sharedBitsNeeded:"sharedSuperblocks * 52 if sharedQ6k else (sharedSuperblocks * 44 if sharedQ5k else sharedRowsEnd * sharedQuantBits)",sharedScalesNeeded:"sharedSuperblocks * 5 if sharedQ4knc else (sharedSuperblocks * 3 if sharedLutc else (sharedSuperblocks if sharedQ6k else (sharedSuperblocks * 2 if sharedQ5k else sharedRowsEnd)))",sharedBpr:"args.sharedIntermediateSize / 32 if sharedExpert else 0",sharedDownOffset:"args.sharedDownOffset if sharedExpert else 0",decodeSharedContract:"decodeCoreContract and present.sharedBitsT and present.sharedScalesT and args.sharedIntermediateSize is defined and args.sharedDownOffset is defined and args.sharedFormat is defined and sharedFormatOk and sharedTensorsOk and args.sharedIntermediateSize > 0 and args.sharedIntermediateSize % 256 == 0 and (args.sharedDownOffset % 8 == 0 if sharedSuper else true) and dim(shapes.sharedBitsT, 0) >= sharedBitsNeeded and dim(shapes.sharedScalesT, 0) >= sharedScalesNeeded and dim(shapes.intermediateT, 0) >= args.topK * args.moeIntermediateSize + args.sharedIntermediateSize",prefillBaseContract:"commonContract and prefillMode and args.iqRaw is undefined and ranks.intermediateT == 2 and ranks.yT == 2 and args.seqLen > 0 and numel(shapes.intermediateT) >= slotCount * args.moeIntermediateSize and dim(shapes.routeIdxT, 0) >= slotCount and dim(shapes.routeWT, 0) >= slotCount and numel(shapes.yT) >= rowCount * args.hiddenSize",weightElements:'(dim(shapes.weightsT, 0) * (2 if tensorDtypes.weightsT == "uint32" else 1)) if present.weightsT else 0',decodeDenseContract:'decodeBaseContract and denseSource and ranks.weightsT == 1 and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and f16Ok(dtypes.W) and f16Ok(dtypes.I) and args.moeIntermediateSize % 4 == 0 and args.downOffset % 4 == 0 and weightElements >= denseWeightEnd',prefillDenseContract:'prefillBaseContract and denseSource and ranks.weightsT == 1 and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and f16Ok(tensorDtypes.intermediateT) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.yT) and args.moeIntermediateSize % 4 == 0 and args.downOffset % 4 == 0 and weightElements >= denseWeightEnd',intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',yScalar:'"f32" if decodeMode else dtypes.Y',useSubgroups:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32 and 8 + 4 * subgroupCoveringWorkgroup <= device.limits.maxComputeWorkgroupStorageSize',downSegLanes:"8 if args.moeIntermediateSize <= 256 else (16 if args.moeIntermediateSize <= 512 else 32)",decodeLaneSegmentsWant:"floor(subgroupCoveringWorkgroup / downSegLanes) if (useSubgroups and downSegLanes < subgroupCoveringWorkgroup) else 1",decodeSegHTile:"max(1, floor(tunables.H_TILE_DECODE / decodeLaneSegmentsWant))",decodeLaneSegments:"decodeLaneSegmentsWant if args.hiddenSize % (decodeLaneSegmentsWant * decodeSegHTile) == 0 else 1",decodeIqHTile:"decodeSegHTile if decodeLaneSegments > 1 else 1",decodeQ4HTile:"decodeSegHTile if decodeLaneSegments > 1 else decodeHTile",scaleScalar:"dtypes.S",blocksPerRow:"args.moeIntermediateSize / 32",downOffset:"args.downOffset",weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec2<u32>" if tensorDtypes.weightsT == "uint32" else "vec4<f32>"',intermediateVec4Count:"args.moeIntermediateSize / 4",downOffsetVec4:"args.downOffset / 4",q4knCompact:"q4knc",lutId:"args.lut if args.lut else 0",hiddenSize:"args.hiddenSize",intermediateSize:"args.moeIntermediateSize",topK:"args.topK",workgroupSize:"subgroupCoveringWorkgroup",denseWorkgroup:"32",decodeQ4Contract:'decodeBaseContract and packedSource and regularPacked and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if u32Metadata else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and f16Ok(dtypes.S) and f16Ok(dtypes.I) and args.moeIntermediateSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and not q8Pair and lutcPackOk and kSuperPackOk and dim(shapes.bitsT, 0) >= packedBitsNeeded and dim(shapes.scalesT, 0) >= packedScalesNeeded and (8 + 4 * subgroupCoveringWorkgroup if useSubgroups else 4 * subgroupCoveringWorkgroup * decodeHTile) <= device.limits.maxComputeWorkgroupStorageSize',decodeQ8Tile2Contract:'decodeQ4Contract and args.format == "q8_0" and quantBits == 8 and not hasMin and not iq and not lut16 and args.hiddenSize % 2 == 0 and (8 + 4 * subgroupCoveringWorkgroup if useSubgroups else 8 * subgroupCoveringWorkgroup) <= device.limits.maxComputeWorkgroupStorageSize',prefillQ4Contract:'prefillBaseContract and packedSource and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if u32Metadata else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and f16Ok(tensorDtypes.intermediateT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.yT) and args.moeIntermediateSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and not q8Pair and lutcPackOk and kSuperPackOk and dim(shapes.bitsT, 0) >= packedBitsNeeded and dim(shapes.scalesT, 0) >= packedScalesNeeded and (8 + 4 * subgroupCoveringWorkgroup if useSubgroups else 4 * subgroupCoveringWorkgroup * downHTile) <= device.limits.maxComputeWorkgroupStorageSize',prefillQ8PairTile2Contract:'prefillBaseContract and packedSource and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.intermediateT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.yT) and args.hiddenSize % 2 == 0 and q8Pair and packedWeightRowsEnd % 8 == 0 and dim(shapes.bitsT, 0) >= (packedWeightRowsEnd / 8) * 68 and (8 + 4 * subgroupCoveringWorkgroup if useSubgroups else 8 * subgroupCoveringWorkgroup) <= device.limits.maxComputeWorkgroupStorageSize',weightScalar:"dtypes.W"},bindings:{intermediate:{arg:"intermediateT",elementType:"$intermediateVec4"},route_idx:{arg:"routeIdxT",elementType:"u32"},route_w:{arg:"routeWT",elementType:"f32"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},y:{arg:"yT",elementType:"$yScalar"}},variants:[{id:"q4",priority:0,when:["prefillQ4Contract"],derive:{hTile:"downHTile"},passes:[{id:"main",name:"LlamaPrefillMoeDownQ4",shader:"moe-down-q4.wgsl.jinja",derive:{inPlace:!1},bindings:["intermediate","q4_bits","q4_scales","route_idx","route_w","y"],dispatch:{x:"args.hiddenSize / downHTile",y:"rowCount"}}]}]},assets:[["moe-down-q4-dot.ad20ecd73f.wgsl.jinja",'{% set rowBase = rowBase | default("row_block_base") %}{% set dotName = dotName | default("dotk") %}let ds=f32(q4_scales[{{ rowBase }}+b]);let dw0=q4_bits[({{ rowBase }}+b)*2u];let dw1=q4_bits[({{ rowBase }}+b)*2u+1u];{{ dotName }}={{ dotName }}+ds*(q8dot(dw0.x,x0)+q8dot(dw0.y,x1)+q8dot(dw0.z,x2)+q8dot(dw0.w,x3)+q8dot(dw1.x,x4)+q8dot(dw1.y,x5)+q8dot(dw1.z,x6)+q8dot(dw1.w,x7));'],["moe-down-q4-expert-loop.816982b125.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.8e1bf311b5.wgsl.jinja" %}{% set laneSegments = 1 %}{% set hSuffixes = [""] if hTile == 1 else range(hTile) %}const HIDDEN_SIZE:u32={{ hiddenSize }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const BPR:u32={{ blocksPerRow }}u;const TOP_K:u32={{ topK }}u;const W_DOWN_Q4:u32={{ downOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% if hTile > 1 %}const H_TILE:u32={{ hTile }}u;
{% endif %}{% if not useSubgroups %}{% for sfx in hSuffixes %}var<workgroup>partials{{ sfx }}:array<f32,{{ workgroupSize }}>;
{% endfor %}{% endif %}{% include "ops/_shared/q4-dequant-helpers.8aab4e5b5a.wgsl.jinja" %}{% if useSubgroups %}{% set localFoldWorkgroupSize = workgroupSize %}{% set localFoldWidth = workgroupSize %}{% include "ops/_shared/subgroup-local-fold.5eb84b12bf.wgsl.jinja" %}{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let h=wid.x{{ " * H_TILE" if hTile > 1 else "" }};let t=wid.y;let lid=lid3.x;
{% if useSubgroups %}let activeMask=subgroupBallot(true);let activeCount=countOneBits(activeMask.x)+countOneBits(activeMask.y)+countOneBits(activeMask.z)+countOneBits(activeMask.w);let nativeTopology=subgroupLocalLayoutAccepted(lid,activeCount==WORKGROUP_SIZE);
{% endif %}if(h>=HIDDEN_SIZE){return;}
{% for sfx in hSuffixes %}var acc{{ sfx }}=0.0;
{% endfor %}for(var k=0u;k<TOP_K;k=k+1u){let expert=route_idx[t*TOP_K+k];
{% for sfx in hSuffixes %}let row_block_base{{ sfx }}=W_DOWN_Q4+(expert*HIDDEN_SIZE+h{{ (" + " ~ loop.index0 ~ "u") if loop.index0 > 0 else "" }})*BPR;
{% endfor %}let inter_vec4_base=t*(TOP_K*INTERMEDIATE_SIZE/4u)+(k*INTERMEDIATE_SIZE)/4u;
{% for sfx in hSuffixes %}var dotk{{ sfx }}=0.0;
{% endfor %}for(var b={{ "lid % SEG_LANES" if laneSegments > 1 else "lid" }};b<BPR;b=b+{{ "SEG_LANES" if laneSegments > 1 else "WORKGROUP_SIZE" }}){let ibase=inter_vec4_base+b*8u;let x0=vec4<f32>(intermediate[ibase]);let x1=vec4<f32>(intermediate[ibase+1u]);let x2=vec4<f32>(intermediate[ibase+2u]);let x3=vec4<f32>(intermediate[ibase+3u]);let x4=vec4<f32>(intermediate[ibase+4u]);let x5=vec4<f32>(intermediate[ibase+5u]);let x6=vec4<f32>(intermediate[ibase+6u]);let x7=vec4<f32>(intermediate[ibase+7u]);
{% for sfx in hSuffixes %}{% set rowBase = "row_block_base" ~ sfx %}{% set dotName = "dotk" ~ sfx %}{% if hTile > 1 %}{
{% endif %}{% include "moe-down-q4-dot.ad20ecd73f.wgsl.jinja" %}{% if hTile > 1 %}}
{% endif %}{% endfor %}}
{% for sfx in hSuffixes %}acc{{ sfx }}=acc{{ sfx }}+route_w[t*TOP_K+k]*dotk{{ sfx }};
{% endfor %}}
{% if useSubgroups %}{% for sfx in hSuffixes %}let total{{ sfx }}={{ "sumLocalSegment" if laneSegments > 1 else "sumLocalSubgroup" }}(acc{{ sfx }},lid,nativeTopology);
{% endfor %}{% else %}{% for sfx in hSuffixes %}partials{{ sfx }}[lid]=acc{{ sfx }};
{% endfor %}workgroupBarrier();
{% set foldNs = namespace(arrays=[]) %}{% for sfx in hSuffixes %}{% set foldNs.arrays = foldNs.arrays + ["partials" ~ sfx] %}{% endfor %}{{ wgsl_tree_fold(foldNs.arrays, form="head", barrierFirst=true) }}
{% for sfx in hSuffixes %}let total{{ sfx }}=partials{{ sfx }}[0];
{% endfor %}{% endif %}`],["moe-down-q4.wgsl.jinja",`{% include "ops/_shared/moe-routed-shared-packed-compat.1514a9f3cc.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "moe-down-q4-expert-loop.816982b125.wgsl.jinja" %}  if ({{ "lid % SEG_LANES" if laneSegments > 1 else "lid" }} == 0u) {
{% for sfx in hSuffixes %}{% set hIndex = "h" ~ ((" + " ~ loop.index0 ~ "u") if loop.index0 > 0 else "") %}y[t*HIDDEN_SIZE+{{ hIndex }}]={{ yScalar }}(total{{ sfx }});
{% endfor %}} }`]],assumes:[["moe-down-q4.wgsl.jinja",{quantBits:8,q8Pair:0,hasMin:0,iq:0,lut16:0,lut2:0,lutc:0,lutc16:0,q5:0,q2k:0,q3k:0,q6k:0,q5k:0,iq4xs:0,sharedExpert:0,sharedQuantBits:4,sharedQ5k:0,sharedQ6k:0,sharedQ4knc:0,sharedLutc:0,sharedLutId:0,sharedBpr:0,sharedDownOffset:0,yScalar:"f32",q4knCompact:0,lutId:0}]]}],["com.xenova.LlamaPrefillRopeCacheKv",{manifest:{domain:"com.xenova",name:"LlamaPrefillRopeCacheKv",sinceVersion:1,inputs:{projectionsT:{onnx:"Projections",dtype:"A",rank:1,optional:!0},qgT:{onnx:"Qg",dtype:"A",rank:2,optional:!0},kT:{onnx:"K",dtype:"A",rank:2,optional:!0},vT:{onnx:"V",dtype:"A",rank:2,optional:!0},qkvT:{onnx:"Qkv",dtype:"A",rank:2},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1},cosT:{onnx:"RopeCos",dtype:"float32"},sinT:{onnx:"RopeSin",dtype:"float32"},ropePositionsT:{onnx:"RopePositions",dtype:"uint32",rank:2,optional:!0}},outputs:{qkvT:{onnx:"Qkv",dtype:"A",rank:2,shape:"shapes.qkvT"},gateT:{onnx:"Gate",dtype:"A",rank:2,optional:!0,shape:["args.seqLen","args.numHeads * args.headDim"]},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,shape:"shapes.cacheKeysT"},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,shape:"shapes.cacheValuesT"}},args:{layer:{kind:"u32"},cacheLen:{kind:"u32",onnx:"cache_len"},cacheRing:{kind:"u32",onnx:"cache_ring",required:!1},pastLen:{kind:"u32",onnx:"past_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len"},numHeads:{kind:"u32",onnx:"num_heads"},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},headDim:{kind:"u32",onnx:"head_dim"},rotaryDim:{kind:"u32",onnx:"rotary_dim",required:!1},qNormOffset:{kind:"u32",onnx:"q_norm_offset",required:!1},kNormOffset:{kind:"u32",onnx:"k_norm_offset",required:!1},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},qgProjectionOffset:{kind:"u32",onnx:"qg_projection_offset",required:!1},kProjectionOffset:{kind:"u32",onnx:"k_projection_offset",required:!1},vProjectionOffset:{kind:"u32",onnx:"v_projection_offset",required:!1},mropeSectionT:{kind:"u32",onnx:"mrope_section_t",required:!1},mropeSectionH:{kind:"u32",onnx:"mrope_section_h",required:!1},mropeSectionW:{kind:"u32",onnx:"mrope_section_w",required:!1}},typeConstraints:{A:["float32","float16"],C:["float32","float16"],W:["float32","float16","uint32"]},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",qHidden:"args.numHeads * args.headDim",kvHidden:"args.numKvHeads * args.headDim",qkvStride:"qHidden + 2 * kvHidden",totalHeads:"args.numHeads + args.numKvHeads",effectiveRotaryDim:"args.rotaryDim if args.rotaryDim else args.headDim",workgroupSize:"64",pastLen:"args.pastLen if args.pastLen else 0",legacyLayout:"not present.weightsT and not present.projectionsT and not present.qgT and not present.kT and not present.vT and not present.gateT and not present.ropePositionsT",packedNormLayout:"present.weightsT and not present.projectionsT and not present.qgT and not present.kT and not present.vT and not present.gateT",projectedNormLayout:"present.weightsT and present.projectionsT and present.gateT and not present.qgT and not present.kT and not present.vT",separateNormLayout:"present.weightsT and not present.projectionsT and present.qgT and present.kT and present.vT and present.gateT",commonNormArgs:"args.qNormOffset is defined and args.kNormOffset is defined and args.rmsEps is defined",commonShape:"ranks.qkvT == 2 and ranks.cacheKeysT == 1 and ranks.cacheValuesT == 1 and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and args.cacheLen > 0 and args.seqLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.headDim > 0 and effectiveRotaryDim > 0 and effectiveRotaryDim % 2 == 0 and effectiveRotaryDim <= args.headDim and totalHeads <= device.limits.maxComputeWorkgroupsPerDimension and workgroupSize <= deviceWorkgroupCap and 4 * workgroupSize <= device.limits.maxComputeWorkgroupStorageSize and dim(shapes.qkvT, 1) == qkvStride and numel(shapes.qkvT) >= args.seqLen * qkvStride and numel(shapes.cosT) >= (pastLen + args.seqLen) * (effectiveRotaryDim / 2) and numel(shapes.sinT) >= (pastLen + args.seqLen) * (effectiveRotaryDim / 2) and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * kvHidden and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * kvHidden",commonNormDtypes:'(tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16") and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16" or tensorDtypes.weightsT == "uint32") and f16Ok(tensorDtypes.qkvT) and f16Ok(tensorDtypes.cacheKeysT) and f16Ok(tensorDtypes.weightsT)',weightsShape:'ranks.weightsT == 1 and numel(shapes.weightsT) * (2 if tensorDtypes.weightsT == "uint32" else 1) >= max(args.qNormOffset, args.kNormOffset) + args.headDim',gateShape:"ranks.gateT == 2 and tensorDtypes.gateT == tensorDtypes.qkvT and dim(shapes.gateT, 0) == args.seqLen and dim(shapes.gateT, 1) == qHidden",activationScalar:"dtypes.A",cacheScalar:"dtypes.C",layer:"args.layer",cacheLen:"args.cacheLen",numKvHeads:"args.numKvHeads",headDim:"args.headDim",weightScalar:"dtypes.W",seqLen:"args.seqLen",numHeads:"args.numHeads",rmsEps:"args.rmsEps",qNormOffset:"args.qNormOffset",kNormOffset:"args.kNormOffset",writeRotatedK:"not args.rotaryDim",mrope:"present.ropePositionsT",mropeContract:'present.ropePositionsT and ranks.ropePositionsT == 2 and tensorDtypes.ropePositionsT == "uint32" and dim(shapes.ropePositionsT, 0) == 3 and dim(shapes.ropePositionsT, 1) == args.seqLen and args.mropeSectionT is defined and args.mropeSectionH is defined and args.mropeSectionW is defined and args.mropeSectionT + args.mropeSectionH + args.mropeSectionW == effectiveRotaryDim / 2',mropeSectionH:"args.mropeSectionH if args.mropeSectionH is defined else 0",mropeSectionW:"args.mropeSectionW if args.mropeSectionW is defined else 0",ringRows:"args.cacheLen if args.cacheRing else 0"},bindings:{qkv:{arg:"qkvT",elementType:"$activationScalar"},cache_keys:{arg:"cacheKeysT",elementType:"$cacheScalar"},cache_values:{arg:"cacheValuesT",elementType:"$cacheScalar"},rope_cos:{arg:"cosT",elementType:"f32"},rope_sin:{arg:"sinT",elementType:"f32"},params:{struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"},{name:"cache_len",type:"u32",value:"args.cacheLen",required:!0,internal:!0},{name:"seq_len",type:"u32",value:"args.seqLen"}]},weights:{arg:"weightsT",elementType:"$weightScalar"}},when:["not args.cacheRing or pow2ceil(args.cacheLen) == args.cacheLen"],variants:[{id:"qk_norm",priority:1,when:["packedNormLayout","commonNormArgs","commonShape","commonNormDtypes","weightsShape","not mrope"],derive:{rotaryDim:"effectiveRotaryDim",assembledInputs:!1},passes:[{id:"main",name:"LlamaPrefillQkNormRopeCacheKv",shader:"prefill-qk-norm-rope-cache-kv.wgsl.jinja",bindings:["qkv","weights","cache_keys","cache_values","rope_cos","rope_sin","params"],dispatch:{x:"totalHeads",y:"min(args.seqLen, DISPATCH_FOLD_WIDTH)",z:"ceilDiv(args.seqLen, DISPATCH_FOLD_WIDTH)"}}]}]},assets:[["prefill-qk-norm-rope-cache-kv.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.629ddaa87a.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const SEQ_LEN:u32={{ seqLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const TOTAL_HEADS:u32=NUM_HEADS+NUM_KV_HEADS;const HEAD_DIM:u32={{ headDim }}u;const ROTARY_DIM:u32={{ rotaryDim }}u;const ROTARY_HALF:u32=ROTARY_DIM/2u;const Q_HIDDEN:u32={{ qHidden }}u;const KV_HIDDEN:u32={{ kvHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const RMS_EPS:f32={{ rmsEps }};const W_Q_NORM:u32={{ qNormOffset }}u;const W_K_NORM:u32={{ kNormOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/kv-cache-index.26c94f7350.wgsl.jinja" %}fn source_value(token:u32,head:u32,dim:u32,is_k:bool)->f32{let region=select(0u,Q_HIDDEN,is_k);return f32(qkv[token*QKV_STRIDE+region+head*HEAD_DIM+dim]);}fn normalized_checkpoint(value:f32,scale:f32,norm_offset:u32,dim:u32)->{{ activationScalar }} {return {{ activationScalar }}(value*scale*f32(weights[norm_offset+dim]));}fn store_cache(value:f32)->{{ cacheScalar }} {return {{ cacheScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let token=wid.y+wid.z*{{ DISPATCH_FOLD_WIDTH }}u;let head_index=wid.x;if(token>=SEQ_LEN||head_index>=TOTAL_HEADS){return;}let is_k=head_index>=NUM_HEADS;let head=select(head_index,head_index-NUM_HEADS,is_k);let lid=lid3.x;let row=token*QKV_STRIDE;let norm_offset=select(W_Q_NORM,W_K_NORM,is_k);var sum=0.0;for(var dim=lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){let value=source_value(token,head,dim,is_k);sum=fma(value,value,sum);}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"]) }} let scale=inverseSqrt(partials[0]/f32(HEAD_DIM)+RMS_EPS);let pos=params.past_len+token;let cache_valid=token<params.seq_len{{ "" if (ringRows is defined and ringRows) else " && pos < CACHE_LEN" }};let qkv_base=row+select(0u,Q_HIDDEN,is_k)+head*HEAD_DIM;for(var dim=lid;dim<ROTARY_HALF;dim=dim+WORKGROUP_SIZE){let first_checkpoint=normalized_checkpoint(source_value(token,head,dim,is_k),scale,norm_offset,dim);let second_dim=dim+ROTARY_HALF;let second_checkpoint=normalized_checkpoint(source_value(token,head,second_dim,is_k),scale,norm_offset,second_dim);let first=f32(first_checkpoint);let second=f32(second_checkpoint);var out_first=first;var out_second=second;if(cache_valid){let rope=pos*ROTARY_HALF+dim;let c=rope_cos[rope];let s=rope_sin[rope];out_first=first*c-second*s;out_second=second*c+first*s;}if(!is_k){qkv[qkv_base+dim]={{ activationScalar }}(out_first);qkv[qkv_base+second_dim]={{ activationScalar }}(out_second);}else{qkv[qkv_base+dim]=select(first_checkpoint,{{ activationScalar }}(out_first),cache_valid);qkv[qkv_base+second_dim]=select(second_checkpoint,{{ activationScalar }}(out_second),cache_valid);if(cache_valid){cache_keys[cache_index(pos,head,dim)]=store_cache(out_first);cache_keys[cache_index(pos,head,second_dim)]=store_cache(out_second);} } }for(var dim=ROTARY_DIM+lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){let checkpoint=normalized_checkpoint(source_value(token,head,dim,is_k),scale,norm_offset,dim);qkv[qkv_base+dim]=checkpoint;if(is_k&&cache_valid){cache_keys[cache_index(pos,head,dim)]=store_cache(f32(checkpoint));} }if(is_k){let vbase=row+Q_HIDDEN+KV_HIDDEN+head*HEAD_DIM;for(var dim=lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){let checkpoint=qkv[vbase+dim];if(cache_valid){cache_values[cache_index(pos,head,dim)]=store_cache(f32(checkpoint));} } } }`]],assumes:[["prefill-qk-norm-rope-cache-kv.wgsl.jinja",{activationScalar:"f32",cacheScalar:"f32",weightScalar:"f32",writeRotatedK:!0,mrope:!1,mropeSectionH:0,mropeSectionW:0}]]}],["com.xenova.LogMelSpectrogram",{manifest:{domain:"com.xenova",name:"LogMelSpectrogram",sinceVersion:1,inputs:{audioT:{onnx:"Audio",dtype:"T",rank:1},windowT:{onnx:"Window",dtype:"T",rank:1},melSpansT:{onnx:"MelSpans",dtype:"uint32",rank:1},melWeightsT:{onnx:"MelBandWeights",dtype:"T",rank:1}},outputs:{outT:{onnx:"Out",dtype:"T",rank:2,shape:["args.n_mels","args.n_frames"]}},args:{n_fft:{kind:"u32"},hop:{kind:"u32"},n_mels:{kind:"u32"},n_frames:{kind:"u32"},audio_length:{kind:"u32"},center:{kind:"bool",required:!1},log_floor:{kind:"f32",required:!1},clamp_to_peak:{kind:"bool",required:!1},clamp_delta:{kind:"f32",required:!1},post_add:{kind:"f32",required:!1},post_scale:{kind:"f32",required:!1}},typeConstraints:{T:["float32"]},tunables:{WORKGROUP_SIZE:{default:128},REDUCE_PARTIALS:{default:256}},derive:{numBins:"floor(args.n_fft / 2) + 1",melCount:"args.n_mels * args.n_frames",logFloor:"args.log_floor if args.log_floor is defined else 0.0000000001",clampDelta:"args.clamp_delta if args.clamp_delta is defined else 8.0",postAdd:"args.post_add if args.post_add is defined else 4.0",postScale:"args.post_scale if args.post_scale is defined else 0.25",centerFrames:"args.center if args.center is defined else true",clampToPeak:"args.clamp_to_peak if args.clamp_to_peak is defined else true",layoutOk:"ranks.audioT == 1 and ranks.windowT == 1 and ranks.melSpansT == 1 and ranks.melWeightsT == 1 and ranks.outT == 2 and dim(shapes.windowT, 0) == args.n_fft and dim(shapes.melSpansT, 0) == 2 * args.n_mels and dim(shapes.outT, 0) == args.n_mels and dim(shapes.outT, 1) == args.n_frames and args.audio_length <= dim(shapes.audioT, 0) and args.n_fft >= 2 and args.n_mels > 0 and args.n_frames > 0",fftRadix2:"pick([[args.n_fft % 256 == 0, 256], [args.n_fft % 128 == 0, 128], [args.n_fft % 64 == 0, 64], [args.n_fft % 32 == 0, 32], [args.n_fft % 16 == 0, 16], [args.n_fft % 8 == 0, 8], [args.n_fft % 4 == 0, 4]], 1)",fftTailRadix:"args.n_fft / fftRadix2",sharedFftFits:"args.n_fft * 16 <= device.limits.maxComputeWorkgroupStorageSize",AUDIO_LEN:"args.audio_length",WINDOW_LEN:"args.n_fft",MEL_SPANS_LEN:"2 * args.n_mels",MEL_OUT_LEN:"melCount",PARTIALS_LEN:"tunables.REDUCE_PARTIALS"},when:["layoutOk"],bindings:{mel_out:{arg:"outT",elementType:"f32",length:"$MEL_OUT_LEN"},audio:{arg:"audioT",elementType:"f32",length:"$AUDIO_LEN"},window:{arg:"windowT",elementType:"f32",length:"$WINDOW_LEN"},mel_spans:{arg:"melSpansT",elementType:"u32",length:"$MEL_SPANS_LEN"},mel_weights:{arg:"melWeightsT",elementType:"f32"},params__uniform:{name:"params",struct:[{name:"numFrames",type:"u32",value:"args.n_frames"},{name:"hop",type:"u32"},{name:"audioLength",type:"u32",value:"args.audio_length"},{name:"nMels",type:"u32",value:"args.n_mels"},{name:"logFloor",type:"f32",value:"logFloor"}]},params_dft_fallback:{name:"params",struct:[{name:"count",type:"u32",value:"melCount"},{name:"numFrames",type:"u32",value:"args.n_frames"},{name:"hop",type:"u32"},{name:"audioLength",type:"u32",value:"args.audio_length"},{name:"logFloor",type:"f32",value:"logFloor"}]},mel_in:{arg:"outT",buffer:"read-only-storage",elementType:"f32",length:"$MEL_OUT_LEN"},partials_out:{scratch:"max_partials",name:"partials",elementType:"f32",length:"$PARTIALS_LEN"},params_count:{name:"params",struct:[{name:"count",type:"u32",value:"melCount"}]},partials_in:{scratch:"max_partials",name:"partials",buffer:"read-only-storage",elementType:"f32",length:"$PARTIALS_LEN"},gmax_out:{scratch:"global_max",name:"gmax",elementType:"f32",length:1},gmax_in:{scratch:"global_max",name:"gmax",buffer:"read-only-storage",elementType:"f32",length:1},params_normalize:{name:"params",struct:[{name:"count",type:"u32",value:"melCount"},{name:"clampDelta",type:"f32",value:"clampDelta"},{name:"postAdd",type:"f32",value:"postAdd"},{name:"postScale",type:"f32",value:"postScale"}]}},variants:[{id:"fused_fft_mixed",priority:26,when:["pow2ceil(args.n_fft) != args.n_fft","fftTailRadix > 1","fftTailRadix <= 128","fftRadix2 >= 4","sharedFftFits"],intermediates:[{id:"max_partials",dtype:"float32",shape:"[tunables.REDUCE_PARTIALS]"},{id:"global_max",dtype:"float32",shape:"[1]"}],passes:[{id:"frame_fft_mel",name:"LogMelSpectrogram.FftMixedRecurrent",shader:"ops/_shared/fft-stockham-shared.wgsl.jinja",derive:{modeSpec:'"logmel"',fftLength:"args.n_fft",radix2Length:"fftRadix2",mixedRadixTail:!0,numBins:"numBins",inverse:!1},bindings:["audio","window","mel_spans","mel_weights","mel_out","params__uniform"],dispatch:{workgroups:"args.n_frames"}},{id:"max_partial",name:"LogMelSpectrogram.MaxPartial",shader:"logmel-global-max.wgsl.jinja",derive:{firstPass:!0},bindings:["mel_in","partials_out","params_count"],dispatch:{x:"tunables.REDUCE_PARTIALS if clampToPeak else 0"}},{id:"max_final",name:"LogMelSpectrogram.MaxFinal",shader:"logmel-global-max.wgsl.jinja",derive:{firstPass:!1},bindings:["partials_in","gmax_out"],dispatch:{x:"1 if clampToPeak else 0"}},{id:"normalize",name:"LogMelSpectrogram.Normalize",shader:"logmel-normalize.wgsl.jinja",bindings:["mel_out","gmax_in","params_normalize"],dispatch:{threads:"melCount if clampToPeak else 0",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"dft_fallback",priority:0,intermediates:[{id:"max_partials",dtype:"float32",shape:"[tunables.REDUCE_PARTIALS]"},{id:"global_max",dtype:"float32",shape:"[1]"}],passes:[{id:"frame_dft_mel",name:"LogMelSpectrogram.DftFallback",shader:"logmel-dft-fallback.wgsl.jinja",derive:{fftLength:"args.n_fft"},bindings:["audio","window","mel_spans","mel_weights","mel_out","params_dft_fallback"],dispatch:{threads:"melCount",workgroupSize:"tunables.WORKGROUP_SIZE"}},{id:"max_partial",name:"LogMelSpectrogram.MaxPartial",shader:"logmel-global-max.wgsl.jinja",derive:{firstPass:!0},bindings:["mel_in","partials_out","params_count"],dispatch:{x:"tunables.REDUCE_PARTIALS if clampToPeak else 0"}},{id:"max_final",name:"LogMelSpectrogram.MaxFinal",shader:"logmel-global-max.wgsl.jinja",derive:{firstPass:!1},bindings:["partials_in","gmax_out"],dispatch:{x:"1 if clampToPeak else 0"}},{id:"normalize",name:"LogMelSpectrogram.Normalize",shader:"logmel-normalize.wgsl.jinja",bindings:["mel_out","gmax_in","params_normalize"],dispatch:{threads:"melCount if clampToPeak else 0",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["ops/_shared/fft-stockham-shared.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% set MIXED_TAIL = true %}{% include "ops/_shared/dft-precise-twiddle.16fc24b25e.wgsl.jinja" %}{% set TAIL_ANCHOR = 32 %}{% set REANCHOR_TAIL = MIXED_TAIL
    and fftLength / radix2Length > TAIL_ANCHOR %}const N:u32={{ fftLength }}u;const N_HALF:u32=N/2u;const RADIX2_N:u32={{ radix2Length }}u;const RADIX2_HALF:u32=RADIX2_N/2u;const TAIL_RADIX:u32=N/RADIX2_N;
{% if REANCHOR_TAIL %}const TAIL_ANCHOR:u32={{ TAIL_ANCHOR }}u;
{% endif %}const NUM_BINS:u32={{ numBins }}u;const WG:u32={{ tunables.WORKGROUP_SIZE }}u;const SIGN:f32={{ "1.0" if inverse else "-1.0" }};fn complex_mul(a:vec2<f32>,b:vec2<f32>)->vec2<f32>{return vec2<f32>(a.x*b.x-a.y*b.y,a.x*b.y+a.y*b.x);}var<workgroup>shRe:array<f32,2u*N>;var<workgroup>shIm:array<f32,2u*N>;@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(workgroup_id)wgid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let tid=lid.x;let flat=wgid.x+wgid.y*{{ DISPATCH_FOLD_WIDTH }}u;let frame=flat;if(frame>=params.numFrames){return;}let frameStart=i32(frame*params.hop){{ " - i32(N / 2u)" if centerFrames else "" }};for(var n=tid;n<N;n=n+WG){let sample_n=(n/RADIX2_N)+TAIL_RADIX*(n%RADIX2_N);let position=frameStart+i32(sample_n);let mirrored=u32(select(position,-position,position<0));var xr=0.0;if(mirrored<params.audioLength){xr=audio[mirrored]*window[sample_n];}shRe[n]=xr;shIm[n]=0.0;}workgroupBarrier();var srcOff=0u;var dstOff=N;for(var s=0u;(1u<<s)<RADIX2_N;s=s+1u){let ns=1u<<s;for(var j=tid;j<N_HALF;j=j+WG){let sequence=j/RADIX2_HALF;let local_j=j-sequence*RADIX2_HALF;let sequence_base=sequence*RADIX2_N;let jm=local_j&(ns-1u);let outBase=sequence_base+((local_j>>s)<<(s+1u))+jm;let ar=shRe[srcOff+sequence_base+local_j];let ai=shIm[srcOff+sequence_base+local_j];let br=shRe[srcOff+sequence_base+RADIX2_HALF+local_j];let bi=shIm[srcOff+sequence_base+RADIX2_HALF+local_j];let twiddle=dft_precise_twiddle(jm,2u*ns,SIGN);let c=twiddle.x;let sn=twiddle.y;let tr=br*c-bi*sn;let ti=br*sn+bi*c;shRe[dstOff+outBase]=ar+tr;shIm[dstOff+outBase]=ai+ti;shRe[dstOff+outBase+ns]=ar-tr;shIm[dstOff+outBase+ns]=ai-ti;}workgroupBarrier();let swap=srcOff;srcOff=dstOff;dstOff=swap;}for(var k=tid;k<NUM_BINS;k=k+WG){let k1=k%RADIX2_N;var sum_re=0.0;var sum_im=0.0;let tail_step=dft_precise_twiddle(k,N,SIGN);var tail_twiddle=vec2<f32>(1.0,SIGN*0.0);for(var n2=0u;n2<TAIL_RADIX;n2=n2+1u){
{% if REANCHOR_TAIL %}if(n2%TAIL_ANCHOR==0u){tail_twiddle=dft_precise_twiddle(k*n2,N,SIGN);}
{% endif %}let v_index=srcOff+n2*RADIX2_N+k1;let vr=shRe[v_index];let vi=shIm[v_index];sum_re=sum_re+vr*tail_twiddle.x-vi*tail_twiddle.y;sum_im=sum_im+vr*tail_twiddle.y+vi*tail_twiddle.x;tail_twiddle=complex_mul(tail_twiddle,tail_step);}shRe[dstOff+k]=sum_re*sum_re+sum_im*sum_im;}workgroupBarrier();for(var m=tid;m<params.nMels;m=m+WG){var weightOffset=0u;for(var q=0u;q<m;q=q+1u){weightOffset=weightOffset+mel_spans[2u*q+1u];}let startBin=mel_spans[2u*m];let bandLength=mel_spans[2u*m+1u];var acc=0.0;for(var o=0u;o<bandLength;o=o+1u){acc=acc+mel_weights[weightOffset+o]*shRe[dstOff+startBin+o];}mel_out[m*params.numFrames+frame]={{ "" if clampToPeak else "(" }}log(max(acc,params.logFloor))*0.43429448190325176{{ "" if clampToPeak else " + " ~ postAdd ~ ") * " ~ postScale }};} }`],["logmel-global-max.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const WG:u32={{ tunables.WORKGROUP_SIZE }}u;const NPART:u32={{ tunables.REDUCE_PARTIALS }}u;const F32_LOWEST:f32=-3.4028234663852886e38;var<workgroup>shMax:array<f32,WG>;
{% include "ops/_shared/reduce-tree.e93bd4c37a.wgsl.jinja" %}{{ wgsl_tree_reduce_f32("reduce_max", "max", "shMax", "WG", false) }}
@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})
{% if firstPass %}fn main(@builtin(workgroup_id)wgid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){
{% else %}fn main(@builtin(local_invocation_id)lid:vec3<u32>){
{% endif %}var value=F32_LOWEST;
{% if firstPass %}for(var i=wgid.x*WG+lid.x;i<params.count;i=i+NPART*WG){value=max(value,mel_in[i]);}
{% else %}for(var i=lid.x;i<NPART;i=i+WG){value=max(value,partials[i]);}
{% endif %}let m=reduce_max(value,lid.x);if(lid.x==0u){
{% if firstPass %}partials[wgid.x]=m;
{% else %}gmax[0]=m;
{% endif %}} }`],["logmel-normalize.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const WG:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WG") }} let floorValue=gmax[0]-params.clampDelta;mel_out[i]=(max(mel_out[i],floorValue)+params.postAdd)*params.postScale;}`],["logmel-dft-fallback.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/dft-precise-twiddle.16fc24b25e.wgsl.jinja" %}const N:u32={{ fftLength }}u;const WG:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WG", "index") }} let mel=index/params.numFrames;let frame=index-mel*params.numFrames;let frameStart=i32(frame*params.hop){{ " - i32(N / 2u)" if centerFrames else "" }};var weightOffset=0u;for(var q=0u;q<mel;q=q+1u){weightOffset=weightOffset+mel_spans[2u*q+1u];}let startBin=mel_spans[2u*mel];let bandLength=mel_spans[2u*mel+1u];var acc=0.0;for(var o=0u;o<bandLength;o=o+1u){let bin=startBin+o;var re=0.0;var im=0.0;var phase=0u;let distanceToWrap=N-(bin%N);for(var n=0u;n<N;n=n+1u){let position=frameStart+i32(n);let mirrored=u32(select(position,-position,position<0));var sample=0.0;if(mirrored<params.audioLength){sample=audio[mirrored]*window[n];}let twiddle=dft_precise_twiddle(phase,N,-1.0);re=re+sample*twiddle.x;im=im+sample*twiddle.y;phase=select(phase+(bin%N),phase-distanceToWrap,phase>=distanceToWrap);}acc=acc+mel_weights[weightOffset+o]*(re*re+im*im);}mel_out[mel*params.numFrames+frame]={{ "" if clampToPeak else "(" }}log(max(acc,params.logFloor))*0.43429448190325176{{ "" if clampToPeak else " + " ~ postAdd ~ ") * " ~ postScale }};}`]],assumes:[["ops/_shared/fft-stockham-shared.wgsl.jinja",{centerFrames:!1,clampToPeak:!1}],["logmel-dft-fallback.wgsl.jinja",{centerFrames:!1,clampToPeak:!1}]]}],["com.xenova.RmsNormResidualAdd",{manifest:{domain:"com.xenova",name:"RmsNormResidualAdd",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X",rank:2},wT:{onnx:"W",dtype:"W",rank:1},yT:{onnx:"Y",dtype:"Y",rank:2},w2T:{onnx:"W2",dtype:"W",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:2,shape:"shapes.yT"},normed2T:{onnx:"N2",dtype:"Y",rank:2,optional:!0,shape:"shapes.yT"}},args:{rows:{kind:"u32"},dim:{kind:"u32"},eps:{kind:"f32"},eps2:{kind:"f32",required:!1},weightBaseOffset:{kind:"u32",required:!1},weight2BaseOffset:{kind:"u32",required:!1}},typeConstraints:{X:["float32"],W:["float32","uint32"],Y:["float32"]},tunables:{WG_CAP:{default:1024}},derive:{subgroupsWide32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32',normWgCap:"tunables.WG_CAP if subgroupsWide32 else min(128, tunables.WG_CAP)",normWg:"min(normWgCap, max(32, pow2ceil(args.dim / 8)))",hasSubgroupId:'device.wgslLanguageFeatures.has("subgroup_id")',SCALE_VEC4_LEN:"(args.dim + (args.weightBaseOffset if args.weightBaseOffset else 0)) / 4",modeSpec:'"rms"',vec4:!0,scalar:"dtypes.Y",hidden:"args.dim",hiddenVec:"args.dim / 4",wg:"normWg",epsilon:"args.eps",vecType:'"vec4<" ~ dtypes.Y ~ ">"',combineSubgroups:"subgroupsWide32 and hasSubgroupId",rmsResidualAdd:!0},when:["ranks.wT == 1",'dim(shapes.wT, 0) == args.dim if args.weightBaseOffset is undefined else (args.weightBaseOffset % 4 == 0 and dim(shapes.wT, 0) * (2 if tensorDtypes.wT == "uint32" else 1) >= args.weightBaseOffset + args.dim)',"args.rows >= 0","args.dim > 0","args.dim % 4 == 0","numel(shapes.xT) >= args.rows * args.dim","numel(shapes.yT) >= args.rows * args.dim",'tensorDtypes.xT == "float32"','tensorDtypes.wT == "float32" or (tensorDtypes.wT == "uint32" and args.weightBaseOffset is defined)','tensorDtypes.yT == "float32"'],bindings:{x:{arg:"xT",elementType:"vec4<f32>"},y:{arg:"yT",elementType:"vec4<f32>"},params:{struct:[{name:"rows",type:"u32"},{name:"rowStride",type:"u32",value:"max(1, min(args.rows, device.limits.maxComputeWorkgroupsPerDimension))"}]},normed2:{arg:"normed2T",elementType:"vec4<f32>"},scale_vec4:{arg:"wT",name:"scale",elementType:"$arenaScaleVec4",length:"$SCALE_VEC4_LEN"}},variants:[{id:"weighted_vec4_arena",when:["not present.w2T","not present.normed2T","args.eps2 is undefined","args.weightBaseOffset is defined"],derive:{rmsScaleBaseVec4:"args.weightBaseOffset / 4",rmsScaleBf16:'tensorDtypes.wT == "uint32"',arenaScaleVec4:'"vec2<u32>" if tensorDtypes.wT == "uint32" else "vec4<f32>"'},passes:[{id:"main",name:"RmsNormResidualAdd",shader:"ops/_shared/norm-row-stats.wgsl.jinja",bindings:["x","scale_vec4","y","params"],dispatch:{workgroups:"args.rows"},subgroupCollectivesWidth:"portable"}]},{id:"weighted_vec4_arena_chained",when:["args.weightBaseOffset is defined","args.weight2BaseOffset is defined","present.w2T","present.normed2T","args.eps2 is defined","ranks.w2T == 1","tensorDtypes.w2T == tensorDtypes.wT","args.weight2BaseOffset % 4 == 0",'dim(shapes.w2T, 0) * (2 if tensorDtypes.w2T == "uint32" else 1) >= args.weight2BaseOffset + args.dim',"numel(shapes.normed2T) >= args.rows * args.dim",'tensorDtypes.normed2T == "float32"'],derive:{rmsScaleBaseVec4:"args.weightBaseOffset / 4",rmsScale2BaseVec4:"args.weight2BaseOffset / 4",rmsScaleBf16:'tensorDtypes.wT == "uint32"',arenaScaleVec4:'"vec2<u32>" if tensorDtypes.wT == "uint32" else "vec4<f32>"',SCALE2_VEC4_LEN:"(args.dim + args.weight2BaseOffset) / 4"},passes:[{id:"main",name:"RmsNormResidualAddChained",shader:"ops/_shared/norm-row-stats.wgsl.jinja",derive:{epsilon2:"args.eps2",rmsChainNorm:!0},bindings:["x","scale_vec4",{arg:"w2T",name:"scale2",elementType:"$arenaScaleVec4",length:"$SCALE2_VEC4_LEN"},"y","normed2","params"],dispatch:{workgroups:"args.rows"},subgroupCollectivesWidth:"portable"}]}]},assets:[["ops/_shared/norm-row-stats.wgsl.jinja",`{% set combineSubgroups = combineSubgroups and not (scalar == "f16" and compensateHalfStats is defined and compensateHalfStats and (device.adapterInfo.subgroupMinSize | default(0)) > 32) %}{% set packedF32 = "vec4<f32>" %}{% set rmsChainNorm = rmsChainNorm if rmsChainNorm is defined else false %}{% set epsilon2 = epsilon2 | default("0.0") %}{% set rmsScaleVec = "vec4<f32>(scale[i])" %}{% set rmsScaleVec = ("unpack_bf16_x4" if rmsScaleBf16 else "vec4<f32>") ~ "(scale[" ~ rmsScaleBaseVec4 ~ "u + i])" %}{% set rmsScale2Vec = (("unpack_bf16_x4" if rmsScaleBf16 else "vec4<f32>") ~ "(scale2[" ~ rmsScale2BaseVec4 ~ "u + i])") if rmsScale2BaseVec4 is defined else "vec4<f32>(scale2[i])" %}{% set reduceThreadParameters = ", sg_id: u32, num_sg: u32, sg_size: u32"
    if combineSubgroups else ", tid: u32" %}{% set reduceThreadArguments = ", sg_id, num_sg, sg_size"
    if combineSubgroups else ", tid" %}{% if combineSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }} const HIDDEN:u32={{ hidden }}u;const HIDDEN_V:u32={{ hiddenVec }}u;const WG:u32={{ wg }}u;const EPSILON:f32={{ epsilon }};
{% if rmsChainNorm %}const EPSILON2:f32={{ epsilon2 }};
{% endif %}{% if combineSubgroups %}var<workgroup>sg_partials:array<f32,WG>;fn reduce_scalar(value:f32{{ reduceThreadParameters }})->f32{let s=subgroupAdd(value);let fold_count=max(num_sg,(WG+sg_size-1u)>>firstTrailingBit(sg_size));if(fold_count==1u){return s;}if(subgroupElect()){sg_partials[sg_id]=s;}workgroupBarrier();var total=0.0;for(var i=0u;i<fold_count;i++){total+=sg_partials[i];}workgroupBarrier();return total;}
{% else %}var<workgroup>tr0:array<f32,WG>;fn reduce_scalar(value:f32,tid:u32)->f32{tr0[tid]=value;workgroupBarrier();var stride:u32=WG/2u;loop{if(stride==0u){break;}if(tid<stride){tr0[tid]=tr0[tid]+tr0[tid+stride];}stride=stride/2u;workgroupBarrier();}let reduced=tr0[0];workgroupBarrier();return reduced;}
{% endif %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg_id:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if combineSubgroups %},
@builtin(subgroup_id)sg_id:u32,@builtin(num_subgroups)num_sg:u32,
  @builtin(subgroup_size) sg_size: u32{% endif %}){let row=wg_id.x+wg_id.y*params.rowStride;if(row>=params.rows){return;}let tid=lid.x;let base=row*HIDDEN_V;var acc=0.0;for(var i=tid;i<HIDDEN_V;i=i+WG){let v={{ packedF32 }}(x[base+i]);acc=acc+dot(v,v);}let total=reduce_scalar(acc{{ reduceThreadArguments }});let inv=inverseSqrt(total/f32(HIDDEN)+EPSILON);
{% if rmsChainNorm %}var acc2=0.0;
{% endif %}for(var i=tid;i<HIDDEN_V;i=i+WG){let idx=base+i;let v={{ packedF32 }}(x[idx]);
{% if rmsChainNorm %}let hv=y[idx]+fma(v*inv,{{ rmsScaleVec }},vec4<f32>(0.0));y[idx]=hv;acc2=acc2+dot(hv,hv);
{% else %}y[idx]=y[idx]+fma(v*inv,{{ rmsScaleVec }},vec4<f32>(0.0));
{% endif %}}
{% if rmsChainNorm %}workgroupBarrier();let total2=reduce_scalar(acc2{{ reduceThreadArguments }});let inv2=inverseSqrt(total2/f32(HIDDEN)+EPSILON2);for(var i=tid;i<HIDDEN_V;i=i+WG){let idx=base+i;let hv=vec4<f32>(y[idx]);normed2[idx]={{ vecType }}(hv*inv2*{{ rmsScale2Vec }});}
{% endif %}}`]],assumes:[["ops/_shared/norm-row-stats.wgsl.jinja",{scalar:"f32",vecType:"vec4<f32>",rmsScaleBf16:!1}]]}],["com.xenova.SampleRows",{manifest:{domain:"com.xenova",name:"SampleRows",sinceVersion:1,inputs:{logitsT:{onnx:"Logits",dtype:"float32",rank:2},lastSeenT:{onnx:"LastSeen",dtype:"float32",rank:2},controlT:{onnx:"Control",dtype:"float32",rank:1},drawT:{onnx:"Draw",dtype:"uint32",rank:1}},outputs:{idsT:{onnx:"Ids",dtype:"uint32",rank:1,shape:"[dim(shapes.logitsT, 0)]"}},tunables:{WORKGROUP_SIZE:{default:256}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",rowCount:"dim(shapes.logitsT, 0)",rowLength:"dim(shapes.logitsT, 1)",workgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)"},when:["ranks.logitsT == 2","ranks.lastSeenT == 2","dim(shapes.lastSeenT, 0) == rowCount","dim(shapes.lastSeenT, 1) == rowLength","numel(shapes.controlT) >= 3","numel(shapes.drawT) >= 2","dim(shapes.idsT, 0) >= rowCount","rowCount > 0","rowLength > 0","rowCount <= device.limits.maxComputeWorkgroupsPerDimension","pow2ceil(workgroupSize) == workgroupSize"],variants:[{id:"row_workgroups",priority:10,passes:[{id:"main",name:"SampleRows",shader:"sample-rows.wgsl.jinja",bindings:[{arg:"logitsT",name:"logits"},{arg:"lastSeenT",name:"last_seen"},{arg:"controlT",name:"control"},{arg:"drawT",name:"draw"},{arg:"idsT",name:"ids"}],dispatch:{x:"rowCount"}}]}]},assets:[["sample-rows.wgsl.jinja",`{% set modeSpec = "max" %}{% set selectLastIndex = false %}{% include "ops/_shared/float-is-nan.73deb0ed6d.wgsl.jinja" %}{% include "ops/_shared/reduce-arg-tie-rule.55e239089f.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const COLS:u32={{ rowLength }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const SENTINEL_IDX:u32=0xffffffffu;var<workgroup>best_value:array<f32,{{ workgroupSize }}>;var<workgroup>best_id:array<u32,{{ workgroupSize }}>;
{% include "ops/_shared/pcg-hash.2e09754c20.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let row=wid.x;let lid=lid3.x;let window_start=control[0];let penalty=control[1];let temperature=control[2];let stream=pcg_hash(pcg_hash(draw[1])^draw[0]);var top=0.0;var top_at=SENTINEL_IDX;for(var col=lid;col<COLS;col=col+WORKGROUP_SIZE){let index=row*COLS+col;var score=logits[index];if(is_nan_f32(score)){continue;}if(last_seen[index]>=window_start){score=select(score/penalty,score*penalty,score<0.0);}if(temperature>0.0){let u=(f32(pcg_hash(stream^index)>>8u)+0.5)*(1.0/16777216.0);score=score/temperature-log(-log(u));}if(top_at==SENTINEL_IDX||score>top){top=score;top_at=col;} }best_value[lid]=top;best_id[lid]=top_at;workgroupBarrier();for(var stride=WORKGROUP_SIZE/2u;stride>0u;stride=stride/2u){if(lid<stride&&{{ arg_merge_wins("best_value[lid + stride]", "best_id[lid + stride]", "best_value[lid]", "best_id[lid]") }}){best_value[lid]=best_value[lid+stride];best_id[lid]=best_id[lid+stride];}workgroupBarrier();}if(lid==0u){ids[row]=select(best_id[0],0u,best_id[0]==SENTINEL_IDX);} }`]]}],["com.xenova.Silu",{manifest:{domain:"com.xenova",name:"Silu",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"T"}},outputs:{xT:{onnx:"X",dtype:"T",rank:"ranks.xT",shape:"shapes.xT"}},args:{count:{kind:"u32",onnx:"kernel.count"}},typeConstraints:{T:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:64}},derive:{baseContract:"args.count >= 0 and numel(shapes.xT) >= args.count and f16Ok(dtypes.T)"},when:["baseContract"],bindings:{x:{arg:"xT",elementType:"$storageScalar"}},variants:[{id:"main",derive:{vectorized:"args.count % 4 == 0",width:"4 if vectorized else 1",storageScalar:'"vec4<" ~ dtypes.T ~ ">" if vectorized else dtypes.T'},passes:[{id:"main",name:"silu",shader:"silu-elementwise.wgsl.jinja",bindings:["x",{name:"params",struct:[{name:"count",type:"u32",value:"args.count / width"},{name:"dispatchedX",type:"u32",value:"min(ceil((args.count / width) / tunables.WORKGROUP_SIZE), device.limits.maxComputeWorkgroupsPerDimension)"}]}],dispatch:{threads:"args.count / width",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["silu-elementwise.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const WORKGROUP_SIZE:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size({{ tunables.WORKGROUP_SIZE }},1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let i=(wid.x+wid.y*params.dispatchedX)*WORKGROUP_SIZE+lid.x;if(i>=params.count){return;}
{% if vectorized %}let value=vec4<f32>(x[i]);let result={{ storageScalar }}(value/(vec4<f32>(1.0)+exp(-value)));
{% else %}let value=f32(x[i]);let result={{ storageScalar }}(value/(1.0+exp(-value)));
{% endif %}x[i]=result;}`]]}],["com.xenova.StridedCopy",{manifest:{domain:"com.xenova",name:"StridedCopy",sinceVersion:1,inputs:{srcT:{onnx:"Src",dtype:"S"}},outputs:{dstT:{onnx:"Dst",dtype:"D"}},args:{rows:{kind:"u32"},srcStride:{kind:"u32"},srcStart:{kind:"u32",required:!1},dstStride:{kind:"u32"},dstStart:{kind:"u32",required:!1},copyCols:{kind:"u32"}},typeConstraints:{S:["float32","float16","uint32","int32"],D:["float32","float16","uint32","int32"]},derive:{colChunk:4096,workgroupSize:64},bindings:{s:{arg:"srcT",elementType:"$sourceElement"},d:{arg:"dstT",elementType:"$destElement"},p:{struct:[{name:"rows",type:"u32"},{name:"copyCols",type:"u32"},{name:"srcStride",type:"u32"},{name:"srcStart",type:"u32",default:0},{name:"dstStride",type:"u32"},{name:"dstStart",type:"u32",default:0}]}},variants:[{id:"scalar",priority:0,when:["f16Ok(dtypes.S) and f16Ok(dtypes.D)"],derive:{vectorized:!1,directScalarCopy:"dtypes.S == dtypes.D",sourceElement:"dtypes.S",destElement:"dtypes.D",destScalar:"dtypes.D",storageItemWidth:1},passes:[{id:"main",name:"StridedCopy",shader:"strided-copy.wgsl.jinja",bindings:["s","d","p"],dispatch:{workgroups:"args.rows * ceil(args.copyCols / (colChunk * storageItemWidth))"}}]}]},assets:[["strided-copy.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const COL_CHUNK:u32={{ colChunk }}u;const STORAGE_ITEM_WIDTH:u32={{ storageItemWidth }}u;@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let copyCols=p.copyCols/STORAGE_ITEM_WIDTH;let colTiles=max((copyCols+COL_CHUNK-1u)/COL_CHUNK,1u);let flat=wg.x+wg.y*{{ DISPATCH_FOLD_WIDTH }}u;if(flat>=p.rows*colTiles){return;}let r=flat/colTiles;let srcRow=(r*p.srcStride+p.srcStart)/STORAGE_ITEM_WIDTH;let dstRow=(r*p.dstStride+p.dstStart)/STORAGE_ITEM_WIDTH;let col_base=(flat-r*colTiles)*COL_CHUNK;let col_end=min(col_base+COL_CHUNK,copyCols);var i:u32=col_base+lid.x;loop{if(i>=col_end){break;}d[dstRow+i]=s[srcRow+i];i=i+{{ workgroupSize }}u;} }"]],assumes:[["strided-copy.wgsl.jinja",{directScalarCopy:!0,destScalar:"f32"}]]}],["com.xenova.SupertonicDense",{manifest:{domain:"com.xenova",name:"SupertonicDense",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"T",rank:2,optional:!0},wT:{onnx:"W",dtype:"W",rank:2},bT:{onnx:"B",dtype:"T",rank:1},rT:{onnx:"R",dtype:"T",rank:2,optional:!0},outputScaleT:{onnx:"OutputScale",dtype:"T",rank:1,optional:!0},maskT:{onnx:"Mask",dtype:"T",rank:1,optional:!0},vecT:{onnx:"Vec",dtype:"T",rank:1,optional:!0},lnScaleT:{onnx:"LnScale",dtype:"T",rank:1,optional:!0},lnBiasT:{onnx:"LnBias",dtype:"T",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"T",rank:"3 if effectiveOutputGroups > 1 else 2",shape:"[effectiveOutputGroups, args.M, args.outFeatures / effectiveOutputGroups] if effectiveOutputGroups > 1 else [args.M, args.outFeatures]"}},args:{M:{kind:"u32"},inFeatures:{kind:"u32"},outFeatures:{kind:"u32"},outputGroups:{kind:"u32",required:!1},inputGroups:{kind:"u32",required:!1},gelu:{kind:"u32",oneOf:[0,1]},splitK:{kind:"u32",required:!1},eps:{kind:"f32",required:!1}},typeConstraints:{T:["float32"],W:["float32","float16"]},tunables:{SCALAR_WORKGROUP_SIZE:{default:64},REDUCE_WORKGROUP_SIZE:{default:256},TILED_MIN_WORKGROUPS:{default:16},DIRECT_LOAD_MAX_TILES:{default:320},REGISTER_TILE_MIN_WORKGROUPS:{default:128},REGISTER_TILE_MIN_OUT_FEATURES:{default:256},GEMV_MAX_M:{default:7},GEMV_K_PER_LANE:{default:16},GEMV_WORKGROUP_MAX:{default:128},SUBGROUP_GEMV_MAX_OUTPUTS_PER_WORKGROUP:{default:8},SUBGROUP_GEMV_MAX_K_PARTITIONS:{default:4},SUBGROUP_GEMV_TARGET_K_PER_PARTITION:{default:768},SUBGROUP_GEMV_K_UNROLL:{default:4},SUBGROUP_GEMV_MULTIROW_MAX_M:{default:40},SUBGROUP_GEMV_MULTIROW_MAX_OUTPUTS_PER_WORKGROUP:{default:8},SUBGROUP_GEMV_MULTIROW_K_UNROLL:{default:4},SUBGROUP_GEMV_MULTIROW_REGISTER_BUDGET:{default:64},MMA_LARGE_TILE_M:{default:64},MMA_LARGE_TILE_N:{default:64},MMA_LARGE_TILE_K:{default:32},MMA_LARGE_MIN_M:{default:512},AUTO_SPLIT_K_MIN_M:{default:8},AUTO_SPLIT_K2_MIN_K:{default:512},AUTO_SPLIT_K4_MIN_K:{default:1024},AUTO_SPLIT_K8_MIN_K:{default:2048},AUTO_SPLIT_K8_MAX_BASE_WORKGROUPS:{default:32},AUTO_SPLIT_K_MAX_BASE_WORKGROUPS:{default:512},AUTO_SPLIT_K_TARGET_WORKGROUPS:{default:192},AUTO_SPLIT_K_MIN_K_PER_SLICE:{default:48}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",tiledGridY:"ceil(args.M / 32)",scalarWorkgroupSize:"min(tunables.SCALAR_WORKGROUP_SIZE, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",mmaWorkgroupSize:128,splitKStageF32Tail:"args.M % 16 != 0",mmaResourcesFit:"wave32Effective and mmaWorkgroupSize <= deviceWorkgroupCap and 13324 <= device.limits.maxComputeWorkgroupStorageSize",mmaDispatchFits:"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and tiledGridY <= device.limits.maxComputeWorkgroupsPerDimension",W_LEN:"args.outFeatures * args.inFeatures",inFeatures:"args.inFeatures",outFeatures:"args.outFeatures",wF16:'tensorDtypes.wT == "float16"',biasValid:"dim(shapes.bT, 0) == args.outFeatures",denseOutputValid:"present.yT and numel(shapes.yT) >= args.M * args.outFeatures",residualValid:"present.rT and numel(shapes.rT) >= args.M * args.outFeatures",maskValid:"present.maskT and dim(shapes.maskT, 0) >= args.M",rowVectorValid:"present.vecT and dim(shapes.vecT, 0) == args.outFeatures",reduceWorkgroupSize:"min(tunables.REDUCE_WORKGROUP_SIZE, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",hasMask:"present.maskT",hasRowVec:"present.vecT",OUT_V4_LEN:"args.outFeatures / 4",subgroupMatrix8F16Available:'subgroupMatrixConfigAvailable("f16", "f32", 8, 8, 8)',subgroupMatrix8F32Available:'subgroupMatrixConfigAvailable("f32", "f32", 8, 8, 8)',subgroupMatrixTopologyBytes:"3 * 16",denseFlops:"2 * args.M * args.inFeatures * args.outFeatures",denseInputsValid:"present.xT and numel(shapes.xT) >= args.M * args.inFeatures and numel(shapes.wT) >= args.outFeatures * args.inFeatures",splitKOverridePresent:'has(args, "splitK")',effectiveInputGroups:'args.inputGroups if has(args, "inputGroups") else 1',rowMajorInputMode:"effectiveInputGroups == 1",groupedInputMode:"effectiveInputGroups > 1 and args.inFeatures % effectiveInputGroups == 0",inputGroupWidth:"args.inFeatures / effectiveInputGroups if effectiveInputGroups > 0 else 0",effectiveOutputGroups:'args.outputGroups if has(args, "outputGroups") else 1',rowMajorOutputMode:"effectiveOutputGroups == 1",groupedOutputMode:"effectiveOutputGroups > 1 and args.outFeatures % effectiveOutputGroups == 0",outputScaleValid:"present.outputScaleT and dim(shapes.outputScaleT, 0) == args.outFeatures",plainMode:"rowMajorInputMode and rowMajorOutputMode and args.gelu == 0 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",geluMode:"rowMajorInputMode and rowMajorOutputMode and args.gelu == 1 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",groupedInputPlainMode:"groupedInputMode and not splitKOverridePresent and rowMajorOutputMode and args.gelu == 0 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",groupedInputGeluMode:"groupedInputMode and not splitKOverridePresent and rowMajorOutputMode and args.gelu == 1 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",residualMode:"rowMajorInputMode and rowMajorOutputMode and args.gelu == 0 and residualValid and not present.outputScaleT and not present.maskT and not present.vecT",residualMaskMode:"rowMajorInputMode and rowMajorOutputMode and args.gelu == 0 and residualValid and not present.outputScaleT and maskValid and not present.vecT",residualVecMaskMode:"rowMajorInputMode and rowMajorOutputMode and args.gelu == 0 and residualValid and not present.outputScaleT and maskValid and rowVectorValid",scaledResidualMode:"(rowMajorInputMode or (groupedInputMode and not splitKOverridePresent)) and rowMajorOutputMode and args.gelu == 0 and residualValid and outputScaleValid and not present.maskT and not present.vecT and not present.lnScaleT",groupedTypesValid:'present.xT and present.yT and tensorDtypes.xT == "float32" and f16Ok(tensorDtypes.wT) and tensorDtypes.yT == "float32"',tiledGridX:"ceil(args.outFeatures / 32)",sgmatOperandFeedMode:"tunables.SGMAT_OPERAND_FEED_MODE if tunables.SGMAT_OPERAND_FEED_MODE is defined else 1",sgmatOperandFeedModeValid:"sgmatOperandFeedMode == 0 or sgmatOperandFeedMode == 1",regularSgmatMode0DirectLoad:'rowMajorInputMode and tensorDtypes.wT != "float16" and ceil(args.M / 32) * ceil(args.outFeatures / 64) < tunables.DIRECT_LOAD_MAX_TILES',regularSgmatOperandFeedMode:"1 if sgmatOperandFeedMode == 1 and args.inFeatures >= 128 and not regularSgmatMode0DirectLoad else 0",regularSgmatMode1Geometry:"args.M >= 8 and args.inFeatures % 32 == 0 and (rowMajorInputMode or (groupedInputMode and inputGroupWidth % 32 == 0))",regularSgmatStorageBytes:"(16384 if regularSgmatOperandFeedMode == 1 or not regularSgmatMode0DirectLoad else 4096) + subgroupMatrixTopologyBytes",regularSgmatResourcesFit:"sgmatOperandFeedModeValid and (regularSgmatOperandFeedMode == 0 or regularSgmatMode1Geometry) and wave32Effective and mmaWorkgroupSize <= deviceWorkgroupCap and regularSgmatStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",largeTileF32:'present.xT and present.yT and tensorDtypes.xT == "float32" and tensorDtypes.wT == "float32" and tensorDtypes.yT == "float32"',largeTileGeometry:"args.M >= tunables.MMA_LARGE_MIN_M and ceilDiv(args.outFeatures, tunables.MMA_LARGE_TILE_N) >= 8 and args.inFeatures > 0 and args.outFeatures > 0",largeTileRowsValid:"(tunables.MMA_LARGE_TILE_M == 16 or tunables.MMA_LARGE_TILE_M == 32 or tunables.MMA_LARGE_TILE_M == 64) and tunables.MMA_LARGE_TILE_N == 64 and tunables.MMA_LARGE_TILE_K == 32",largeTileWorkgroupSize:"tunables.MMA_LARGE_TILE_M * 4",largeSgmatOperandFeedMode:"1 if sgmatOperandFeedMode == 1 and args.inFeatures >= 128 and args.M >= 8 and args.inFeatures % tunables.MMA_LARGE_TILE_K == 0 else 0",largeTileStorageBytes:"((2 * tunables.MMA_LARGE_TILE_N if largeSgmatOperandFeedMode == 1 else tunables.MMA_LARGE_TILE_M + tunables.MMA_LARGE_TILE_N) * tunables.MMA_LARGE_TILE_K * 4) + subgroupMatrixTopologyBytes",largeTileAValuesPerThread:"floor(tunables.MMA_LARGE_TILE_M * tunables.MMA_LARGE_TILE_K / largeTileWorkgroupSize)",largeTileBValuesPerThread:"floor(tunables.MMA_LARGE_TILE_N * tunables.MMA_LARGE_TILE_K / largeTileWorkgroupSize)",largeTileConfigValid:"sgmatOperandFeedModeValid and largeTileRowsValid and largeTileAValuesPerThread >= 1 and largeTileBValuesPerThread >= 1 and tunables.MMA_LARGE_TILE_K % largeTileAValuesPerThread == 0 and tunables.MMA_LARGE_TILE_K % largeTileBValuesPerThread == 0 and (largeSgmatOperandFeedMode == 0 or (args.M >= 8 and args.inFeatures % tunables.MMA_LARGE_TILE_K == 0))",largeTileResourcesFit:"wave32Effective and largeTileConfigValid and largeTileWorkgroupSize <= deviceWorkgroupCap and largeTileStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",largeTileDispatchFits:"ceilDiv(args.outFeatures, tunables.MMA_LARGE_TILE_N) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv(args.M, tunables.MMA_LARGE_TILE_M) <= device.limits.maxComputeWorkgroupsPerDimension",tiledWorkgroupDim:16,tiledResourcesFit:"tiledWorkgroupDim <= device.limits.maxComputeWorkgroupSizeX and tiledWorkgroupDim <= device.limits.maxComputeWorkgroupSizeY and tiledWorkgroupDim * tiledWorkgroupDim <= device.limits.maxComputeInvocationsPerWorkgroup and 4096 <= device.limits.maxComputeWorkgroupStorageSize",registerTiledResourcesFit:"tiledResourcesFit and 8192 <= device.limits.maxComputeWorkgroupStorageSize",tiledOccupancyPreferred:"args.M >= 32 or tiledGridX * tiledGridY >= tunables.TILED_MIN_WORKGROUPS",tiledDispatchFits:"tiledGridX <= device.limits.maxComputeWorkgroupsPerDimension and tiledGridY <= device.limits.maxComputeWorkgroupsPerDimension",registerTiledDispatchFits:"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and ceil(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension",autoSplitKTypes:'present.xT and present.yT and tensorDtypes.xT == "float32" and (tensorDtypes.wT == "float32" or tensorDtypes.wT == "float16") and tensorDtypes.yT == "float32"',autoSplitKFeatures:'device.features.has("chromium-experimental-subgroup-matrix") and device.features.has("subgroups") and (tensorDtypes.wT == "float32" or device.features.has("shader-f16"))',autoSplitKMode:"rowMajorOutputMode and (plainMode or geluMode or residualMode or residualMaskMode or residualVecMaskMode or (rowMajorInputMode and scaledResidualMode))",autoSplitKBaseWorkgroups:"ceilDiv(args.M, 32) * ceilDiv(args.outFeatures, 64)",autoSplitKUnit:'32 if tensorDtypes.wT == "float16" else 16',autoSplitKDepthCandidate:"8 if autoSplitKBaseWorkgroups <= tunables.AUTO_SPLIT_K8_MAX_BASE_WORKGROUPS and args.inFeatures >= tunables.AUTO_SPLIT_K8_MIN_K and args.inFeatures % (8 * autoSplitKUnit) == 0 else (4 if args.inFeatures >= tunables.AUTO_SPLIT_K4_MIN_K and args.inFeatures % (4 * autoSplitKUnit) == 0 else (2 if args.inFeatures >= tunables.AUTO_SPLIT_K2_MIN_K and args.inFeatures % (2 * autoSplitKUnit) == 0 else 1))",autoSplitKGridCandidate:"8 if autoSplitKBaseWorkgroups * 8 <= tunables.AUTO_SPLIT_K_TARGET_WORKGROUPS and args.inFeatures >= 8 * tunables.AUTO_SPLIT_K_MIN_K_PER_SLICE and args.inFeatures % (8 * autoSplitKUnit) == 0 else (4 if autoSplitKBaseWorkgroups * 4 <= tunables.AUTO_SPLIT_K_TARGET_WORKGROUPS and args.inFeatures >= 4 * tunables.AUTO_SPLIT_K_MIN_K_PER_SLICE and args.inFeatures % (4 * autoSplitKUnit) == 0 else (2 if autoSplitKBaseWorkgroups * 2 <= tunables.AUTO_SPLIT_K_TARGET_WORKGROUPS and args.inFeatures >= 2 * tunables.AUTO_SPLIT_K_MIN_K_PER_SLICE and args.inFeatures % (2 * autoSplitKUnit) == 0 else 1))",autoSplitKCandidate:"max(autoSplitKDepthCandidate, autoSplitKGridCandidate)",autoSplitKScratchBytes:"autoSplitKCandidate * tiledGridY * 32 * args.outFeatures * 4",autoSplitKSafe:"not splitKOverridePresent and autoSplitKMode and autoSplitKTypes and autoSplitKFeatures and args.M >= tunables.AUTO_SPLIT_K_MIN_M and args.outFeatures % 64 == 0 and autoSplitKBaseWorkgroups <= tunables.AUTO_SPLIT_K_MAX_BASE_WORKGROUPS and autoSplitKCandidate >= 2 and mmaResourcesFit and mmaDispatchFits and autoSplitKCandidate <= device.limits.maxComputeWorkgroupsPerDimension and autoSplitKScratchBytes <= device.limits.maxStorageBufferBindingSize and autoSplitKScratchBytes <= device.limits.maxBufferSize",effectiveSplitK:"args.splitK if splitKOverridePresent else (autoSplitKCandidate if autoSplitKSafe else 1)",splitKRequested:"effectiveSplitK >= 2",groupedPlainMode:"rowMajorInputMode and groupedOutputMode and not splitKRequested and args.gelu == 0 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",groupedGeluMode:"rowMajorInputMode and groupedOutputMode and not splitKRequested and args.gelu == 1 and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT and not present.lnScaleT",splitKDispatchFits:"not splitKRequested or (mmaDispatchFits and effectiveSplitK <= device.limits.maxComputeWorkgroupsPerDimension)",noSgSplitKDispatchFits:"not splitKRequested or (tiledDispatchFits and effectiveSplitK <= device.limits.maxComputeWorkgroupsPerDimension)",splitKScratchBytes:"(effectiveSplitK if splitKRequested else 1) * tiledGridY * 32 * args.outFeatures * 4",splitKScratchFits:"splitKScratchBytes <= device.limits.maxStorageBufferBindingSize and splitKScratchBytes <= device.limits.maxBufferSize",f32MmaSplitKGeometry:'splitKRequested and args.M >= 8 and args.inFeatures % (effectiveSplitK * 16) == 0 and args.outFeatures % 64 == 0 and tensorDtypes.wT == "float32"',f16MmaSplitKGeometry:'splitKRequested and args.M >= 8 and args.inFeatures % (effectiveSplitK * 32) == 0 and args.outFeatures % 64 == 0 and tensorDtypes.wT == "float16"',mmaDenseGeometry:"args.M >= 8 and args.inFeatures % 32 == 0 and args.outFeatures % 64 == 0",f16MmaDenseGeometry:'args.M > 0 and args.inFeatures % 32 == 0 and args.outFeatures % 64 == 0 and tensorDtypes.wT == "float16"',densePositiveGeometry:"args.M > 0 and args.inFeatures > 0 and args.outFeatures > 0",smallMGeometry:"args.M >= 1 and args.M <= tunables.GEMV_MAX_M and args.inFeatures > 0 and args.outFeatures > 0",gemvWorkgroupSize:"min(max(pow2ceil(ceilDiv(args.inFeatures, tunables.GEMV_K_PER_LANE)), 32), tunables.GEMV_WORKGROUP_MAX, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",gemvResourcesFit:"gemvWorkgroupSize > 0 and gemvWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize",gemvDispatchFits:"args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension and args.M <= device.limits.maxComputeWorkgroupsPerDimension",subgroupGemvGeometry:"smallMGeometry",subgroupGemvF32:"largeTileF32",fixedSubgroupGeometry:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMinSize >= 4',subgroupGemvSubgroupSize:"device.adapterInfo.subgroupMinSize if fixedSubgroupGeometry else 1",subgroupGemvAvailableSubgroups:"max(1, floor(min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX) / subgroupGemvSubgroupSize))",subgroupGemvSubgroupsPerOutput:"min(tunables.SUBGROUP_GEMV_MAX_K_PARTITIONS, subgroupGemvAvailableSubgroups, pow2ceil(ceilDiv(args.inFeatures, tunables.SUBGROUP_GEMV_TARGET_K_PER_PARTITION)))",subgroupGemvOutputsPerWorkgroup:"max(1, min(tunables.SUBGROUP_GEMV_MAX_OUTPUTS_PER_WORKGROUP, floor(subgroupGemvAvailableSubgroups / subgroupGemvSubgroupsPerOutput)))",subgroupGemvSubgroupsPerWorkgroup:"subgroupGemvOutputsPerWorkgroup * subgroupGemvSubgroupsPerOutput",subgroupGemvWorkgroupSize:"subgroupGemvSubgroupSize * subgroupGemvSubgroupsPerWorkgroup",subgroupGemvStep:"subgroupGemvSubgroupSize * tunables.SUBGROUP_GEMV_K_UNROLL",subgroupGemvPartitionSize:"ceilDiv(args.inFeatures, subgroupGemvSubgroupsPerOutput * subgroupGemvStep) * subgroupGemvStep",subgroupGemvHasKTail:"args.inFeatures % (subgroupGemvSubgroupsPerOutput * subgroupGemvPartitionSize) != 0",multiRowGemvGeometry:"args.M >= 2 and args.M <= tunables.SUBGROUP_GEMV_MULTIROW_MAX_M and args.inFeatures > 0 and args.outFeatures > 0 and not splitKOverridePresent",multiRowGemvOutputsPerWorkgroup:"max(1, min(tunables.SUBGROUP_GEMV_MULTIROW_MAX_OUTPUTS_PER_WORKGROUP, subgroupGemvAvailableSubgroups))",multiRowGemvWorkgroupSize:"subgroupGemvSubgroupSize * multiRowGemvOutputsPerWorkgroup",multiRowGemvKUnroll:"max(1, min(tunables.SUBGROUP_GEMV_MULTIROW_K_UNROLL, floor(tunables.SUBGROUP_GEMV_MULTIROW_REGISTER_BUDGET / args.M)))",multiRowGemvHasKTail:"args.inFeatures % (subgroupGemvSubgroupSize * multiRowGemvKUnroll) != 0",multiRowGemvResourcesFit:"fixedSubgroupGeometry and multiRowGemvKUnroll >= 1 and multiRowGemvKUnroll <= 8 and multiRowGemvWorkgroupSize <= deviceWorkgroupCap and 8 + 4 * multiRowGemvWorkgroupSize <= device.limits.maxComputeWorkgroupStorageSize",multiRowGemvDispatchFits:"ceilDiv(args.outFeatures, multiRowGemvOutputsPerWorkgroup) <= device.limits.maxComputeWorkgroupsPerDimension",subgroupGemvDispatchFits:"ceilDiv(args.outFeatures, subgroupGemvOutputsPerWorkgroup) <= device.limits.maxComputeWorkgroupsPerDimension and args.M <= device.limits.maxComputeWorkgroupsPerDimension",subgroupGemvCommonBuffersFit:"present.xT and present.yT and numel(shapes.xT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.xT) * 4 <= device.limits.maxBufferSize and numel(shapes.wT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.wT) * 4 <= device.limits.maxBufferSize and numel(shapes.bT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.bT) * 4 <= device.limits.maxBufferSize and numel(shapes.yT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.yT) * 4 <= device.limits.maxBufferSize",subgroupGemvResidualBufferFits:"present.rT and numel(shapes.rT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.rT) * 4 <= device.limits.maxBufferSize",subgroupGemvPreNormBuffersFit:"present.lnScaleT and present.lnBiasT and numel(shapes.lnScaleT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.lnScaleT) * 4 <= device.limits.maxBufferSize and numel(shapes.lnBiasT) * 4 <= device.limits.maxStorageBufferBindingSize and numel(shapes.lnBiasT) * 4 <= device.limits.maxBufferSize",registerTilePreferred:"ceil(args.M / 64) * ceil(args.outFeatures / 64) >= tunables.REGISTER_TILE_MIN_WORKGROUPS and args.outFeatures >= tunables.REGISTER_TILE_MIN_OUT_FEATURES",registerTileWidthOk:'device.features.has("subgroups") or not (has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize < 16)',preNormValid:"present.lnScaleT and present.lnBiasT and dim(shapes.lnScaleT, 0) == args.inFeatures and dim(shapes.lnBiasT, 0) == args.inFeatures",preNormMode:"rowMajorOutputMode and preNormValid and not present.rT and not present.outputScaleT and not present.maskT and not present.vecT",epilogueMode:"plainMode or geluMode",groupedInputEpilogueMode:"groupedInputPlainMode or groupedInputGeluMode",preNormEpilogueMode:"preNormMode and (args.gelu == 0 or args.gelu == 1)",groupedEpilogueMode:"groupedPlainMode or groupedGeluMode",largeTileExactM:"args.M % tunables.MMA_LARGE_TILE_M == 0",largeTileExactN:"args.outFeatures % tunables.MMA_LARGE_TILE_N == 0",largeTileExactK:"args.inFeatures % tunables.MMA_LARGE_TILE_K == 0",BIAS_LEN:"args.outFeatures",LN_LEN:"args.inFeatures",geluEpilogue:"args.gelu == 1",hasResidual:"present.rT",wScalar:'"f16" if tensorDtypes.wT == "float16" else "f32"',splitK:"effectiveSplitK",hasGeluPartialsIn:!1,subgroupGemvResourcesFit:"fixedSubgroupGeometry and tunables.SUBGROUP_GEMV_K_UNROLL >= 1 and tunables.SUBGROUP_GEMV_K_UNROLL <= 8 and subgroupGemvWorkgroupSize <= deviceWorkgroupCap and 8 + 4 * subgroupGemvWorkgroupSize + (8 * subgroupGemvSubgroupsPerWorkgroup if preNormEpilogueMode else 0) + (4 * subgroupGemvSubgroupsPerWorkgroup if subgroupGemvSubgroupsPerOutput > 1 else 0) <= device.limits.maxComputeWorkgroupStorageSize"},when:["biasValid","denseInputsValid","denseOutputValid"],bindings:{x:{arg:"xT",elementType:"f32"},params:{struct:[{name:"M",type:"u32"}]},w_w_t:{arg:"wT",name:"w",elementType:"$wScalar",length:"$W_LEN"},bias_f32:{arg:"bT",name:"bias",elementType:"f32",length:"$BIAS_LEN"},y_f32:{arg:"yT",name:"y",elementType:"f32"}},variants:[{id:"scalar",priority:0,when:["(epilogueMode and f16Ok(tensorDtypes.wT)) or (groupedEpilogueMode and groupedTypesValid) or (groupedInputEpilogueMode and groupedTypesValid)","densePositiveGeometry","scalarWorkgroupSize > 0"],derive:{inputGroups:"effectiveInputGroups",outputGroups:"effectiveOutputGroups",workgroupSize:"scalarWorkgroupSize"},passes:[{id:"main",shader:"fused-dense-scalar.wgsl.jinja",bindings:["x","w_w_t","bias_f32","y_f32","params"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"scalarWorkgroupSize"},profile:{flops:"denseFlops",gemmK:"args.inFeatures"}}]},{id:"gemv_subgroup_multirow",priority:137,when:["epilogueMode or groupedEpilogueMode","multiRowGemvGeometry","subgroupGemvF32","subgroupGemvCommonBuffersFit","multiRowGemvResourcesFit","multiRowGemvDispatchFits"],requires:{features:["subgroups"]},derive:{outputGroups:"effectiveOutputGroups",mRows:"args.M",subgroupSize:"subgroupGemvSubgroupSize",outputsPerWorkgroup:"multiRowGemvOutputsPerWorkgroup",workgroupSize:"multiRowGemvWorkgroupSize",kUnroll:"multiRowGemvKUnroll",kTail:"multiRowGemvHasKTail"},passes:[{id:"main",shader:"fused-dense-gemv-subgroup-multirow.wgsl.jinja",bindings:["x","w_w_t","bias_f32","y_f32"],dispatch:{x:"ceilDiv(args.outFeatures, multiRowGemvOutputsPerWorkgroup)"},profile:{flops:"denseFlops",gemmK:"args.inFeatures"}}]}]},assets:[["fused-dense-scalar.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.ba00dfd67d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const IN_F:u32={{ inFeatures }}u;const OUT_F:u32={{ outFeatures }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d("WORKGROUP_SIZE", bound="params.M * OUT_F") }} let m_idx=i/OUT_F;let n_idx=i%OUT_F;let outputIndex=i;var acc=bias[n_idx];let wBase=n_idx*IN_F;for(var k=0u;k<IN_F;k=k+1u){let xValue=x[m_idx*IN_F+k];acc=acc+xValue*w[wBase+k];}y[outputIndex]=acc;}`],["fused-dense-gemv-subgroup-multirow.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const IN_F:u32={{ inFeatures }}u;const OUT_F:u32={{ outFeatures }}u;const SUBGROUP_WIDTH:u32={{ subgroupSize }}u;const OUTPUTS_PER_WORKGROUP:u32={{ outputsPerWorkgroup }}u;const K_STEP:u32=SUBGROUP_WIDTH*{{ kUnroll }}u;
{% set localFoldWorkgroupSize = workgroupSize %}{% set localFoldWidth = subgroupSize %}{% include "ops/_shared/subgroup-local-fold.5eb84b12bf.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(subgroup_size)subgroupSize:u32,@builtin(subgroup_invocation_id)subgroupLane:u32){let activeMask=subgroupBallot(true);let activeCount=countOneBits(activeMask.x)+countOneBits(activeMask.y)+countOneBits(activeMask.z)+countOneBits(activeMask.w);let firstLocal=subgroupBroadcastFirst(lid.x);let nativeTopology=subgroupLocalLayoutAccepted(lid.x,subgroupSize==SUBGROUP_WIDTH&&activeCount==SUBGROUP_WIDTH&&subgroupLane==lid.x%SUBGROUP_WIDTH&&firstLocal==(lid.x/SUBGROUP_WIDTH)*SUBGROUP_WIDTH);let lane=lid.x%SUBGROUP_WIDTH;let subgroup=lid.x/SUBGROUP_WIDTH;let n=wid.x*OUTPUTS_PER_WORKGROUP+subgroup;let valid=n<OUT_F;
{% for m in range(mRows) %}var acc{{ m }}=0.0;
{% endfor %}if(valid){let wBase=n*IN_F;for(var kBase=lane;kBase<IN_F;kBase+=K_STEP){
{% for u in range(kUnroll) %}{let k=kBase+{{ u }}u*SUBGROUP_WIDTH;
{% if kTail %}if(k<IN_F){let wv=w[wBase+k];
{% for m in range(mRows) %}acc{{ m }}+=x[{{ m }}u*IN_F+k]*wv;
{% endfor %}}
{% else %}let wv=w[wBase+k];
{% for m in range(mRows) %}acc{{ m }}+=x[{{ m }}u*IN_F+k]*wv;
{% endfor %}{% endif %}}
{% endfor %}} }
{% for m in range(mRows) %}let total{{ m }}=sumLocalSubgroup(acc{{ m }},lid.x,nativeTopology);
{% endfor %}if(valid&&lane==0u){let b=bias[n];
{% for m in range(mRows) %}{var result=b+total{{ m }};y[{{ m }}u*OUT_F+n]=result;}
{% endfor %}} }`]],assumes:[["fused-dense-scalar.wgsl.jinja",{hasMask:!1,hasRowVec:!1,geluEpilogue:!1,hasResidual:!1,wScalar:"f32",inputGroups:1,outputGroups:1}],["fused-dense-gemv-subgroup-multirow.wgsl.jinja",{geluEpilogue:!1,hasResidual:!1,outputGroups:1}]]}]]);var yl=new Map,D_=new Map;function qm(e){let t=D_.get(e);if(t===void 0)return;if(Array.isArray(t.assets)){let a=new Map(yl);for(let[s,o]of t.assets)a.set(s,o);return{manifest:t.manifest,assets:Cn(a)}}let r=t.assets;if(yl.size===0)return{manifest:t.manifest,assets:r};let n=yl;return{manifest:t.manifest,assets:Object.freeze({readText(a){if(r.has(a))return r.readText(a);let s=n.get(a);return s===void 0?r.readText(a):s},has(a){return r.has(a)||n.has(a)}})}}function Om(e){throw new Error(`WebGPU op package ${e} is not embedded in this runtime`)}function ns(e,t={}){let r=Dm(e);return new ts(r.manifest,{...r.renderedSourceAudit===void 0?{}:{renderedSourceAudit:r.renderedSourceAudit},...t,assets:r.assets})}function Dm(e){let t=qm(e);if(t)return t;let r=Lm.get(e);if(r){let n=new Map(Gm);for(let[a,s]of r.assets)n.set(a,s);return{manifest:r.manifest,assets:Cn(n,r.assumes)}}return Om(e)}function ve(e,t=new Set){Go(e,t,!0)}function Go(e,t,r){if(!(e==null||typeof e!="object")&&!t.has(e)&&(t.add(e),!(ArrayBuffer.isView(e)||e instanceof ArrayBuffer))){if(F_(e)){r&&e.destroy();return}if(Q_(e)){r&&e.destroy();return}if(e instanceof Map){for(let n of e.values())Go(n,t,r);r&&e.clear();return}if(e instanceof Set){for(let n of e.values())Go(n,t,r);r&&e.clear();return}for(let n of Object.keys(e))Go(e[n],t,r)}}function F_(e){let t=e;return Array.isArray(t.shape)&&typeof t.dtype=="string"&&typeof t.destroy=="function"&&t.buffer!=null&&typeof t.buffer.destroy=="function"}function Q_(e){let t=e;return typeof t.destroy=="function"&&typeof t.mapAsync=="function"&&typeof t.getMappedRange=="function"}function xt(e,t){if(e.length===1)throw e[0];if(e.length>1)throw new AggregateError(e,t)}async function cr(e,t){try{await e.queueIdle(),ve(t)}catch{}}function Pn(e){throw new Error(`This library was built without ${e}, which this model needs: load it with a library built for it.`)}var j_=`<|channel>thought
`,N_="<channel|>",uP=j_+N_;var Al="<think>",qo="</think>",kl="webgpu-template-policy-probe",z_="webgpu-template-policy-probe-next",K_="webgpu-template-policy-probe-result",Mm=`<tool_call>
<function=webgpu_probe>
<parameter=x>
1
</parameter>
</function>
</tool_call>`,Pr=Object.freeze({preserve_thinking:!0});function U_(e,t,r){let n=$r(e,t,[r],!0,{...Pr,enable_thinking:!0}),a=$r(e,t,[r],!0,{...Pr,enable_thinking:!1}),s=n.slice(Math.max(0,n.lastIndexOf(Al))),o=a.slice(Math.max(0,a.lastIndexOf(Al)));return/^<think>\s*$/.test(s)&&/^<think>\s*<\/think>\s*$/.test(o)?{open:s,closed:o}:null}function Lo(e,t,r){return e.endsWith(t.open)?{prelude:t.open,kind:"open"}:e.endsWith(t.closed)?{prelude:t.closed,kind:"closed"}:r?{prelude:"",kind:"model"}:null}function Fm(e,t,r,n){let a=[...r.map(s=>`reasoning
</think>

${s}`),...n?["unfinished reasoning"]:[]];return e==="open"?a:e==="closed"?r:[...a.map(s=>t.open+s),...r.map(s=>t.closed+s)]}function $r(e,t,r,n,a){return e.render({...a,messages:r,tools:null,bos_token:t.bos_token,eos_token:t.eos_token,add_generation_prompt:n})}function Sl({prelude:e,kind:t,rawText:r}){return!(t==="open"||t==="model"&&r.includes(Al))||r.includes(qo)?e+r:e+r+(r.endsWith(`
`)?"":`
`)+`</think>

`}var xl=e=>({content:Sl(e)});function Wm(e,t,r,n,a,s){let o=new WeakMap,i=l=>{if(o.has(l))return o.get(l);let c=$r(e,t,[{role:"user",content:kl}],!0,l),d=Lo(c,s,a);return o.set(l,d),d},u=l=>{let c=i(l.templateArgs),d=l.position==="after_last_user"?n:r;return c===null||d===null?{content:l.visibleText}:d({...c,rawText:l.rawText})};return Object.freeze({templateArgs:Pr,assistantContent:l=>u(l).content,assistantMessageFields:l=>{let{content:c,...d}=u(l);return d}})}function Qm(e,t,r,n,{openPrelude:a=!1}={}){let s={role:"user",content:kl},o={role:"user",content:z_},i=Object.freeze({...Pr,enable_thinking:!0}),u=Object.freeze({...Pr,enable_thinking:!1});try{let l=U_(e,t,s);if(l===null)return;let c=Wm(e,t,r,null,a,l);if(a&&Lo($r(e,t,[s],!0,Pr),l,!1)!==null)return;for(let p of[i,u,Pr]){let f=$r(e,t,[s],!0,p),m=Lo(f,l,a);if(m===null)return;for(let h of Fm(m.kind,l,["answer"],!0)){let g={visibleText:h,rawText:h,templateArgs:p},w={...c.assistantMessageFields(g),role:"assistant",content:c.assistantContent(g)};if(!$r(e,t,[s,w,o],!0,p).startsWith(f+h))return}}let d=n.find(p=>$_(e,t,p,a,l))??null;return Wm(e,t,r,d,a,l)}catch{return}}function $_(e,t,r,n,a){let s={role:"user",content:kl},o={role:"tool",content:K_};try{for(let i of[!0,!1,void 0]){let u=i===void 0?Pr:{...Pr,enable_thinking:i},l=$r(e,t,[s],!0,u),c=Lo(l,a,n);if(c===null)return!1;for(let d of Fm(c.kind,a,[Mm,`checking

${Mm}`,"answer"],!1)){let p={...r({...c,rawText:d}),role:"assistant"};if(!$r(e,t,[s,p,o],!0,u).startsWith(l+d))return!1}}return!0}catch{return!1}}function jm(e,t,r){return Qm(e,t,xl,r)}var H_=({kind:e,rawText:t})=>{if(e==="closed")return{reasoning_content:"",content:t};let r=t.indexOf(qo);return r===-1?{reasoning_content:t,content:""}:{reasoning_content:t.slice(0,r),content:t.slice(r+qo.length).replace(/^\n\n/,"")}},V_=e=>({reasoning_content:"",content:Sl(e)}),Nm=[xl,H_,V_];var Z_={"qwen-thinking":(e,t)=>jm(e,t,Nm)};function zm(e,t){return X_(e,t)?.policy}function X_(e,t){for(let[r,n]of Object.entries(Z_)){let a=n(e,t);if(a)return{name:r,policy:a}}}async function Km(e,t){let r=Math.max(1,Math.floor(t.baseDepth)),n=Math.max(r,Math.floor(t.maxDepth)),a=J_(t.minSampleTokens,24,2),s=.97,o=5,i=[r];for(;i.length<o&&i.at(-1)<n;){let d=i.at(-1);i.push(Math.min(n,d*2))}let u=[],l=null,c=0;for(let d of i){let p=Math.max(a,d+1),f=await e(d,p);if(!Number.isFinite(f.tokensPerSecond)||f.tokensPerSecond<=0)throw new Error(`decode calibration at depth ${d} returned invalid throughput`);if(f.tokenIds.length!==p)throw new Error(`decode calibration at depth ${d} produced ${f.tokenIds.length}/${p} tokens`);let m=l===null||Y_(l,f.tokenIds);if(u.push({depth:d,sampleTokens:p,tokensPerSecond:f.tokensPerSecond,outputMatchesReference:m}),!m)return Tl(u,s,"output-mismatch");if((l===null||f.tokenIds.length>l.length)&&(l=f.tokenIds),c>0&&f.tokensPerSecond<=c/s)return Tl(u,s,"saturated");c=f.tokensPerSecond}return Tl(u,s,"max-depth")}function Tl(e,t,r){let n=e.filter(o=>o.outputMatchesReference),a=Math.max(...n.map(o=>o.tokensPerSecond));return{depth:n.find(o=>o.tokensPerSecond>=a*t)?.depth??n[0]?.depth??1,samples:e,totalSampleTokens:e.reduce((o,i)=>o+i.sampleTokens,0),stoppedBecause:r}}function Y_(e,t){let r=Math.min(e.length,t.length);for(let n=0;n<r;++n)if(e[n]!==t[n])return!1;return!0}function J_(e,t,r){return Number.isFinite(e)?Math.max(r,Math.floor(e)):t}async function Um(e,t=6){let r=Math.max(1,Math.floor(t)),n=e.tensorFromTypedArray("uint32",[1,1],new Uint32Array([0]));try{await e.readTensor(n);let a=[];for(let s=0;s<r;++s){let o=performance.now();await e.readTensor(n),a.push(performance.now()-o)}return a.sort((s,o)=>s-o),a[a.length>>1]}finally{n.destroy()}}function $m(e){let t=globalThis;return typeof t.ImageBitmap=="function"&&e instanceof t.ImageBitmap||typeof t.VideoFrame=="function"&&e instanceof t.VideoFrame}var Hm="https://huggingface.co";function ey(e){return typeof e=="string"&&/^https?:/i.test(e)}function ty(e){return ey(e)||e.startsWith("/")||e.startsWith(".")}function Cl(e,t={}){let{revision:r="main",file:n}=t,a=n?n.slice(n.lastIndexOf(".")).toLowerCase():"";if(a&&e.toLowerCase().endsWith(a))return e;if(ty(e))return n?`${ry(e)}/${n}`:e;let s=`${Hm}/${e}/resolve/${r}`;return n?`${s}/${n}`:s}function ry(e){return e.endsWith("/")?e.slice(0,-1):e}function ny(e){try{return new URL(e).origin===Hm}catch{return!1}}function Vm(e){return(t,r={})=>{if(!ny(t))return globalThis.fetch(t,r);let n=new Headers(r.headers);return n.set("Authorization",`Bearer ${e}`),globalThis.fetch(t,{...r,headers:n})}}var ay=new Set(["mozilla","brave","chrome","graphics","generic renderer","webkit webgl","webgl","webgpu","angle","unknown","subsystem opengl es translator"]),sy=new Set(["apple","apple gpu","intel","intel graphics","hd graphics","intel hd graphics","intel hd graphics family","intel uhd graphics","intel arc graphics","intel iris","intel iris graphics","intel iris plus","intel iris plus graphics","intel iris pro","intel iris pro graphics","amd","amd radeon","amd radeon graphics","amd radeon pro graphics","amd unknown","nvidia","nvidia tegra","nvidia graphics device","nvidia geforce gpu","adreno","adreno gpu","mali","powervr"]),IP=Math.max(...[...ay].map(e=>e.length)),BP=Math.max(...[...sy].map(e=>e.length));var oy="shader-f16",iy="subgroups",uy="chromium-experimental-subgroup-matrix",ly="timestamp-query",cy="packed_4x8_integer_dot_product";function Zm(e){let t=Nt(e),r=t.adapterInfo,n=new Set(t.features),a=new Set(t.wgslLanguageFeatures),s={};for(let o of Na){let i=t.limits[o];typeof i=="number"&&(s[o]=i)}return{vendor:r.vendor??"",architecture:r.architecture??"",device:r.device??"",description:r.description??"",isFallbackAdapter:r.isFallbackAdapter===!0,subgroupMinSize:r.subgroupMinSize??null,subgroupMaxSize:r.subgroupMaxSize??null,subgroupMatrixConfigs:r.subgroupMatrixConfigs??null,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,capabilities:{shaderF16:n.has(oy),subgroups:n.has(iy),subgroupMatrix:n.has(uy),timestampQuery:n.has(ly),dp4a:a.has(cy)},limits:s}}function Oo(e){let t=Zm(e);return{vendor:t.vendor,architecture:t.architecture,device:t.device,description:t.description,isFallbackAdapter:t.isFallbackAdapter,subgroupMinSize:t.subgroupMinSize??void 0,subgroupMaxSize:t.subgroupMaxSize??void 0,features:{shaderF16:t.capabilities.shaderF16,subgroups:t.capabilities.subgroups,subgroupMatrix:t.capabilities.subgroupMatrix,timestampQuery:t.capabilities.timestampQuery}}}function Pl(e){return e.fetch??(e.accessToken?Vm(e.accessToken):void 0)}async function Xm(e){let t=globalThis.navigator?.gpu;if(!t)return{ok:!1,reason:"WebGPU isn't available in this browser. Try a recent Chrome or Edge."};let r=null;try{r=await t.requestAdapter({powerPreference:"high-performance"})}catch{}if(!r)return{ok:!1,reason:"No WebGPU adapter is available on this device."};try{let n=await e(r);return n?{ok:!1,reason:n}:{ok:!0}}catch{return{ok:!0}}}var Do=class{#e;#t;model;tokenizer;#n;#r;#a;#s;eosTokenIds;#l;generationState;#u;#o;#i;#c;#d;#p;#f;#m;#g;#h=null;#w=new Set;#b=null;#y=new WeakMap;#v=!1;#T=Promise.resolve();destroyed=!1;#G=null;#C=!0;#_=null;#P;lost=new Promise(t=>{this.#P=t});chatTemplateArgs={};constructor(t){this.#e=t.runtime,this.#t=t.ownsRuntime,this.model=t.model,this.tokenizer=t.tokenizer,this.#n=t.chatTemplate,this.#r=t.tokenizerConfig,this.#a=t.chatTemplatePolicy===null?void 0:t.chatTemplatePolicy??zm(t.chatTemplate,t.tokenizerConfig),this.#s=new Xn(t.tokenizer),this.eosTokenIds=t.eosTokenIds,this.generationState=t.generationState,this.#l=t.defaultMaxNewTokens,this.#u=t.prefixSnapshotStore??null,this.#o=t.prefixSnapshotIdentity??null,this.#d=t.vision??null,this.#p=t.speech??null,this.#f=t.createGenerationState??null,this.#m=t.disposeGenerationState??null,this.#g=t.generationConfig??Object.freeze({do_sample:!1});let r=t.tokenizer.token_to_id?.bind(t.tokenizer);this.#i=this.#a?.replyMarkup??(r&&r("<think>")!=null&&r("</think>")!=null?rp:null),this.#c=r?dp(this.#q(),r):null;let n=new WeakRef(this);t.runtime.host?.device?.lost?.then(a=>{let s=n.deref();s&&s.#L(a)})}#L(t){if(!this.#C||this.#_)return;this.#_=t;let r=new ta(t.reason,t.message);for(let n of this.#w)n.deviceLost(r);this.#P(r)}#A(){if(this.destroyed)throw new Error("The model has been disposed");if(this.#_)throw new ta(this.#_.reason,this.#_.message)}get generation_config(){return this.#g}get runtime(){return this.#e}acquireGenerationLease(){if(this.#A(),this.#v)throw new hn("The model is already generating");if(this.#h)throw new Error("The model's cache is held by a session; dispose the session first");return this.#I()}#I(){this.#v=!0;let t;this.#T=new Promise(n=>{t=n});let r=!0;return()=>{r&&(r=!1,this.#v=!1,t())}}get contextLength(){return this.generationState.cache.maxLength}#k(){return Object.freeze({...this.#a?.templateArgs??{},...this.chatTemplateArgs})}#S(t,r,n,a=null){let s=this.#c?.inMarkup===!0;return this.#n.render({...n,messages:t.map(o=>typeof o.content=="string"?s?pp(o):o:{...o,content:o.content.map(i=>i.type==="text"?i:{type:i.type})}),tools:a,bos_token:this.#r.bos_token,eos_token:this.#r.eos_token,add_generation_prompt:r})}#q(){try{return this.#S([{role:"user",content:"probe"}],!0,this.#k(),cp)}catch{return null}}#B(t,r){let n=[],a=this.#p;for(let o of t){let i=r?.get(o);if(i){n.push(...i);continue}if(typeof o.content!="string")for(let u of o.content){if(u.type==="audio"||u.type==="audio_out"){if(!a)throw new Error("This model takes no speech; it cannot take audio messages");n.push(u.type==="audio"?a.planAudio(u.audio):a.planSpeech(u.codes));continue}if(u.type!=="image")continue;if(!this.#d)throw new Error("This model has no vision tower; it cannot take image messages");let l=u.image,c=$m(l),d=c?this.#y.get(l):void 0;d||(d=this.#d.planImage(l),c&&this.#y.set(l,d)),n.push(d)}}if(!this.#d&&!a)return;let s=[...this.#d?[this.#d.imageTokenId]:[],...a?[a.audioTokenId,a.audioOutTokenId]:[]];return{imageTokenId:this.#d?.imageTokenId??-1,placeholderTokenIds:s,images:n}}#E(t,r){if(!this.#u||this.#o===null)return null;let n=t.prefixSnapshotLayout?.(r.length);return n?`${this.#o}|layout=${n}|ids=${r.join(",")}`:null}async#O(t,r){let n=this.#E(t,r);if(n===null||!t.importPrefixSnapshot)return!1;let a=await this.#u?.load(n);return!a||a.length!==r.length||!await t.importPrefixSnapshot(a)?!1:t.get_seq_length()!==r.length?(t.truncate(0),!1):!0}async#D(t,r){let n=this.#E(t,r);if(n===null||!t.exportPrefixSnapshot)return;let a;try{a=await t.exportPrefixSnapshot()}catch{return}!a||a.length!==r.length||this.#u?.save({identity:n,...a}).catch(()=>{})}renderPrompt(t,r=!0){return this.#S(t,r,this.#k())}encodePrompt(t){return this.tokenizer.encode(this.renderPrompt(t),{add_special_tokens:!1}).ids}async benchmarkFixedTokenIds(t,r,n,a){let s=this.acquireGenerationLease();try{return await this.#R(t,r,n,a)}finally{s()}}benchmarkMessages(t,r,n){let a=this.#B(t),s=new Xn(this.tokenizer).prepare(this.renderPrompt(t),a);return this.benchmarkFixedTokenIds(s.inputIds,r,n,a?.images)}async calibrateDecodePipeline(t,r,n=[0]){let a=this.acquireGenerationLease();try{return await Km(async(s,o)=>{let i=await this.#R(n,o,r(s));return{tokensPerSecond:i.decodeTps,tokenIds:i.ids}},t)}finally{a()}}async#R(t,r,n,a){this.resetCache();let s=[],o=0,i=0,u=performance.now();try{for await(let l of this.streamTokens({suffixIds:[...t],maxNewTokens:r,eosTokenId:this.eosTokenIds,stopOnEos:!1,...a&&a.length>0?{images:a}:{}},n))s.length===0&&(o=performance.now()),s.push(l);i=performance.now()}finally{this.resetCache()}return{ttftMs:s.length>0?o-u:NaN,decodeTps:s.length>1?(s.length-1)/((i-o)/1e3):0,tokens:s.length,ids:s}}deviceInfo(){return Oo(this.#e.device)}create_session(t={}){this.#A();let r=this.#h===null,n=this.generationState;if(!r){if(!this.#f)throw new Error("The model holds one session at a time; dispose the current one first");n=this.#f()}let a=n.cache.rewindPointCapacity,s;try{s=new Ns({host:this.#b??=this.#M(),state:n,ledger:r?this.#s:new Xn(this.tokenizer),options:t,release:()=>{if(this.#w.delete(s),!r){this.#x(n);return}a!==void 0&&n.cache.setRewindPointCapacity?.(a),this.#h=null}})}catch(o){throw r||this.#x(n),o}return r&&(this.#h=s),this.#w.add(s),s}#M(){return{tokenizer:this.tokenizer,eosTokenIds:this.eosTokenIds,replyMarkup:this.#i,toolCallDelimiters:this.#c,defaultMaxNewTokens:this.#l,generationConfig:this.#g,templateArgs:t=>Object.freeze({...this.#k(),...t}),renderPrompt:(t,r,n,a)=>this.#S(t,r,n,a??null),planImages:(t,r)=>this.#B(t,r),speech:this.#p,historyMessage:t=>np(this.#a,t),streamTokens:(t,{decodePipelineDepth:r,...n})=>this.streamTokens({...n,generationState:t},r!==void 0?{decodePipelineDepth:r}:{}),acquireLease:()=>{if(this.#A(),this.#v)throw new hn("The model is already generating; one turn runs at a time");return this.#I()},restoreSystemPrefix:(t,r)=>this.#O(t.cache,r),persistSystemPrefix:(t,r)=>this.#D(t.cache,r)}}reset(){this.destroyed||(this.#h?this.#h.reset():this.resetCache())}resetCache(){this.generationState.cache.truncate(0),this.#s.clear()}#x(t){this.#m?this.#m(t):t.cache.dispose?.()}dispose(){return this.#G??=this.#W()}async#W(){this.destroyed=!0;let t=[];for(let n of await Promise.allSettled([...this.#w].map(a=>a.dispose())))n.status==="rejected"&&t.push(n.reason);this.#_||await this.#T;let r=n=>{try{n()}catch(a){t.push(a)}};r(()=>this.#x(this.generationState)),r(()=>this.model.dispose()),this.#C=!1,this.#t&&await this.#e.destroy().catch(n=>t.push(n)),xt(t,"The model failed to release everything it held")}};async function*Ym(e){let t=[],r=()=>{let a=e.submit();return{unit:a,result:my(a.result)}},n=[];try{for(;t.length<e.depth&&e.shouldSubmit();)t.push(r());for(;t.length>0;){let a=t[0];e.shouldSubmit()&&t.length-1<e.depth&&t.push(r()),t.shift();let s=await a.result;if(yield*e.accept(a.unit,s))return}}catch(a){throw n.push(a),a}finally{let a=await Promise.allSettled(t.map(({result:s})=>s));if(n.length>0){let s=a.flatMap(o=>o.status==="rejected"?[o.reason]:[]);if(s.length>0)throw new AggregateError([...n,...s],"Pipelined generation and in-flight cleanup both failed")}}}var Ir=64,Il=new WeakMap;async function dy(e){let t=Il.get(e);t!==void 0&&(Il.delete(e),await t)}var py=4;function fy(e){return Math.max(e+1,Math.min(Ir,e*py))}function In(e,t,r=t){let n=e.decode_pipeline_depth;return typeof n!="number"||!Number.isFinite(n)||t<=1?t:Math.max(1,Math.min(Math.floor(n),r))}async function Bl(e,t,r){for(let n of r){for await(let a of e.streamTokenIdsFromCache({input_ids:[new Array(n).fill(0)],generation_state:{cache:t},max_new_tokens:2,stop_on_eos:!1}));t.truncate(0)}}async function*Jm(e,t,r){if(t.length===0)throw new Error("generation requires at least one input token");if(e.sampling===void 0&&Qr(r)!==null)throw new Error("this model decodes greedily only; it cannot sample (do_sample)");let{maxNewTokens:n,eosTokenId:a,stopOnEos:s,onPrefillDone:o,onPrefillProgress:i,signal:u}=Yn(r,e.eosTokenId!==void 0?{eosTokenId:e.eosTokenId}:{}),l=e.cache,c=S=>s&&Jn(S,a);await dy(l),u?.throwIfAborted();let d=l.get_seq_length(),p=u?await l.mutableStateCheckpoint?.()??null:null;p?.capture();let f;try{f=await e.prefill(t,d,{signal:u,onProgress:i&&(S=>i({done:S,total:t.length}))})}catch(S){try{p?.restore()}catch(C){throw new AggregateError([S,C],"Prefill and its rollback both failed")}throw S}if(l.advance(t.length),o?.({tokens:t.length,cache_length:l.get_seq_length()}),u?.throwIfAborted(),n<=0||c(f))return;yield f;let m=1;if(m>=n)return;let h={value:null},g=async S=>h.value??=await e.beginDecode(S,n-m),w=!1,b=async function*(S){return l.advance(1),c(S)?!0:u?.aborted?(w=!0,!0):(yield S,m+=1,m>=n)},v=l.get_seq_length();if(v>=l.maxLength)return;let y=[];try{if(e.pipelineDepth>1){let k=await l.mutableStateCheckpoint?.()??null;if(k){let O=fy(e.pipelineDepth),N=await l.alternateMutableStateCheckpoint?.()??null,Q=N?[k,N]:[k],V=await g(v),$=Math.min(n-m,l.maxLength-v),M=[f],j=0,A=0,T=!1;for(;j<$;){let E=j,X=N?$:Math.min($,E+O),z=[];try{yield*Ym({depth:e.pipelineDepth,shouldSubmit:()=>j<X,submit:()=>{j%O===0&&Q[j/O%Q.length].capture();let G=V.submitStep(j===E?M[E]:null,v+j);return j+=1,{result:G}},accept:async function*(G,L){return l.advance(1),A+=1,c(L)?(T=!0,!0):u?.aborted?(w=!0,!0):(yield L,m+=1,M.push(L),!1)}})}catch(G){throw z.push(...G instanceof AggregateError?G.errors:[G]),G}finally{try{if(A<j){let G=Math.floor(Math.max(0,A-1)/O),L=G*O;Q[G%Q.length].restore();let K=M.slice(L,A);if(V.replayKnownInputs){let D=V.replayKnownInputs(K,v+L);T?(Il.set(l,D),D.catch(()=>{})):await D}else for(let D=0;D<K.length;++D)await V.step(K[D],v+L+D)}}catch(G){throw z.length>0?new AggregateError([...z,G],"Pipelined decode and cleanup both failed"):G}}if(T)return;w&&u.throwIfAborted()}return}let B=n-m,P=v,_=0,x=!0,R=await g(v);yield*Ym({depth:e.pipelineDepth,shouldSubmit:()=>_<B&&P<l.maxLength,submit:()=>{let O=R.submitStep(x?f:null,P);return P+=1,_+=1,x=!1,{result:O}},accept:(O,N)=>b(N)}),w&&u.throwIfAborted();return}let S=f,C=v;for(;m<n&&C<l.maxLength;){let B=await(await g(C)).step(S,C);if(yield*b(B))break;S=B,C=l.get_seq_length()}w&&u.throwIfAborted()}catch(S){throw y.push(S),S}finally{try{await h.value?.finish()}catch(S){throw y.length>0?new AggregateError([...y,S],"Token decode and session cleanup both failed"):S}}}function my(e){let t=e instanceof Promise?e:new Promise((r,n)=>{e.then(r,n)});return t.catch(()=>{}),t}var ca=4,Mo=class extends Do{#e;#t=0;constructor(t){super(t)}streamTokens(t,r){return this.model.streamTokenIdsFromCache({input_ids:[t.suffixIds],generation_state:t.generationState??this.generationState,max_new_tokens:t.maxNewTokens,eos_token_id:t.eosTokenId,stop_on_eos:t.stopOnEos,decode_pipeline_depth:r.decodePipelineDepth??this.#e,...t.images?{images:t.images}:{},...t.signal?{signal:t.signal}:{},...t.onPrefillProgress?{on_prefill_progress:t.onPrefillProgress}:{},...t.sampling?{do_sample:!0,...t.sampling}:{},...t.speech?{on_speech:t.speech.sink,generation_mode:t.speech.generationMode,audio_chunk_frames:t.speech.audioChunkFrames,...t.speech.toolCallToken!==null?{tool_call_token:t.speech.toolCallToken}:{}}:{}})}get decodePipelineDepth(){return this.#e}get readbackLatencyMs(){return this.#t}async tuneDecodePipeline(t={}){if(this.#t=await Um(this.runtime),t.decodePipelineDepth!==void 0)return this.#e=In({decode_pipeline_depth:t.decodePipelineDepth},ca,Ir),this.#e;let r=this.contextLength,n=Math.min(Ir,r-1);if(n<=ca)return this.#e=ca,ca;let a=await this.calibrateDecodePipeline({baseDepth:ca,maxDepth:n,minSampleTokens:Math.min(24,r)},s=>({decodePipelineDepth:s}));return this.#e=a.depth,a.depth}async benchmark(t,r={}){let n=In({decode_pipeline_depth:r.decodePipelineDepth??this.#e},ca,Ir);return{...await this.benchmarkMessages(t,r.maxNewTokens??128,{decodePipelineDepth:n}),depth:n}}};var gy=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let r of e){let n=t;for(let a=0;a<r.length;++a){let s=r[a];n=n[s]??=Object.create(null)}n.end=r}return t}split(e){let t=[],r=e.length,n=0,a=0;for(;a<r;){let s=this.trie,o=null,i=a;for(;i<r&&(s=s[e[i]]);)s.end&&(o=s.end),++i;o?(a>n&&t.push(e.slice(n,a)),t.push(o),a+=o.length,n=a):++a}return n<r&&t.push(e.slice(n)),t}},e2=gy,hy=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},wy=hy,i2=(()=>{let e=[...Array.from({length:94},(a,s)=>s+33),...Array.from({length:12},(a,s)=>s+161),...Array.from({length:82},(a,s)=>s+174)],t=e.slice(),r=0;for(let a=0;a<256;++a)e.includes(a)||(e.push(a),t.push(256+r),r+=1);let n=t.map(a=>String.fromCharCode(a));return Object.fromEntries(e.map((a,s)=>[a,n[s]]))})(),by=e=>Object.fromEntries(Object.entries(e).map(([t,r])=>[r,t])),vy=by(i2),t2=".,!?\u2026\u3002\uFF0C\u3001\u0964\u06D4\u060C",_y=new Map([["(?i:'s|'t|'re|'ve|'m|'ll|'d)","(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))"],["(?i:[sdmt]|ll|ve|re)","(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])"],["[^\\r\\n\\p{L}\\p{N}]?+","[^\\r\\n\\p{L}\\p{N}]?"],["[^\\s\\p{L}\\p{N}]++","[^\\s\\p{L}\\p{N}]+"],["(?>\\p{Nd}{510})","(?:\\p{Nd}{510})"],["\\p{Nd}{3}+","(?:\\p{Nd}{3})+"],["\\G",""],[` ?[^(\\s|[${t2}])]+`,` ?[^\\s${t2}]+`]]),Wo="\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E",Rl=e=>e.replace(/ \./g,".").replace(/ \?/g,"?").replace(/ \!/g,"!").replace(/ ,/g,",").replace(/ \' /g,"'").replace(/ n't/g,"n't").replace(/ 'm/g,"'m").replace(/ 's/g,"'s").replace(/ 've/g,"'ve").replace(/ 're/g,"'re"),Fo=(e,t=!0)=>{if(e.Regex!==void 0){let r=e.Regex.replace(/\\([#&~])/g,"$1");r=r.replace(/\\A/g,"^").replace(/\\z/g,"$").replace(/\\Z/g,"(?=\\r?\\n?$)");for(let[n,a]of _y)r=r.replaceAll(n,a);try{return new RegExp(r,"gu")}catch(n){if(!(n instanceof SyntaxError)||!n.message.toLowerCase().includes("invalid property name"))throw n;let a=!1,s=r.replace(/(\\[pP])\{([^}=]+)\}/g,(o,i,u)=>{try{return new RegExp(`\\p{${u}}`,"u"),`${i}{${u}}`}catch{return a=!0,`${i}{Script=${u}}`}});if(!a)throw n;try{return new RegExp(s,"gu")}catch{throw n}}}else if(e.String!==void 0){let r=yy(e.String);return new RegExp(t?r:`(${r})`,"gu")}else return console.warn("Unknown pattern type:",e),null},yy=e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),Ay=(e,t,r)=>{let n=[],a=0;for(;a<e.length;){if(n.push(e[a]),(t.get(e[a])??r)!==r){++a;continue}for(;++a<e.length&&(t.get(e[a])??r)===r;)t.get(n.at(-1))!==r&&(n[n.length-1]+=e[a])}return n},ky=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,Sy=e=>Number.isInteger(e)||typeof e=="bigint",xy=e=>{let t=0;for(let r of e)++t;return t},Ty=e=>u2(e.toLowerCase()),Vt=(...e)=>Array.prototype.concat.apply([],e),Gl=e=>new Map(Object.entries(e)),Cy=(e,t)=>{let r=[],n=0;for(let a of e.matchAll(t)){let s=a[0];n<a.index&&r.push(e.slice(n,a.index)),s.length>0&&r.push(s),n=a.index+s.length}return n<e.length&&r.push(e.slice(n)),r},u2=e=>e.replace(/\p{M}/gu,""),r2=(e,t,r=[])=>{if(!e||Array.isArray(e)||typeof e!="object")return`${t} must be a valid object`;for(let n of r)if(!(n in e))return`${t} must contain a "${n}" property`;return null},Py=e=>e.match(/\S+/g)||[],Iy=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},as=Iy,By=class extends as{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},Br=By,Ey=class extends Br{tokenize_chinese_chars(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r],a=n.charCodeAt(0);ky(a)?(t.push(" "),t.push(n),t.push(" ")):t.push(n)}return t.join("")}strip_accents(e){return e.normalize("NFD").replace(/\p{Mn}/gu,"")}is_control(e){switch(e){case"	":case`
`:case"\r":return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let r of e){let n=r.charCodeAt(0);n===0||n===65533||this.is_control(r)||(/^\s$/.test(r)?t.push(" "):t.push(r))}return t.join("")}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},Ry=Ey,Gy=class extends Br{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,""),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm," "),e.includes("\uFF5E")?e=e.split("\uFF5E").map(r=>r.normalize("NFKC")).join("\uFF5E"):e=e.normalize("NFKC"),e}},Ly=Gy,qy=class extends Br{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(t=>l2(t))}normalize(e){return this.normalizers.reduce((t,r)=>r?r.normalize(t):t,e)}},Oy=qy,Dy=class extends Br{normalize(e){let t=Fo(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??"")}},My=Dy,Wy=class extends Br{constructor(){super(...arguments),this.form="NFC"}normalize(e){return e=e.normalize(this.form),e}},Qo=Wy,Fy=class extends Qo{constructor(){super(...arguments),this.form="NFC"}},Qy=Fy,jy=class extends Qo{constructor(){super(...arguments),this.form="NFD"}},Ny=jy,zy=class extends Qo{constructor(){super(...arguments),this.form="NFKC"}},Ky=zy,Uy=class extends Qo{constructor(){super(...arguments),this.form="NFKD"}},$y=Uy,Hy=class extends Br{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},Vy=Hy,Zy=class extends Br{normalize(e){return u2(e)}},Xy=Zy,Yy=class extends Br{normalize(e){return e.toLowerCase()}},Jy=Yy,eA=class extends Br{normalize(e){return e=this.config.prepend+e,e}},tA=eA;function rA(e){if(e===null)return null;switch(e.type){case"BertNormalizer":return new Ry(e);case"Precompiled":return new Ly(e);case"Sequence":return new Oy(e);case"Replace":return new My(e);case"NFC":return new Qy(e);case"NFD":return new Ny(e);case"NFKC":return new Ky(e);case"NFKD":return new $y(e);case"Strip":return new Vy(e);case"StripAccents":return new Xy(e);case"Lowercase":return new Jy(e);case"Prepend":return new tA(e);default:throw new Error(`Unknown Normalizer type: ${e.type}`)}}var l2=rA,nA=class extends as{pre_tokenize(e,t){return(Array.isArray(e)?e.map(r=>this.pre_tokenize_text(r,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},Zt=nA,aA=class extends Zt{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=i2,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(" ")&&(e=" "+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(n=>Array.from(this.text_encoder.encode(n),a=>this.byte_encoder[a]).join(""))}},sA=aA,oA=class extends Zt{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},iA=oA,uA=class extends Zt{constructor(e){super(),this.replacement=e.replacement??"\u2581",this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??"always"}pre_tokenize_text(e,t){let{section_index:r=void 0}=t??{},n=e.replaceAll(" ",this.str_rep);return!n.startsWith(this.replacement)&&(this.prepend_scheme==="always"||this.prepend_scheme==="first"&&r===0)&&(n=this.str_rep+n),[n]}},lA=uA,cA=class extends Zt{constructor(e){super(),this.config=e,this.pattern=Fo(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()==="removed"?e.split(this.pattern).filter(t=>t):Cy(e,this.pattern)}},dA=cA,pA=class extends Zt{constructor(e){super(),this.config=e,this.pattern=new RegExp(`[^${Wo}]+|[${Wo}]+`,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},fA=pA,mA=class extends Zt{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?"":"+"}`;this.pattern=new RegExp(t,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},gA=mA,hA=class extends Zt{constructor(){super(),this.pattern=new RegExp(`[^\\s${Wo}]+|[${Wo}]`,"gu")}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},wA=hA,bA=class extends Zt{constructor(e){super(),this.config=e,this.pattern=Fo(this.config.pattern??{}),this.content=this.config.content??""}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??"")]}},vA=bA,_A=class extends Zt{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(t=>c2(t))}pre_tokenize_text(e,t){return this.tokenizers.reduce((r,n)=>n?n.pre_tokenize(r,t):r,[e])}},yA=_A,AA=class extends Zt{pre_tokenize_text(e){return Py(e)}},kA=AA,SA=class extends Zt{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let r=0;r<e.length;r+=this._length)t.push(e.slice(r,r+this._length));return t}},xA=SA;function TA(e){if(e===null)return null;switch(e.type){case"BertPreTokenizer":return new wA;case"Sequence":return new yA(e);case"Whitespace":return new iA;case"WhitespaceSplit":return new kA;case"Metaspace":return new lA(e);case"ByteLevel":return new sA(e);case"Split":return new dA(e);case"Punctuation":return new fA(e);case"Digits":return new gA(e);case"Replace":return new vA(e);case"FixedLength":return new xA(e);default:throw new Error(`Unknown PreTokenizer type: ${e.type}`)}}var c2=TA,CA=class extends as{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=Ay(t,this.tokens_to_ids,this.unk_token_id)),t}},jo=CA,PA=class extends jo{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=Gl(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=new Array(this.tokens_to_ids.size);for(let[t,r]of this.tokens_to_ids)this.vocab[r]=t}encode(e){let t=[];for(let r of e){let n=[...r];if(n.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let a=!1,s=0,o=[];for(;s<n.length;){let i=n.length,u=null;for(;s<i;){let l=n.slice(s,i).join("");if(s>0&&(l=this.config.continuing_subword_prefix+l),this.tokens_to_ids.has(l)){u=l;break}--i}if(u===null){a=!0;break}o.push(u),s=i}a?t.push(this.unk_token):t.push(...o)}return t}},n2=PA,a2=class d2{constructor(t,r){this.is_leaf=t,this.children=r}static default(){return new d2(!1,new Map)}},IA=class{constructor(){this.root=a2.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let r of e){let n=t.children.get(r);n===void 0&&(n=a2.default(),t.children.set(r,n)),t=n}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let r="";for(let n of e){if(r+=n,t=t.children.get(n),t===void 0)return;t.is_leaf&&(yield r)}}},BA=IA,El=class p2{constructor(t,r,n,a,s){this.token_id=t,this.node_id=r,this.pos=n,this.length=a,this.score=s,this.prev=null,this.backtrace_score=0}clone(){let t=new p2(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},EA=class{constructor(e,t,r){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=r,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let n=new El(this.bos_token_id??0,0,0,0,0),a=new El(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(n.clone()),this.nodes.push(a.clone()),this.begin_nodes[this.len].push(a),this.end_nodes[0].push(n)}insert(e,t,r,n){let a=this.nodes.length,s=new El(n,a,e,t,r);this.begin_nodes[e].push(s),this.end_nodes[e+t].push(s),this.nodes.push(s)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let o of this.begin_nodes[t]){o.prev=null;let i=0,u=null;for(let l of this.end_nodes[t]){let c=l.backtrace_score+o.score;(u===null||c>i)&&(u=l.clone(),i=c)}if(u!==null)o.prev=u,o.backtrace_score=i;else return[]}++t}let r=[],a=this.begin_nodes[e][0].prev;if(a===null)return[];let s=a.clone();for(;s.prev!==null;)r.push(s.clone()),s=s.clone().prev.clone();return r.reverse(),r}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join("")}tokens(){return this.viterbi().map(t=>this.piece(t))}token_ids(){return this.viterbi().map(t=>t.token_id)}},RA=EA;function GA(e){if(e.length===0)throw new Error("Array must not be empty");let t=e[0],r=0;for(let n=1;n<e.length;++n)e[n]<t&&(t=e[n],r=n);return[t,r]}var LA=class extends jo{constructor(e,t){super(e);let r=e.vocab.length;this.vocab=new Array(r),this.scores=new Array(r);for(let n=0;n<r;++n)[this.vocab[n],this.scores[n]]=e.vocab[n];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((n,a)=>[n,a])),this.bos_token=" ",this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=GA(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new BA,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,r=1,n=0;for(;n<t.length;){let a=!1,s=[],o=t.slice(n).join(""),i=this.trie.common_prefix_search(o);for(let u of i){s.push(u);let l=this.tokens_to_ids.get(u),c=this.scores[l],d=xy(u);e.insert(n,d,c,l),!a&&d===r&&(a=!0)}a||e.insert(n,r,this.unk_score,this.unk_token_id),n+=r}}tokenize(e){let t=new RA(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let r of e){let n=this.tokenize(r);t.push(...n)}return t}},s2=LA,qA=class{constructor(e=(r,n)=>r>n,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let r=this._smallest();this._comparator(t,this._heap[r])&&(this._heap[r]=t,this._sift_up_from(r))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let r=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=r}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},OA=qA,DA=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},MA=DA,WA=class extends jo{constructor(e){super(e),this.tokens_to_ids=Gl(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=new Array(this.tokens_to_ids.size);for(let[r,n]of this.tokens_to_ids)this.vocab[n]=r;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(r=>r.split(" ",2)),this.bpe_ranks=new Map(this.merges.map((r,n)=>[JSON.stringify(r),n])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new MA(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let r=Array.from(e);this.end_of_word_suffix&&(r[r.length-1]+=this.end_of_word_suffix);let n=[];if(r.length>1){let a=new OA((i,u)=>i.score<u.score),s={token:r[0],bias:0,prev:null,next:null},o=s;for(let i=1;i<r.length;++i){let u={bias:i/r.length,token:r[i],prev:o,next:null};o.next=u,this.add_node(a,o),o=u}for(;!a.is_empty();){let i=a.pop();if(i.deleted||!i.next||i.next.deleted)continue;if(i.deleted=!0,i.next.deleted=!0,i.prev){let l={...i.prev};i.prev.deleted=!0,i.prev=l,l.prev?l.prev.next=l:s=l}let u={token:i.token+i.next.token,bias:i.bias,prev:i.prev,next:i.next.next};u.prev?(u.prev.next=u,this.add_node(a,u.prev)):s=u,u.next&&(u.next.prev=u,this.add_node(a,u))}for(let i=s;i!==null;i=i.next)n.push(i.token)}else n=r;if(this.continuing_subword_suffix)for(let a=0;a<n.length-1;++a)n[a]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,n),n}add_node(e,t){let r=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));r!==void 0&&(t.score=r+t.bias,e.push(t))}encode(e){let t=[];for(let r of e){if(this.ignore_merges&&this.tokens_to_ids.has(r)){t.push(r);continue}let n=this.bpe(r);for(let a of n)if(this.tokens_to_ids.has(a))t.push(a);else if(this.byte_fallback){let s=Array.from(this.text_encoder.encode(a)).map(o=>`<0x${o.toString(16).toUpperCase().padStart(2,"0")}>`);s.every(o=>this.tokens_to_ids.has(o))?t.push(...s):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},o2=WA,FA=class extends jo{constructor(e,t){super(e);let r=e.vocab;this.tokens_to_ids=Gl(t.target_lang?r[t.target_lang]:r),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=new Array(this.tokens_to_ids.size);for(let[n,a]of this.tokens_to_ids)this.vocab[a]=n}encode(e){return e}},QA=FA;function jA(e,t){switch(e.type){case"WordPiece":return new n2(e);case"Unigram":return new s2(e,t.eos_token);case"BPE":return new o2(e);default:if(e.vocab)return Array.isArray(e.vocab)?new s2(e,t.eos_token):Object.hasOwn(e,"continuing_subword_prefix")&&Object.hasOwn(e,"unk_token")?Object.hasOwn(e,"merges")?new o2(e):new n2(e):new QA(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw new Error(`Unknown TokenizerModel type: ${e?.type}`)}}var NA=jA,zA=class extends as{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},ss=zA,KA=class extends ss{post_process(e,t=null,r=!0){let n=t===null?this.config.single:this.config.pair,a=[],s=[];for(let o of n)"SpecialToken"in o?r&&(a.push(o.SpecialToken.id),s.push(o.SpecialToken.type_id)):"Sequence"in o&&(o.Sequence.id==="A"?(a=Vt(a,e),s=Vt(s,new Array(e.length).fill(o.Sequence.type_id))):o.Sequence.id==="B"&&(a=Vt(a,t),s=Vt(s,new Array(t.length).fill(o.Sequence.type_id))));return{tokens:a,token_type_ids:s}}},UA=KA,$A=class extends ss{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},HA=$A,VA=class extends ss{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,r=!0){r&&(e=Vt([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let a=[],s=r?[this.sep[0]]:[];e=Vt(e,a,t,s),n=Vt(n,new Array(t.length+a.length+s.length).fill(1))}return{tokens:e,token_type_ids:n}}},ZA=VA,XA=class extends ss{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,r=!0){r&&(e=Vt([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let a=r?[this.sep[0]]:[],s=r?[this.sep[0]]:[];e=Vt(e,a,t,s),n=Vt(n,new Array(t.length+a.length+s.length).fill(1))}return{tokens:e,token_type_ids:n}}},YA=XA,JA=class extends ss{constructor(e){super(e),this.processors=(e.processors??[]).map(t=>f2(t))}post_process(e,t=null,r=!0){let n={tokens:e,tokens_pair:t};for(let a of this.processors)n=a.post_process(n.tokens,n.tokens_pair,r);return n}},ek=JA;function tk(e){if(e===null)return null;switch(e.type){case"TemplateProcessing":return new UA(e);case"ByteLevel":return new HA(e);case"BertProcessing":return new ZA(e);case"RobertaProcessing":return new YA(e);case"Sequence":return new ek(e);default:throw new Error(`Unknown PostProcessor type: ${e.type}`)}}var f2=tk,rk=class extends as{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets="trim_offsets"in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join("")}},dr=rk,nk=class extends dr{constructor(e){super(e),this.byte_decoder=vy,this.text_decoder=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(""),r=new Uint8Array([...t].map(n=>this.byte_decoder[n]));return this.text_decoder.decode(r)}decode_chain(e){let t=[],r=[];for(let n of e)this.added_tokens.find(a=>a.content===n)!==void 0?(r.length>0&&(t.push(this.convert_tokens_to_string(r)),r=[]),t.push(n)):r.push(n);return r.length>0&&t.push(this.convert_tokens_to_string(r)),t}},ak=nk,sk=class extends dr{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((t,r)=>{if(r!==0){let n=this.config.prefix;n&&t.startsWith(n)?t=t.replace(n,""):t=" "+t}return this.cleanup&&(t=Rl(t)),t})}},ok=sk,ik=class extends dr{constructor(e){super(e),this.replacement=e.replacement??"\u2581"}decode_chain(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r].replaceAll(this.replacement," ");r==0&&n.startsWith(" ")&&(n=n.substring(1)),t.push(n)}return t}},uk=ik,lk=class extends dr{constructor(e){super(e),this.suffix=e.suffix??""}decode_chain(e){return e.map((t,r)=>t.replaceAll(this.suffix,r===e.length-1?"":" "))}},ck=lk,dk=class extends dr{constructor(e){super(e),this.pad_token=e.pad_token??"",this.word_delimiter_token=e.word_delimiter_token??"",this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return"";let t=[e[0]];for(let a=1;a<e.length;++a)e[a]!==t.at(-1)&&t.push(e[a]);let n=t.filter(a=>a!==this.pad_token).join("");return this.cleanup&&(n=Rl(n).replaceAll(this.word_delimiter_token," ").trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},pk=dk,fk=class extends dr{constructor(e){super(e),this.decoders=(e.decoders??[]).map(t=>m2(t))}decode_chain(e){return this.decoders.reduce((t,r)=>r.decode_chain(t),e)}},mk=fk,gk=class extends dr{decode_chain(e){let t=Fo(this.config.pattern),r=this.config.content??"";return t===null?e:e.map(n=>n.replaceAll(t,r))}},hk=gk,wk=class extends dr{decode_chain(e){return[e.join("")]}},bk=wk,vk=class extends dr{constructor(e){super(e),this.content=e.content??"",this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(t=>{let r=0;for(let a=0;a<this.start&&t[a]===this.content;++a){r=a+1;continue}let n=t.length;for(let a=0;a<this.stop;++a){let s=t.length-a-1;if(t[s]===this.content){n=s;continue}else break}return t.slice(r,n)})}},_k=vk,yk=class extends dr{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],r=[];for(let n of e){let a=null;if(n.length===6&&n.startsWith("<0x")&&n.endsWith(">")){let s=parseInt(n.slice(3,5),16);isNaN(s)||(a=s)}if(a!==null)r.push(a);else{if(r.length>0){let s=this.text_decoder.decode(Uint8Array.from(r));t.push(s),r=[]}t.push(n)}}if(r.length>0){let n=this.text_decoder.decode(Uint8Array.from(r));t.push(n),r=[]}return t}},Ak=yk;function kk(e){if(e===null)return null;switch(e.type){case"ByteLevel":return new ak(e);case"WordPiece":return new ok(e);case"Metaspace":return new uk(e);case"BPEDecoder":return new ck(e);case"CTC":return new pk(e);case"Sequence":return new mk(e);case"Replace":return new hk(e);case"Fuse":return new bk(e);case"Strip":return new _k(e);case"ByteFallback":return new Ak(e);default:throw new Error(`Unknown Decoder type: ${e.type}`)}}var m2=kk,Sk=class{constructor(e,t){let r=r2(e,"Tokenizer",["model","decoder","post_processor","pre_tokenizer","normalizer"]);if(r)throw new Error(r);let n=r2(t,"Config");if(n)throw new Error(n);this.tokenizer=e,this.config=t,this.normalizer=l2(this.tokenizer.normalizer),this.pre_tokenizer=c2(this.tokenizer.pre_tokenizer),this.model=NA(this.tokenizer.model,this.config),this.post_processor=f2(this.tokenizer.post_processor),this.decoder=m2(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let a=[],s=[];this.added_tokens_map=new Map;for(let o of this.tokenizer.added_tokens){let i=new wy(o);if(this.added_tokens.push(i),this.model.tokens_to_ids.set(i.content,i.id),this.model.vocab[i.id]=i.content,i.special&&(this.special_tokens.push(i.content),this.all_special_ids.push(i.id)),this.added_tokens_map.set(i.content,i),i.normalized&&this.normalizer!==null){let u=this.normalizer(i.content);s.push(u),this.added_tokens_map.set(u,i)}else a.push(i.content)}(this.config.additional_special_tokens??[]).forEach(o=>{this.special_tokens.includes(o)||this.special_tokens.push(o)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new e2(a),this.splitter_normalized=new e2(s),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:r=!0,return_token_type_ids:n=null}={}){let{tokens:a,token_type_ids:s}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}),o=a.map(u=>this.added_tokens_map.get(u)?.id??this.model.tokens_to_ids.get(u)??this.model.unk_token_id),i={ids:o,tokens:a,attention_mask:new Array(o.length).fill(1)};return n&&s&&(i.token_type_ids=s),i}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Sy(e[0]))throw Error("token_ids must be a non-empty array of integers.");let r=e.map(a=>this.model.vocab[Number(a)]??this.model.unk_token);t.skip_special_tokens&&(r=r.filter(a=>!this.special_tokens.includes(a)));let n=this.decoder?this.decoder(r):r.join(" ");return this.decoder&&this.decoder.end_of_word_suffix&&(n=n.replaceAll(this.decoder.end_of_word_suffix," "),t.skip_special_tokens&&(n=n.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(n=Rl(n)),n}tokenize(e,{text_pair:t=null,add_special_tokens:r=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((r,n)=>{let a=this.added_tokens_map.get(r);a&&(a.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),a.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((r,n)=>{if(r.length===0)return[];if(this.added_tokens_map.has(r))return[r];if(this.remove_space===!0&&(r=r.trim().split(/\s+/).join(" ")),this.do_lowercase_and_remove_accent&&(r=Ty(r)),this.normalizer!==null&&(r=this.normalizer(r)),r.length===0)return[];let a=this.splitter_normalized.split(r);return a.forEach((s,o)=>{let i=this.added_tokens_map.get(s);i&&(i.lstrip&&o>0&&(a[o-1]=a[o-1].trimEnd()),i.rstrip&&o<a.length-1&&(a[o+1]=a[o+1].trimStart()))}),a.flatMap(s=>{if(s.length===0)return[];if(this.added_tokens_map.has(s))return[s];let o=this.pre_tokenizer!==null?this.pre_tokenizer(s,{section_index:n}):[s];return this.model(o)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:r=!0}){let n=this.encode_text(e),a=this.encode_text(t||null);return this.post_processor?this.post_processor(n,a,r):{tokens:Vt(n??[],a??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let r=0;r<this.model.vocab.length;++r){let n=this.model.vocab[r];(e||!this.added_tokens_map.has(n))&&t.set(n,r)}return t}},g2=Sk;function Ll(e){return typeof e=="string"?e.trim():""}function os(e,t){try{return e[t]}catch{return}}function w2(e){let t=os(e,"constructor");return typeof t=="object"||typeof t=="function"?Ll(os(t,"name")):""}function xk(e){let t=Ll(os(e,"name"));if(t&&t!=="Error")return t;let r=w2(e);return r&&r!=="Error"&&r!=="Object"?r:""}function Er(e){if(e===null)return"null thrown";if(e===void 0)return"undefined thrown";if(typeof e=="object"){let t=os(e,"message"),r=typeof t=="string"?t:"",n=xk(e);if(r.trim())return n&&!r.startsWith(n)?`${n}: ${r}`:r;let a=n||w2(e)||"unknown error",s=os(e,"cause"),o=s===void 0?"":Er(s);if(o&&o!==a)return`${a} (no message; caused by ${o})`;if(!(e instanceof Error)){let i=h2(e);if(i&&i!=="[object Object]"&&i!==a)return`${a}: ${i}`}return`${a} (no message)`}return h2(e)||`${typeof e} thrown`}function h2(e){try{return Ll(String(e))}catch{return""}}var Tk=jr,Ck=Object.freeze({"subgroup-size-control":["subgroups"],"chromium-experimental-subgroup-matrix":["subgroups"]});function _2(e,t=[]){let r=new Set(y2(t));for(let[n,a]of Object.entries(Ck))a.some(s=>r.has(s))&&r.add(n);return Tk.filter(n=>e.features.has(n)&&!r.has(n)).map(n=>n)}function y2(e){return typeof e=="string"?e.split(",").map(t=>t.trim()).filter(Boolean):Array.isArray(e)?[...e]:e instanceof Set?[...e]:[]}function Pk(e,t){if(!t||Object.keys(t).length===0)return e;let r=e,n={...r};for(let[a,s]of Object.entries(t)){if(!Object.hasOwn(r,a))throw new Error(`limit override names an unknown limit: ${a}`);let o=r[a];if(typeof o=="number"){if(s>o)throw new Error(`limit override ${a}=${s} exceeds what the adapter reports (${o}); overrides only lower a limit`);n[a]=s}}return n}function A2(e,t=!1){return async r=>{e.pushErrorScope("validation");let n=null;try{let s={layout:"auto",compute:{module:e.createShaderModule({code:r}),entryPoint:"main"}};t?e.createComputePipeline(s):await e.createComputePipelineAsync(s)}catch(a){n=a instanceof Error?a.message:String(a)}try{let a=await e.popErrorScope();n===null&&a!==null&&(n=a.message)}catch(a){n===null&&(n=a instanceof Error?a.message:String(a))}return n}}function k2(e,t=!1){let r=Ik(e.lost);return async n=>{if(r.failure!==null)return r.failure;let a=null,s=0;try{for(let l of ql)e.pushErrorScope(l),s+=1;let u={layout:"auto",compute:{module:e.createShaderModule({code:n}),entryPoint:"main"}};t?e.createComputePipeline(u):await e.createComputePipelineAsync(u)}catch(i){a=Er(i)}let o=[];for(let i=s-1;i>=0;i-=1)try{o.push(e.popErrorScope())}catch(u){a??=Er(u)}for(let i of await Promise.allSettled(o))i.status==="rejected"?a??=Er(i.reason):i.value!==null&&(a??=i.value.message);return a??r.failure}}function Ik(e){let t={failure:null};return e?.then(r=>{t.failure=`WebGPU device lost during capability probe: ${r.message}`},r=>{t.failure=`WebGPU device loss state unavailable: ${Er(r)}`}),t}function S2(e,t,r,n,a,s,o,i){let u=t.info,l=Bk(r?.wgslLanguageFeatures??globalThis.navigator?.gpu?.wgslLanguageFeatures,a),c=n?n.min:u.subgroupMinSize,d=n?n.max:u.subgroupMaxSize,p=i===void 0?na(u.subgroupMatrixConfigs):i;return{adapterInfo:{vendor:u.vendor,architecture:u.architecture,device:u.device,description:u.description,isFallbackAdapter:u.isFallbackAdapter===!0,...typeof c=="number"?{subgroupMinSize:c}:{},...typeof d=="number"?{subgroupMaxSize:d}:{},...p!==null?{subgroupMatrixConfigs:p}:{}},features:ra(e.features),wgslLanguageFeatures:za(l),wgslCompileFailures:Bu(o),limits:Pk($s(e.limits),s)}}function Bk(e,t){let r=y2(t);if(r.length===0||e===null||e===void 0)return e;let n=new Set(r);return Array.from(za(e)).filter(a=>!n.has(a))}function x2(e){return{maxBufferSize:Number(e.limits.maxBufferSize),maxStorageBufferBindingSize:Number(e.limits.maxStorageBufferBindingSize),maxStorageBuffersPerShaderStage:Number(e.limits.maxStorageBuffersPerShaderStage),maxComputeWorkgroupStorageSize:Number(e.limits.maxComputeWorkgroupStorageSize),...b2(e,"maxComputeInvocationsPerWorkgroup"),...b2(e,"maxComputeWorkgroupSizeX")}}function b2(e,t){let r=Number(e.limits[t]);return Number.isFinite(r)?{[t]:r}:{}}function v2(e){return e instanceof Error?e.message:String(e)}var ql=["validation","out-of-memory","internal"];function Ek(e){return t=>{e.push(Er(t.error))}}function T2({adapter:e,device:t,deviceInfo:r,destroy:n,gpu:a,syncPipelineCreation:s,diagnosticSink:o,rejectUncapturedErrors:i}){let u=[];i&&t.addEventListener("uncapturederror",Ek(u));let l=()=>{if(u.length===0)return;let[b,...v]=u.splice(0);throw new Error(`WebGPU dropped work after an error no error scope captured: ${b}`+(v.length>0?`
(${v.length} more uncaptured error(s) followed, usually its cascade)`:""))},c=new WeakMap,d=b=>{if(o)try{o(b)}catch{}},p=s?async b=>{for(let B of ql)t.pushErrorScope(B);let v,y;try{v=t.createComputePipeline(b)}catch(B){y=B}let S=[],C;for(let B=ql.length-1;B>=0;B-=1)try{S.push(t.popErrorScope())}catch(P){C??=P}let k=[];for(let B of await Promise.allSettled(S))B.status==="rejected"?C??=B.reason:B.value!==null&&k.push(B.value);if(y!==void 0)throw y;if(C!==void 0)throw C;if(k.length>0)throw new Error(`createComputePipeline failed: ${k.map(B=>B.message).join(`
`)}`);return v}:b=>t.createComputePipelineAsync(b),f=async b=>{try{return await p(b)}catch(v){if(o){let y=b.compute.module,S=c.get(y),C=v2(v),k=y.getCompilationInfo;if(typeof k=="function")try{let B=await k.call(y);if(B.messages.length>0){let P=B.messages.map(_=>`${_.type} ${_.lineNum}:${_.linePos}: ${_.message}`);C=`${C}
${P.join(`
`)}`}}catch{}d({stage:"build",message:C,...b.label||S?.label?{label:b.label??S?.label}:{},...S?{shaderSource:S.source}:{}})}throw v}},m=0,h=0,g=new Map;return{gpu:a,adapter:e,device:t,deviceInfo:r,createShaderModule:(b,v)=>{try{let y=t.createShaderModule({code:b,label:v});return c.set(y,{source:b,...v?{label:v}:{}}),y}catch(y){throw d({stage:"build",message:v2(y),...v?{label:v}:{},shaderSource:b}),y}},createComputePipeline:f,createBuffer:b=>{let v=r.limits?.maxBufferSize,y=Number(b.size);if(typeof v=="number"&&y>v)throw new Error(`WebGPU buffer${b.label?` "${b.label}"`:""} of ${uo(y,1)} exceeds this device's maxBufferSize of ${uo(v,1)}. This allocation is too large for this GPU/browser.`);let S=t.createBuffer(b);m+=y,m>h&&(h=m);let C=(b.label??"(unlabelled)").replace(/\d+/g,"#"),k=g.get(C);k?(k.bytes+=y,k.count+=1):g.set(C,{bytes:y,count:1});let B=S.destroy.bind(S),P=!0;return S.destroy=()=>{if(P){P=!1,m-=y;let _=g.get(C);_.bytes-=y,_.count-=1,_.count===0&&g.delete(C)}B()},S},memory:{get liveBytes(){return m},get peakBytes(){return h},markPeak(){h=m},census(b=8){return[...g.entries()].map(([v,{bytes:y,count:S}])=>({label:v,bytes:y,count:S})).sort((v,y)=>y.bytes-v.bytes).slice(0,b)}},writeBuffer:(b,v,y)=>t.queue.writeBuffer(b,v,y),submit:async(b,v)=>{t.queue.submit(b),v.wait&&(await t.queue.onSubmittedWorkDone(),l())},mapRead:async(b,v,y)=>{await b.mapAsync(GPUMapMode.READ,v,y);let S=b.getMappedRange(v,y).slice(0);return b.unmap(),l(),S},destroy:i?async()=>{await n(),l()}:n}}var Ml=[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f16",resultComponentType:"f16",M:8,N:8,K:8}];function C2(e,t){return e.componentType===t.componentType&&e.resultComponentType===t.resultComponentType&&e.M===t.M&&e.N===t.N&&e.K===t.K}function Wl(e){return Ml.some(t=>C2(e,t))}function Ol(e){return e==="i8"?"i32":e==="u8"?"u32":e}function Dl(e,t){return e==="i8"||e==="u8"?Math.ceil(t/4):t}function Rk(e){let{componentType:t,resultComponentType:r,M:n,N:a,K:s}=e,o=Dl(t,s),i=Dl(t,a),u=Dl(r,a);return`${t==="f16"||r==="f16"?`enable f16;
`:""}enable subgroups;
enable chromium_experimental_subgroup_matrix;
diagnostic(off, chromium.subgroup_matrix_uniformity);
@group(0) @binding(0) var<storage, read> lhs_values: array<${Ol(t)}>;
@group(0) @binding(1) var<storage, read> rhs_values: array<${Ol(t)}>;
@group(0) @binding(2) var<storage, read_write> output: array<${Ol(r)}>;
@compute @workgroup_size(128, 1, 1)
fn main(@builtin(local_invocation_index) tid: u32) {
  let lhs = subgroupMatrixLoad<subgroup_matrix_left<${t}, ${s}, ${n}>, row_major>(&lhs_values, 0u, ${o}u);
  let rhs = subgroupMatrixLoad<subgroup_matrix_right<${t}, ${a}, ${s}>, row_major>(&rhs_values, 0u, ${i}u);
  var accumulator: subgroup_matrix_result<${r}, ${a}, ${n}>;
  let product = subgroupMatrixMultiplyAccumulate(lhs, rhs, accumulator);
  let bank = subgroupBroadcastFirst(tid);
  subgroupMatrixStore<row_major>(&output, bank * ${n*u}u, product, ${u}u);
}
`}function Gk(e){if(!Wl(e))return Rk(e);let t=e.componentType,r=e.resultComponentType;return`${t==="f16"?`enable f16;
`:""}enable subgroups;
enable chromium_experimental_subgroup_matrix;
diagnostic(off, chromium.subgroup_matrix_uniformity);
@group(0) @binding(0) var<storage, read_write> output: array<${r}>;
var<workgroup> lhs_values: array<${t}, 64>;
var<workgroup> rhs_values: array<${t}, 64>;
@compute @workgroup_size(128, 1, 1)
fn main(@builtin(local_invocation_index) tid: u32) {
  if (tid < 64u) {
    lhs_values[tid] = ${t==="f16"?"1.0h":"1.0f"};
    rhs_values[tid] = ${t==="f16"?"1.0h":"1.0f"};
  }
  workgroupBarrier();
  let lhs = subgroupMatrixLoad<subgroup_matrix_left<${t}, 8, 8>, row_major>(&lhs_values, 0u, 8u);
  let rhs = subgroupMatrixLoad<subgroup_matrix_right<${t}, 8, 8>, row_major>(&rhs_values, 0u, 8u);
  var accumulator: subgroup_matrix_result<${r}, 8, 8>;
  let product = subgroupMatrixMultiplyAccumulate(lhs, rhs, accumulator);
  let bank = subgroupBroadcastFirst(tid);
  subgroupMatrixStore<row_major>(&output, bank * 64u, product, 8u);
}
`}async function P2(e,t,r){let n=ra(e),a=na(t);if(!n.has("chromium-experimental-subgroup-matrix")||!n.has("subgroups"))return a;let s=Ml[0],o=a?.some(d=>d.componentType===s.componentType&&d.resultComponentType===s.resultComponentType&&d.M===8&&d.N===8&&d.K===8),i=a?.some(d=>d.componentType==="f16"&&d.resultComponentType==="f16"&&d.M===8&&d.N===8&&d.K===8),u=a===null?[...Ml]:a.filter(d=>!Wl(d));i&&!o&&u.push(s);let l=a?.filter(Wl)??[],c=[];for(let d of u)if(!c.some(p=>C2(p,d))&&(c.push(d),!((d.componentType==="f16"||d.resultComponentType==="f16")&&!n.has("shader-f16"))))try{await r(Gk(d))===null&&l.push(d)}catch{}return na(l)}function Lk(e){return Er(e)}function I2(e,t){if(e)try{e(t)}catch{}}function qk(e,t){e.addEventListener("uncapturederror",r=>{t?I2(t,{stage:"execute",message:`WebGPU uncaptured error: ${Lk(r.error)}`}):console.error("WebGPU uncaptured error:",r.error)}),t&&e.lost.then(r=>{r.reason!=="destroyed"&&I2(t,{stage:"execute",message:`WebGPU device lost (${r.reason}): ${r.message}`})})}async function B2(e={}){let t=globalThis.navigator?.gpu;if(!t)throw new Error("WebGPU is not available in this browser context");let r=await t.requestAdapter({powerPreference:e.powerPreference??"high-performance"});if(!r)throw new Error("No WebGPU adapter was returned");let n=await r.requestDevice({requiredFeatures:_2(r,e.disabledFeatures),requiredLimits:e.requiredLimits??x2(r),label:e.label??"webgpu-ml-runtime",...e.dawnToggles?{dawnToggles:e.dawnToggles}:{}});qk(n,e.diagnosticSink);let a=await Zf(A2(n)),s=await P2(n.features,r.info.subgroupMatrixConfigs,k2(n));return T2({gpu:t,adapter:r,device:n,deviceInfo:S2(n,r,t,null,e.disabledFeatures,null,a,s),diagnosticSink:e.diagnosticSink,destroy:()=>n.destroy()})}function Ok(e){return e===null||typeof e!="object"?!1:"destroy"in e&&"mapAsync"in e&&"getMappedRange"in e&&!("buffer"in e)}function da(e){return Ok(e)?e:e.buffer}function Fl(e){return da(e.type==="uniform"?e.buffer:e.tensor)}function Hr(e,t,r,n){let a=da(e),s=e,o=s.byteOffset??0;if(!Number.isSafeInteger(o)||o<0)throw new Error(`${n} tensor view has an invalid byteOffset ${o}`);t??=o;for(let[u,l]of[["offset",t],["byteLength",r]])if(!Number.isSafeInteger(l)||l<0||l%4!==0)throw new Error(`${n} ${u} must be a non-negative multiple of 4, got ${l}`);let i=t+r;if(!Number.isSafeInteger(i))throw new Error(`${n} byte range exceeds safe integer storage`);if(typeof s.byteLength=="number"){if(!Number.isSafeInteger(s.byteLength)||s.byteLength<0)throw new Error(`${n} tensor view has an invalid byteLength ${s.byteLength}`);let u=at(s.byteLength,4);if(t<o||i>o+u)throw new Error(`${n} range [${t}, ${i}] exceeds tensor view [${o}, ${o+u}]`)}if(typeof a.size=="number"&&i>a.size)throw new Error(`${n} range [${t}, ${i}] exceeds backing buffer size ${a.size}`);return{buffer:a,offset:t}}var No=class{pipelines=new Map;maxPipelineEntries=Number.POSITIVE_INFINITY;#e=new Map;#t=new Map;physicalPipelines=new Map;pipelineLayouts=new Map;bindGroupLayouts=new Map;bindGroups=new Map;maxBindGroupEntries=4096;bufferIds=new WeakMap;#n=1;captureShaders=!1;capturedShaders=new Map;compileCount=0;compileTotalMs=0;#r=0;#a;constructor(t){this.#a=t}async buildPipeline(t){let r=this.pipelines.get(t.cacheKey);if(r)return this.maxPipelineEntries!==Number.POSITIVE_INFINITY&&(this.pipelines.delete(t.cacheKey),this.pipelines.set(t.cacheKey,r)),r;let n=this.#e.get(t.cacheKey);if(n)return n.promise;let a=this.#r,s=this.#s(t,a),o={generation:a,promise:s};this.#e.set(t.cacheKey,o);try{return await s}finally{this.#e.get(t.cacheKey)===o&&this.#e.delete(t.cacheKey)}}async#s({cacheKey:t,name:r,op:n,kernelId:a,entryPoint:s,layoutSignature:o,layoutFactory:i,sourceFactory:u},l){let c=await u();if(l!==this.#r)throw new Error(`Pipeline build ${t} was invalidated by clearPrograms()`);if(this.captureShaders){let p=a??r;this.capturedShaders.has(p)||this.capturedShaders.set(p,{name:r,source:c,...n===void 0?{}:{op:n},...a===void 0?{}:{kernelId:a}})}let d;if(o){let p=Dk(c,s,o);d=this.physicalPipeline(p,c,s,o)??await this.#o({key:p,source:c,entryPoint:s,layoutSignature:o,generation:l,name:r,layoutFactory:i})}else d=await this.#i({source:c,entryPoint:s,name:r,layoutFactory:i});return l===this.#r&&(this.pipelines.set(t,d),this.#l()),d}#l(){for(;this.pipelines.size>this.maxPipelineEntries;){let t=this.pipelines.entries().next();if(t.done===!0)return;let[r,n]=t.value;this.pipelines.delete(r);let a=!1;for(let s of this.pipelines.values())if(s===n){a=!0;break}a||this.#u(n)}}#u(t){for(let[r,n]of this.physicalPipelines){let a=n.findIndex(s=>s.pipeline===t);if(!(a<0)){n.splice(a,1),n.length===0&&this.physicalPipelines.delete(r);return}}}async#o({key:t,source:r,entryPoint:n,layoutSignature:a,generation:s,name:o,layoutFactory:i}){let u=this.#c(t,r,n,a,s);if(u)return u.promise;let l=this.#i({source:r,entryPoint:n,name:o,layoutFactory:i}),c={key:t,source:r,entryPoint:n,layoutSignature:a,generation:s,promise:l};this.#d(c);try{let d=await l;return s===this.#r&&this.setPhysicalPipeline(t,r,n,a,d),d}finally{this.#p(c)}}async#i({source:t,entryPoint:r,name:n,layoutFactory:a}){let s=performance.now(),o=this.#a.createShaderModule(t,n),i=a(),u=await this.#a.createComputePipeline({label:n,layout:i,compute:{module:o,entryPoint:r}});return this.compileCount+=1,this.compileTotalMs+=performance.now()-s,u}cachedBindGroups({name:t,cacheKey:r,pipeline:n,bindings:a}){return{bindGroup:this.buildGroupBindGroup(t,r,n,a)}}buildGroupBindGroup(t,r,n,a){let s=am(r,a,i=>this.bufferId(Fl(i))),o=this.bindGroups.get(s);if(o===void 0&&(o=this.createBindGroup(t,n,a),this.bindGroups.set(s,o),this.bindGroups.size>this.maxBindGroupEntries)){let i=this.bindGroups.keys().next().value;i!==void 0&&this.bindGroups.delete(i)}return o}createBindGroup(t,r,n){return this.#a.device.createBindGroup({label:`${t}-bind-group`,layout:r.getBindGroupLayout(0),entries:n.map((a,s)=>{let o={buffer:Fl(a),offset:a.offset??0};return a.size!==void 0&&a.size>0&&(o.size=a.size),{binding:a.binding??s,resource:o}})})}bufferId(t){let r=this.bufferIds.get(t);return r===void 0&&(r=this.#n++,this.bufferIds.set(t,r)),r}pipelineLayout(t,r,n=0){let a=E2(t,r,n),s=this.pipelineLayouts.get(a.signature);if(s)return s;let o=this.#a.device.createPipelineLayout({label:`${t}-layout`,bindGroupLayouts:[this.bindGroupLayout(t,a.entries)],...n>0?{immediateSize:n}:{}});return this.pipelineLayouts.set(a.signature,o),o}pipelineLayoutSignature(t,r,n=0){return E2(t,r,n).signature}bindGroupLayout(t,r){let n=r.map(i=>({binding:i.binding,visibility:GPUShaderStage.COMPUTE,buffer:{type:i.type}})),a=n.map(i=>`${i.binding}:${i.buffer.type}`).sort().join("|"),s=this.bindGroupLayouts.get(a);if(s)return s;let o=this.#a.device.createBindGroupLayout({label:`${t}-bgl0`,entries:n});return this.bindGroupLayouts.set(a,o),o}clearBindGroups(){let t=this.bindGroups.size;return this.bindGroups.clear(),t}clearPrograms(){let t=this.pipelines.size;return this.#r+=1,this.#e.clear(),this.#t.clear(),this.bindGroups.clear(),this.pipelines.clear(),this.physicalPipelines.clear(),this.pipelineLayouts.clear(),this.bindGroupLayouts.clear(),this.captureShaders||this.capturedShaders.clear(),t}getRenderedShaders(){return[...this.capturedShaders.values()]}physicalPipeline(t,r,n,a){return this.physicalPipelines.get(t)?.find(o=>o.entryPoint===n&&o.layoutSignature===a&&o.source===r)?.pipeline}#c(t,r,n,a,s){return this.#t.get(t)?.find(i=>i.generation===s&&i.entryPoint===n&&i.layoutSignature===a&&i.source===r)}#d(t){let r=this.#t.get(t.key);r?r.push(t):this.#t.set(t.key,[t])}#p(t){let r=this.#t.get(t.key);if(!r)return;let n=r.indexOf(t);n<0||(r.splice(n,1),r.length===0&&this.#t.delete(t.key))}setPhysicalPipeline(t,r,n,a,s){let o=this.physicalPipelines.get(t),i={source:r,entryPoint:n,layoutSignature:a,pipeline:s};o?o.push(i):this.physicalPipelines.set(t,[i])}};function E2(e,t,r=0){let n=t.map((i,u)=>typeof i=="string"?{binding:u,type:i}:{binding:i.binding??u,type:i.type}),a=new Set;for(let i of n){if(!Number.isInteger(i.binding)||i.binding<0)throw new Error(`pipeline layout ${e} has invalid binding index ${i.binding}`);if(a.has(i.binding))throw new Error(`pipeline layout ${e} has duplicate binding index ${i.binding}`);if(typeof i.type!="string"||i.type.length===0)throw new Error(`pipeline layout ${e} binding ${i.binding} is missing a buffer type`);a.add(i.binding)}let s=n.map(i=>`${i.binding}:${i.type}`).sort();return{signature:r>0?`${s.join("|")}|imm=${r}`:s.join("|"),entries:n}}function Dk(e,t,r){return`${Io(e)}|entry=${t}|layout=${r}`}var zo=class{pool=new Map;bytes=0;maxBytes;constructor(t=64*1024*1024){this.maxBytes=t}acquire(t,r){let n=this.pool.get(r);return n&&n.length>0?(this.bytes-=r,n.pop()):t.createBuffer({label:"tensor-readback",size:r,usage:GPUBufferUsage.COPY_DST|GPUBufferUsage.MAP_READ})}release(t,r){if(this.bytes+t>this.maxBytes){r.destroy();return}let n=this.pool.get(t);n||(n=[],this.pool.set(t,n)),n.push(r),this.bytes+=t}discard(t){try{t.destroy()}catch{}}clear(){let t=0;for(let r of this.pool.values())for(let n of r)n.destroy(),t++;return this.pool.clear(),this.bytes=0,t}};var Uo=64*1024,Mk=4;function Lt(e){return e!==void 0&&e.kind===void 0}var pr=class{runtime;dtype;shape;byteOffset;buffer;size;byteLength;layout;destroyed;constructor({runtime:t,dtype:r,shape:n,buffer:a,byteOffset:s=0,layout:o}){if(!Number.isSafeInteger(s)||s<0)throw new Error(`tensor byteOffset must be a nonnegative integer, got ${s}`);if(this.runtime=t,this.dtype=r,this.shape=n,this.byteOffset=s,this.buffer=a,this.size=ke(n),o!==void 0&&(Hs(r,"WebGpuTensor"),this.layout=o),this.byteLength=Ye(r,this.size,o),!Number.isSafeInteger(s+this.byteLength)||typeof a.size=="number"&&at(s+this.byteLength,4)>a.size)throw new Error(`tensor range (${s}+${this.byteLength} bytes) exceeds backing buffer capacity`);this.destroyed=!1}destroy(){this.destroyed||(this.buffer.destroy(),this.destroyed=!0)}},Ko=class{host;pipelines;readback;destroyed;mappedTensorUploadsAvailable;constructor({host:t}){this.host=t,this.pipelines=new No(t),this.readback=new zo,this.destroyed=!1,this.mappedTensorUploadsAvailable=!0}get device(){return this.host.deviceInfo}async destroy(){this.destroyed||(this.destroyed=!0,this.clearBindGroupCache(),this.clearReadbackPool(),await this.beforeHostDestroy(),await this.host.destroy())}async beforeHostDestroy(){}clearReadbackPool(){return this.readback.clear()}clearBindGroupCache(){return this.pipelines.clearBindGroups()}tensorFromTypedArray(t,r,n,a={}){let s=ke(r);if(n.length!==s)throw new Error(`tensor data length ${n.length} does not match shape element count ${s}`);let o=n;if(a.layout==="packed"){if(Hs(t,"tensorFromTypedArray"),!(n instanceof Uint32Array)&&!(n instanceof Uint8Array))throw new Error("A packed uint8 tensor uploads from a Uint32Array (widened) or a Uint8Array (bytes)");o=Ap(n)}else if(!Wk(t,n)){let l=Qe[t]?.arrayCtor;throw new Error(l===void 0?`unsupported tensor dtype ${t}`:`${t} tensor data must be a ${l.name}, not ${n.constructor.name}`)}let i={label:"tensor",size:ct(o.byteLength),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC};if(this.mappedTensorUploadsAvailable&&o.byteLength>=Uo){let l=null;try{l=this.host.createBuffer({...i,mappedAtCreation:!0});let c=l.getMappedRange();return new Uint8Array(c).set(new Uint8Array(o.buffer,o.byteOffset,o.byteLength)),l.unmap(),new pr({runtime:this,dtype:t,shape:r,buffer:l,layout:a.layout})}catch{try{l?.destroy()}catch{}this.mappedTensorUploadsAvailable=!1}}let u=this.host.createBuffer(i);try{let l=o;if(o.byteLength%4!==0){let c=new Uint8Array(ct(o.byteLength));c.set(new Uint8Array(o.buffer,o.byteOffset,o.byteLength)),l=c}return(o.byteLength>0||a.layout!=="packed")&&this.host.writeBuffer(u,0,l),new pr({runtime:this,dtype:t,shape:r,buffer:u,layout:a.layout})}catch(l){try{u.destroy()}catch{}throw l}}empty(t,r,n="tensor-output",a={}){if(!Ql(t))throw new Error(`Unsupported dtype: ${t}`);let s=Ye(t,ke(r),a.layout),o=this.host.createBuffer({label:n,size:ct(s),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_SRC|GPUBufferUsage.COPY_DST});return new pr({runtime:this,dtype:t,shape:r,buffer:o,layout:a.layout})}readTensor(t){if(t.byteLength===0){let f=Qe[t.dtype]?.arrayCtor;return f?Promise.resolve(new f(0)):Promise.reject(new Error(`Unsupported dtype: ${t.dtype}`))}let r=t.byteOffset??0,n=Math.floor(r/4)*4,a=r-n,s=ct(a+t.byteLength),o=Hr({buffer:t.buffer},n,s,"readback source"),i=this.readback.acquire(this.host,s),u=this.host.device.createCommandEncoder({label:"readTensor"});u.copyBufferToBuffer(o.buffer,o.offset,i,0,s),this.host.device.queue.submit([u.finish()]);let{dtype:l,byteLength:c,layout:d,size:p}=t;return(async()=>{let f;try{f=await this.host.mapRead(i,0,s)}catch(h){throw this.readback.discard(i),h}if(this.readback.release(s,i),(a!==0||s!==c)&&(f=f.slice(a,a+c)),d==="packed")return kp(new Uint32Array(f),p);let m=Qe[l]?.arrayCtor;if(!m)throw new Error(`Unsupported dtype: ${l}`);return new m(f)})()}async runProgram(t,r={}){await this.runProgramSequence([t],r)}async runProgramSequence(t,r={}){let n=await this.prepareProgramSteps(t,!1);for(let a of t)this.#e(a);await this.executePreparedProgramSequence(n,r)}#e(t){let r=`program ${t.name??t.kind??"compute"}`;if(this.destroyed)throw new Error(`${r}: runtime has been destroyed`);if(R2(t))this.#t(t.src,`${r} source`),this.#t(t.dst,`${r} destination`);else if(G2(t))this.#t(t.dst,`${r} destination`);else if(Array.isArray(t.bindings))for(let n of t.bindings)n.type!=="uniform"&&this.#t(n.tensor,`${r} binding ${n.binding}`)}#t(t,r){if(t instanceof pr){if(t.destroyed)throw new Error(`${r}: tensor has been destroyed`);if(t.runtime.destroyed)throw new Error(`${r}: tensor's owning runtime has been destroyed`)}}async prepareProgramSteps(t,r){if(!Array.isArray(t)||t.length===0)throw new Error("prepareProgramSequence requires at least one program");let n=new Array(t.length),a=0,s=!1,o=[],i=async()=>{for(;!s;){let u=a++;if(u>=t.length)return;try{n[u]=await this.#n(t[u],r)}catch(l){o.push({index:u,error:l}),s=!0}}};if(await Promise.all(Array.from({length:Math.min(Mk,t.length)},i)),o.length>0)throw o.sort((u,l)=>u.index-l.index),o[0].error;return n}async#n(t,r){if(this.#e(t),R2(t))return Qk(t);if(G2(t))return Fk(t);let{name:n,op:a,kernelId:s,source:o,entryPoint:i="main",cacheKey:u=n,bindings:l,dispatchWorkgroups:c,profile:d,submitAfter:p,immediateSize:f,immediateData:m}=t;if(typeof o!="string"||o.length===0)throw new Error("program requires WGSL source");if(!Array.isArray(l)||l.length===0)throw new Error("program requires bindings");if(!Array.isArray(c)||c.length!==3)throw new Error("program requires a 3D dispatchWorkgroups array");if(f!==void 0&&f>0&&m===void 0)throw new Error(`program ${n} reserves ${f} immediate bytes but supplied no immediate data`);let h=this.pipelines.pipelineLayoutSignature(n,l,f),g=await this.pipelines.buildPipeline({name:n,...a===void 0?{}:{op:a},...s===void 0?{}:{kernelId:s},entryPoint:i,cacheKey:u,layoutSignature:h,layoutFactory:()=>this.pipelines.pipelineLayout(n,l,f),sourceFactory:()=>o});this.#e(t);let w=r?this.pipelines.cachedBindGroups({name:n,cacheKey:u,pipeline:g,bindings:l}).bindGroup:this.pipelines.createBindGroup(n,g,l);return{pipeline:g,bindGroup:w,...m?{immediateData:m}:{},dispatchWorkgroups:c,name:n,cacheKey:u,profile:d,...p?{submitAfter:!0}:{}}}async executePreparedProgramSequence(t,r={}){if(!Array.isArray(t)||t.length===0)throw new Error("executePreparedProgramSequence requires at least one prepared step");await this.runMergedSteps(t,r.wait??!1)}createUniformU32(t,r){let n=new Uint32Array(t),a=this.host.createBuffer({label:r,size:n.byteLength,usage:GPUBufferUsage.UNIFORM|GPUBufferUsage.COPY_DST});return this.host.writeBuffer(a,0,n),a}async runMergedSteps(t,r){let n=pa(t),a=n.map(s=>{let o=this.host.device.createCommandEncoder({label:"compute-dispatch"});return fa(o,s),o.finish()});for(let s=0;s+1<a.length;++s)await this.host.submit([a[s]],{wait:!0});await this.host.submit([a[a.length-1]],{wait:r||jl(n[n.length-1])})}};function Wk(e,t){let r=Qe[e]?.arrayCtor;return r!==void 0&&t instanceof r}function Ql(e){return Qe[e]!==void 0}function R2(e){return e.kind==="copy"}function G2(e){return e.kind==="clear"}function Fk(e){if(!e.dst)throw new Error("clear step requires dst (buffer or tensor)");let t=e.dstOffset??0,r=e.byteLength;if(!Number.isInteger(t)||t<0||t%4!==0)throw new Error(`clear step offset must be a non-negative multiple of 4, got ${t}`);if(!Number.isInteger(r)||r<0||r%4!==0)throw new Error(`clear step byteLength must be a non-negative multiple of 4, got ${r}`);return{kind:"clear",dst:da(e.dst),dstOffset:t,byteLength:r,name:e.name??"clear"}}function Qk(e){if(!e.src||!e.dst)throw new Error("copy step requires src and dst (buffer or tensor)");let t=e.byteLength;if(t<=0)throw new Error("copy step requires positive integer byteLength");let r=Hr(e.src,e.srcOffset,t,"copy source"),n=Hr(e.dst,e.dstOffset,t,"copy destination");return{kind:"copy",src:r.buffer,dst:n.buffer,srcOffset:r.offset,dstOffset:n.offset,byteLength:t,name:e.name??"copy"}}function pa(e){let t=[],r=0;for(let n=0;n<e.length;++n){let a=e[n];Lt(a)&&a.submitAfter&&!Tt(a.dispatchWorkgroups)&&(t.push(e.slice(r,n+1)),r=n+1)}return r<e.length&&t.push(e.slice(r)),t}function jl(e){let t=e[e.length-1];return Lt(t)&&t.submitAfter===!0&&!Tt(t.dispatchWorkgroups)}function fa(e,t,r){let n=null,a=t.findIndex(c=>Lt(c)&&!Tt(c.dispatchWorkgroups)),s=a,o=!1;for(let c=a+1;c<t.length;++c){let d=t[c];Lt(d)?!Tt(d.dispatchWorkgroups)&&o&&(s=c,o=!1):o=!0}let i=null,u=null,l=()=>{n!==null&&(n.end(),n=null),i=null,u=null};for(let c=0;c<t.length;++c){let d=t[c];if(d.kind==="copy"){l(),d.byteLength>0&&e.copyBufferToBuffer(d.src,d.srcOffset,d.dst,d.dstOffset,d.byteLength);continue}if(d.kind==="clear"){l(),d.byteLength>0&&e.clearBuffer(d.dst,d.dstOffset,d.byteLength);continue}if(!Tt(d.dispatchWorkgroups)){if(n===null){let p=c===a?r?.beginningOfSequenceWriteIndex:void 0,f=c===s?r?.endOfSequenceWriteIndex:void 0,m=p!==void 0||f!==void 0;n=e.beginComputePass({label:"compute-sequence-pass",...m?{timestampWrites:{querySet:r.querySet,...p!==void 0?{beginningOfPassWriteIndex:p}:{},...f!==void 0?{endOfPassWriteIndex:f}:{}}}:{}})}d.pipeline!==i&&(n.setPipeline(d.pipeline),i=d.pipeline),d.bindGroup!==u&&(n.setBindGroup(0,d.bindGroup),u=d.bindGroup),d.immediateData&&$o(n,d.immediateData),n.dispatchWorkgroups(d.dispatchWorkgroups[0],d.dispatchWorkgroups[1],d.dispatchWorkgroups[2])}}l()}function $o(e,t){let r=e;if(typeof r.setImmediates!="function")throw new Error("this WebGPU implementation reports the immediate_address_space language feature but the compute pass encoder has no setImmediates()");r.setImmediates(0,t)}function Tt(e){return e[0]===0||e[1]===0||e[2]===0}var Ho=class extends Ko{profileSession=null;executionDiagnostics=null;pendingProfileRuns=[];#e=new Map;#t=Promise.resolve();#n=!1;get captureShaders(){return this.pipelines.captureShaders}set captureShaders(t){this.pipelines.captureShaders=t}getRenderedShaders(){return this.pipelines.getRenderedShaders()}async beforeHostDestroy(){await this.#t,this.#i()}clearKernelCaches(){return this.pipelines.clearPrograms()}tensorFromWriter(t,r,n){let a=Qe[t];if(!a)throw new Error(`Unsupported dtype: ${t}`);let s=ke(r),o=s*a.storageByteSize,i={label:"tensor",size:ct(o),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC,mappedAtCreation:!0};if(this.mappedTensorUploadsAvailable&&o>=Uo){let l=null;try{l=this.host.createBuffer(i)}catch{this.mappedTensorUploadsAvailable=!1}if(l){let c;try{c=new a.arrayCtor(l.getMappedRange(),0,s)}catch{this.mappedTensorUploadsAvailable=!1;try{l.destroy()}catch{}l=null}if(l){try{n(c)}catch(d){try{l.unmap()}catch{}try{l.destroy()}catch{}throw d}try{l.unmap()}catch(d){this.mappedTensorUploadsAvailable=!1;try{l.destroy()}catch{}throw d}return new pr({runtime:this,dtype:t,shape:r,buffer:l})}}}let u=new a.arrayCtor(s);return n(u),this.tensorFromTypedArray(t,r,u)}beginMappedTensorUpload(t,r,n="tensor"){let a=Qe[t];if(!a)throw new Error(`Unsupported dtype: ${t}`);let s=[...r],o=ke(s)*a.storageByteSize;if(!this.mappedTensorUploadsAvailable||o<Uo)return null;let i;try{i=this.host.createBuffer({label:n,size:ct(o),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC,mappedAtCreation:!0})}catch{return this.mappedTensorUploadsAvailable=!1,null}let u;try{let m=i.getMappedRange();if(m.byteLength<o)throw new Error(`Mapped tensor range has ${m.byteLength} bytes; expected at least ${o}`);u=new Uint8Array(m,0,o)}catch{this.mappedTensorUploadsAvailable=!1;try{i.destroy()}catch{}return null}let l="open",c=null,d=!1,p=()=>{d||(d=!0,i.destroy())},f=m=>{if(l!=="open")throw new Error(`Cannot ${m} mapped tensor upload after it was ${l}`)};return{destination:u,commit:()=>{f("commit");try{i.unmap()}catch(m){l="aborted",this.mappedTensorUploadsAvailable=!1;try{p()}catch{}throw m}try{c=new pr({runtime:this,dtype:t,shape:s,buffer:i})}catch(m){l="aborted";try{p()}catch{}throw m}return l="committed",c},rollback:m=>{if(m!==c)throw new Error("Cannot roll back a tensor that was not returned by this mapped upload");if(l!=="rolled back"){if(l!=="committed")throw new Error(`Cannot roll back mapped tensor upload after it was ${l}`);c.destroy(),l="rolled back"}},abort:()=>{if(l==="aborted")return;f("abort"),l="aborted";let m;try{i.unmap()}catch(h){this.mappedTensorUploadsAvailable=!1,m=h}try{p()}catch(h){if(m===void 0)throw h}if(m!==void 0)throw m}}}allocateWeightsBuffer({byteLength:t,dtype:r,shape:n,label:a="weights"}){if(!Ql(r))throw new Error(`Unsupported dtype: ${r}`);if(!Number.isSafeInteger(t)||t<0)throw new Error(`byteLength must be a nonnegative integer, got ${t}`);let s=Ye(r,ke(n));if(t<s)throw new Error(`weights tensor requires ${s} bytes, but byteLength is ${t}`);let o=this.host.createBuffer({label:a,size:ct(t),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC});return new pr({runtime:this,dtype:r,shape:n,buffer:o})}writeWeightsRange(t,r,n){if(!wt(t))throw new Error("writeWeightsRange expects a GPU tensor");if(!Number.isSafeInteger(r)||r<0)throw new Error(`byteOffset must be a nonnegative integer, got ${r}`);if(r+n.byteLength>t.byteLength)throw new Error(`write range [${r}, ${r+n.byteLength}] exceeds tensor byteLength ${t.byteLength}`);let a=Hr(t,(t.byteOffset??0)+r,n.byteLength,"weight write");n.byteLength>0&&this.host.writeBuffer(a.buffer,a.offset,n)}async copyBufferToBuffer({src:t,dst:r,srcOffset:n,dstOffset:a,byteLength:s,wait:o=!1}){let i=Hr(t,n,s,"copy source"),u=Hr(r,a,s,"copy destination");if(s===0)return;let l=this.host.device.createCommandEncoder({label:"copyBufferToBuffer"});l.copyBufferToBuffer(i.buffer,i.offset,u.buffer,u.offset,s),await this.host.submit([l.finish()],{wait:o})}clearBuffers(t){let r=null;for(let{dst:n,dstOffset:a=0,byteLength:s}of t){if(!Number.isInteger(a)||a<0||a%4!==0)throw new Error(`clear buffer offset must be a non-negative multiple of 4, got ${a}`);if(!Number.isInteger(s)||s<0||s%4!==0)throw new Error(`clear buffer byteLength must be a non-negative multiple of 4, got ${s}`);s!==0&&(r??=this.host.device.createCommandEncoder({label:"clear-buffers"}),r.clearBuffer(da(n),a,s))}r&&this.host.device.queue.submit([r.finish()])}async queueIdle(){await this.host.device.queue.onSubmittedWorkDone()}async prepareProgramSequence(t){return this.prepareProgramSteps(t,!0)}runMergedSteps(t,r){let n=this.executionDiagnostics;if(!n)return super.runMergedSteps(t,r);this.recordPreparedExecution(t,!0);let a=performance.now(),s=super.runMergedSteps(t,r);return n.cpuEncodeAndInitialSubmitMs+=performance.now()-a,s}recordPreparedExecution(t,r){let n=this.executionDiagnostics;if(!n)return;let a=pa(t);n.commandBuffers+=a.length,n.submissions+=a.length;for(let s of a){let o=!1;for(let i of s){if(!Lt(i)){o=!1;continue}Tt(i.dispatchWorkgroups)||(n.dispatches+=1,(!r||!o)&&(n.computePasses+=1),o=!0)}}}async executePreparedProgramSequence(t,r={}){if(!this.profileSession)return super.executePreparedProgramSequence(t,r);if(!Array.isArray(t)||t.length===0)throw new Error("executePreparedProgramSequence requires at least one prepared step");let n=r.wait??!1;await this.#s();try{await this._runSteps(t,{wait:n,mergePass:!1})}finally{this.#u()}}preparedProgramSequenceRequiresCompletionFences(t){return t.some(r=>Lt(r)&&r.submitAfter===!0&&!Tt(r.dispatchWorkgroups))}#r=8;#a=[];async#s(){for(;this.#r<=0;)await new Promise(t=>this.#a.push(t));this.#r-=1}#l(){return this.#r<=0?!1:(this.#r-=1,!0)}#u(){this.#r+=1,this.#a.shift()?.()}enqueuePreparedProgramSequence(t){if(!Array.isArray(t)||t.length===0)throw new Error("enqueuePreparedProgramSequence requires at least one prepared step");let r=pa(t);if(this.profileSession&&r.length===1&&this.#l()){let a=this._runSteps(t,{wait:!1,mergePass:!1});this.pendingProfileRuns.push(a.finally(()=>this.#u()));return}this.recordPreparedExecution(t,!0);let n=this.executionDiagnostics?performance.now():0;for(let a of r){let s=this.host.device.createCommandEncoder({label:"compute-dispatch"});fa(s,a),this.host.device.queue.submit([s.finish()])}this.executionDiagnostics&&(this.executionDiagnostics.cpuEncodeAndInitialSubmitMs+=performance.now()-n)}async flushDeferredProfile(){let t=this.pendingProfileRuns;this.pendingProfileRuns=[],await Promise.all(t)}async measurePreparedSequenceGpuMs(t,r=1,n){if(this.destroyed||!this.host.device.features.has("timestamp-query")||this.#n||!t.some(o=>Lt(o)&&!Tt(o.dispatchWorkgroups)))return null;let a=this.#t,s;this.#t=new Promise(o=>{s=o}),await a;try{if(this.destroyed||this.#n)return null;let o=this.#e.get(2)??this.#c(2),i=pa(t);if(i.length>1)return await this.#o(i,r,o,n);let u=this.host.device.createCommandEncoder({label:"gpu-time-measure"});if(t.some(d=>d.kind==="clear"||d.kind==="copy"))for(let d=0;d<r;++d)fa(u,t,{querySet:o.querySet,...d===0?{beginningOfSequenceWriteIndex:0}:{},...d===r-1?{endOfSequenceWriteIndex:1}:{}});else{let d=u.beginComputePass({label:"gpu-time-pass",timestampWrites:{querySet:o.querySet,beginningOfPassWriteIndex:0,endOfPassWriteIndex:1}}),p=!1,f=null,m=null;for(let h=0;h<r;++h)for(let g of t)!Lt(g)||Tt(g.dispatchWorkgroups)||(g.pipeline!==f&&(d.setPipeline(g.pipeline),f=g.pipeline),g.bindGroup!==m&&(d.setBindGroup(0,g.bindGroup),m=g.bindGroup),g.immediateData&&$o(d,g.immediateData),d.dispatchWorkgroups(g.dispatchWorkgroups[0],g.dispatchWorkgroups[1],g.dispatchWorkgroups[2]),p=!0);if(d.end(),!p)return null}u.resolveQuerySet(o.querySet,0,2,o.resolveBuffer,0),u.copyBufferToBuffer(o.resolveBuffer,0,o.readbackBuffer,0,16);let l=this.host.submit([u.finish()],{wait:!0});try{n?.()}catch{}await l;let c;try{c=await this.readTimestampBuffer(o.readbackBuffer,2)}catch{return this.#n=!0,this.#i(),null}return Number(c[1]-c[0])/1e6}finally{s()}}async measurePreparedSequenceGpuSamples(t,r,n){if(this.destroyed||!this.host.device.features.has("timestamp-query")||this.#n||!Number.isSafeInteger(r)||r<=0||!Number.isSafeInteger(n)||n<=1||this.preparedProgramSequenceRequiresCompletionFences(t)||!Nl(t))return null;let a=this.#t,s;this.#t=new Promise(o=>{s=o}),await a;try{if(this.destroyed||this.#n)return null;let o=n*2,i=this.#e.get(o)??this.#c(o);for(let c=0;c<n;++c)for(let d=0;d<r;++d){let p=this.host.device.createCommandEncoder({label:"gpu-time-sample-batch"});fa(p,t,{querySet:i.querySet,...d===0?{beginningOfSequenceWriteIndex:c*2}:{},...d===r-1?{endOfSequenceWriteIndex:c*2+1}:{}}),this.host.device.queue.submit([p.finish()])}let u=this.host.device.createCommandEncoder({label:"gpu-time-sample-batch-resolve"});u.resolveQuerySet(i.querySet,0,o,i.resolveBuffer,0),u.copyBufferToBuffer(i.resolveBuffer,0,i.readbackBuffer,0,o*8),await this.host.submit([u.finish()],{wait:!1});let l;try{l=await this.readTimestampBuffer(i.readbackBuffer,o)}catch{return this.#n=!0,this.#i(),null}return Array.from({length:n},(c,d)=>Number(l[d*2+1]-l[d*2])/1e6)}finally{s()}}async#o(t,r,n,a){if(!Number.isInteger(r)||r<=0)return null;let s=t.findIndex(Nl),o=-1;for(let l=t.length-1;l>=0;--l)if(Nl(t[l])){o=l;break}if(s<0||o<0)return null;let i=[];for(let l=0;l<r;++l)for(let c=0;c<t.length;++c){let d=this.host.device.createCommandEncoder({label:"gpu-time-measure-boundary"}),p=l===0&&c===s,f=l===r-1&&c===o;fa(d,t[c],{querySet:n.querySet,...p?{beginningOfSequenceWriteIndex:0}:{},...f?{endOfSequenceWriteIndex:1}:{}}),l===r-1&&c===t.length-1&&(d.resolveQuerySet(n.querySet,0,2,n.resolveBuffer,0),d.copyBufferToBuffer(n.resolveBuffer,0,n.readbackBuffer,0,16)),i.push(d.finish())}for(let l=0;l<i.length;++l){let c=this.host.submit([i[l]],{wait:!0});if(l===i.length-1)try{a?.()}catch{}await c}let u;try{u=await this.readTimestampBuffer(n.readbackBuffer,2)}catch{return this.#n=!0,this.#i(),null}return Number(u[1]-u[0])/1e6}#i(){for(let t of this.#e.values())t.querySet.destroy(),t.resolveBuffer.destroy(),t.readbackBuffer.destroy();this.#e.clear()}#c(t){let r=this.createTimestampResources(t);return this.#e.set(t,r),r}pipelineCompileStats(){return{count:this.pipelines.compileCount,totalMs:this.pipelines.compileTotalMs}}memoryStats(){let t=this.host.memory;return t?{liveBytes:t.liveBytes,peakBytes:t.peakBytes}:null}memoryCensus(t=8){return this.host.memory?.census(t)??null}markMemoryPeak(){this.host.memory?.markPeak()}async _runSteps(t,{wait:r=!1,mergePass:n}){let a=!!this.profileSession;if(!a&&n){await this.runMergedSteps(t,r);return}this.recordPreparedExecution(t,!1);let s=pa(t),o=t.map((w,b)=>!Lt(w)||Tt(w.dispatchWorkgroups)?-1:b).filter(w=>w>=0),i=4096,u=2,l=i/u,c=a?Array.from({length:Math.ceil(o.length/l)},(w,b)=>this.createTimestampResources(Math.min(l,o.length-b*l)*u)):[],d=this.executionDiagnostics,p=d?performance.now():0,f=0,m=[];for(let w=0;w<s.length;++w){let b=this.host.device.createCommandEncoder({label:a?"profiled-compute-dispatch":"compute-dispatch"});for(let v of s[w]){if(v.kind==="copy"){v.byteLength>0&&b.copyBufferToBuffer(v.src,v.srcOffset,v.dst,v.dstOffset,v.byteLength);continue}if(v.kind==="clear"){v.byteLength>0&&b.clearBuffer(v.dst,v.dstOffset,v.byteLength);continue}if(Tt(v.dispatchWorkgroups))continue;let y=b.beginComputePass({label:a?`${v.name??"compute"}-profile-pass`:"compute-pass",...c.length>0?{timestampWrites:{querySet:c[Math.floor(f/l)].querySet,beginningOfPassWriteIndex:f%l*2,endOfPassWriteIndex:f%l*2+1}}:{}});y.setPipeline(v.pipeline),y.setBindGroup(0,v.bindGroup),v.immediateData&&$o(y,v.immediateData),y.dispatchWorkgroups(v.dispatchWorkgroups[0],v.dispatchWorkgroups[1],v.dispatchWorkgroups[2]),y.end(),f+=1}if(w===s.length-1)for(let v=0;v<c.length;++v){let y=Math.min(l,o.length-v*l)*u,S=c[v];b.resolveQuerySet(S.querySet,0,y,S.resolveBuffer,0),b.copyBufferToBuffer(S.resolveBuffer,0,S.readbackBuffer,0,y*8)}m.push(b.finish())}let h=!0,g=(w,b)=>{let v=this.host.submit([w],{wait:b});return h&&(h=!1,d&&(d.cpuEncodeAndInitialSubmitMs+=performance.now()-p)),v};for(let w=0;w+1<m.length;++w)await g(m[w],!0);if(!a||c.length===0){await g(m[m.length-1],r||jl(s[s.length-1]));return}try{await g(m[m.length-1],r);let w=(await Promise.all(c.map((b,v)=>this.readTimestampBuffer(b.readbackBuffer,Math.min(l,o.length-v*l)*u)))).flat();for(let b=0;b<o.length;++b){let v=t[o[b]];Lt(v)&&this.profileSession?.record(v,w[b*2],w[b*2+1])}}finally{for(let w of c)w.querySet.destroy(),w.resolveBuffer.destroy(),w.readbackBuffer.destroy()}}createTimestampResources(t){let r=t*8,n=this.host.device.createQuerySet({type:"timestamp",count:t}),a=this.host.createBuffer({label:"timestamp-resolve",size:r,usage:GPUBufferUsage.QUERY_RESOLVE|GPUBufferUsage.COPY_SRC}),s=this.host.createBuffer({label:"timestamp-readback",size:r,usage:GPUBufferUsage.COPY_DST|GPUBufferUsage.MAP_READ});return{querySet:n,resolveBuffer:a,readbackBuffer:s}}async readTimestampBuffer(t,r){let n=await this.host.mapRead(t,0,r*8),a=new BigUint64Array(n);return Array.from(a,s=>s)}};function Nl(e){return e.some(t=>Lt(t)&&!Tt(t.dispatchWorkgroups))}async function L2(e={}){let t=e.host??await B2(e);return new Ho({host:t})}function fr(e){let t=e&32768?-1:1,r=e>>>10&31,n=e&1023;return r===31?n===0?t*(1/0):NaN:r===0?t*2**-14*(n/1024):t*2**(r-15)*(1+n/1024)}var zl=null;function q2(){if(!zl){let e=new Float32Array(65536);for(let t=0;t<e.length;++t)e[t]=fr(t);zl=e}return zl}function Bn(e){let t=new Uint16Array(e.length);if(e instanceof Float32Array){let r=new Uint32Array(e.buffer,e.byteOffset,e.length);for(let n=0;n<e.length;++n){let a=r[n],s=(a>>>23&255)-112;if(s>0&&s<31){let o=a&8388607,i=o&8191,u=o>>>13;(i>4096||i===4096&&(u&1)===1)&&(u+=1,u===1024&&(u=0,s+=1)),t[n]=s>=31?a>>>16&32768|31743:a>>>16&32768|s<<10|u}else t[n]=D2(a)}}else for(let r=0;r<e.length;++r)t[r]=Kl(e[r]);return t}var O2=new Float32Array(1),jk=new Uint32Array(O2.buffer);function Kl(e){return Number.isNaN(e)?32256:(O2[0]=e,D2(jk[0]))}function D2(e){let t=e>>>16&32768,r=e>>>23&255,n=e&8388607;if(r===255)return n!==0?32256:t|31744;let a=r-127+15;if(a>=31)return t|31743;if(a<=0){if(a<-10)return t;let l=n|8388608,c=14-a,d=1<<c-1,p=l&(1<<c)-1,f=l>>>c;return(p>d||p===d&&(f&1)===1)&&(f+=1),t|f}let s=4096,o=n&8191,i=n>>>13,u=a;return(o>s||o===s&&(i&1)===1)&&(i+=1,i===1024&&(i=0,u+=1)),u>=31?t|31743:t|u<<10|i}var Nk=new Set(["F32","BF16","F16"]),zk=new Set(["float32","float16"]);var Kk=(()=>{let e=new Uint16Array([258]);return new Uint8Array(e.buffer)[0]===2})();function Uk(e){return typeof e=="string"&&Nk.has(e)}function $k(e){return typeof e=="string"&&zk.has(e)}function Vo(e){if(!Uk(e))throw new Error(`Unsupported source dtype: ${e}`)}function Hk(e){if(!$k(e))throw new Error(`Unsupported target dtype: ${e}`)}function is(e,t){if(e==="F32")return Vk(t);if(e==="BF16")return N2(t);if(e==="F16")return K2(t);throw new Error(`Unsupported source dtype for float32 copy: ${e}`)}function M2(e,t){if(Vo(e),t==="bfloat16"){if(e!=="BF16")throw new Error(`bfloat16 storage holds BF16 sources only, not ${e}`);return}Hk(t)}function W2(e){if(e==="F32")return 4;if(e==="BF16"||e==="F16")return 2;throw new Error(`Unsupported source dtype: ${e}`)}function F2(e){if(e==="float32")return 4;if(e==="float16"||e==="bfloat16")return 2;throw new Error(`Unsupported target dtype: ${e}`)}function Q2(e){return new Uint8Array(e.buffer,e.byteOffset,e.byteLength)}function Vk(e){let t=e.byteLength/4;return e.byteOffset%4===0?new Float32Array(e.buffer,e.byteOffset,t):Zk(e,Float32Array,4)}function j2(e,t){if(e==="BF16"&&t==="float32")return N2;if(e==="BF16"&&t==="float16")return Jk;if(e==="F16"&&t==="float32")return K2;if(e==="F32"&&t==="float16")return t0;throw new Error(`No converter for ${e} -> ${t}`)}function Zk(e,t,r){let n=new t(e.byteLength/r);return new Uint8Array(n.buffer).set(e),n}function N2(e){let t=z2(e),r=new Float32Array(t);return Xk(e,r)}function Xk(e,t,r=0){let n=z2(e);Yk(t.length,r,n,"BF16 -> F32");let a=new Uint32Array(t.buffer,t.byteOffset,t.length);if(Kk&&e.byteOffset%2===0){let s=new Uint16Array(e.buffer,e.byteOffset,n);for(let o=0;o<n;++o)a[r+o]=s[o]<<16}else for(let s=0;s<n;++s){let o=e[s*2],i=e[s*2+1];a[r+s]=(i<<8|o)<<16}return t}function z2(e){if(e.byteLength%2!==0)throw new Error(`BF16 byte length ${e.byteLength} is not divisible by 2`);return e.byteLength/2}function Yk(e,t,r,n){if(!Number.isSafeInteger(t)||t<0)throw new Error(`${n} destination offset must be a non-negative safe integer, got ${t}`);if(t>e||r>e-t)throw new Error(`${n} destination range [${t}, ${t+r}) exceeds length ${e}`)}function Jk(e){let t=e.byteLength/2,r=new Uint16Array(t);return e0(e,r),r}function e0(e,t,r=0,n=1){let a=e.byteLength/2,s=a0();if(e.byteOffset%2===0){let o=new Uint16Array(e.buffer,e.byteOffset,a);for(let i=0,u=r;i<a;++i,u+=n)t[u]=s[o[i]]}else for(let o=0,i=r;o<a;++o,i+=n){let u=e[o*2],l=e[o*2+1];t[i]=s[l<<8|u]}return t}function t0(e){let t=e.byteLength/4;if(e.byteOffset%4===0)return Bn(new Float32Array(e.buffer,e.byteOffset,t));let r=new Float32Array(t);return new Uint8Array(r.buffer).set(e),Bn(r)}function K2(e){let t=e.byteLength/2,r=new Float32Array(t),n=q2();if(e.byteOffset%2===0){let a=new Uint16Array(e.buffer,e.byteOffset,t);for(let s=0;s<t;++s)r[s]=n[a[s]]}else{let a=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let s=0;s<t;++s)r[s]=n[a.getUint16(s*2,!0)]}return r}var U2=new Uint32Array(1),r0=new Float32Array(U2.buffer);function n0(e){return U2[0]=e<<16>>>0,Kl(r0[0])}var Ul=null;function a0(){if(Ul)return Ul;let e=new Uint16Array(65536);for(let t=0;t<65536;++t)e[t]=n0(t);return Ul=e,e}var I=Object.freeze({F32:0,F16:1,Q4_0:2,Q4_1:3,Q5_0:6,Q5_1:7,Q8_0:8,Q8_1:9,Q2_K:10,Q3_K:11,Q4_K:12,Q5_K:13,Q6_K:14,Q8_K:15,IQ2_XXS:16,IQ2_XS:17,IQ3_XXS:18,IQ1_S:19,IQ4_NL:20,IQ3_S:21,IQ2_S:22,IQ4_XS:23,I8:24,I16:25,I32:26,I64:27,F64:28,IQ1_M:29,BF16:30,TQ1_0:34,TQ2_0:35,MXFP4:39,NVFP4:40,Q1_0:41,Q2_0:42,PRISM_PQ2_0:142,PRISM_PTQ1_0:143}),Vr=Object.freeze(Object.fromEntries(Object.entries(I).map(([e,t])=>[t,e])));var Re=class extends Error{constructor(t){super(t),this.name="GGUFError"}};var ut=256,$l=128,s0=18,o0=64,i0=18,ga=32,u0=18,l0=32,$2=20,c0=32,d0=22,p0=32,f0=24,En=32,mr=34,m0=32,g0=36,h0=292,Hl=ut,us=210,Vl=ut,ma=144,w0=ut,b0=176,v0=ut,_0=66,y0=54,A0=32,k0=17,S0=64,x0=36,T0=50,C0=56;function Rn(e,t){if(!Number.isInteger(t)||t<0)throw new Re(`Invalid tensor element count: ${t}`);if(e===I.F32)return t*4;if(e===I.F16||e===I.BF16)return t*2;if(e===I.I8)return t;if(e===I.I16)return t*2;if(e===I.I32)return t*4;if(e===I.I64||e===I.F64)return t*8;if(e===I.Q1_0){if(t%$l!==0)throw new Re(`Q1_0 tensor element count must be divisible by ${$l}`);return t/$l*s0}if(e===I.Q4_0)return Be(e,t,ga,u0);if(e===I.PRISM_PQ2_0)return Be(e,t,128,34);if(e===I.PRISM_PTQ1_0)return Be(e,t,128,28);if(e===I.Q2_0)return Be(e,t,o0,i0);if(e===I.Q4_1)return Be(e,t,l0,$2);if(e===I.Q5_0)return Be(e,t,c0,d0);if(e===I.Q5_1)return Be(e,t,p0,f0);if(e===I.Q8_0)return Be(e,t,En,mr);if(e===I.Q8_1)return Be(e,t,m0,g0);if(e===I.Q2_K)return Be(e,t,ut,84);if(e===I.Q3_K)return Be(e,t,ut,110);if(e===I.Q4_K)return Be(e,t,Vl,ma);if(e===I.Q5_K)return Be(e,t,w0,b0);if(e===I.Q6_K)return Be(e,t,Hl,us);if(e===I.Q8_K)return Be(e,t,ut,h0);if(e===I.IQ4_NL)return Be(e,t,32,18);if(e===I.IQ4_XS)return Be(e,t,ut,136);if(e===I.IQ2_XXS)return Be(e,t,ut,66);if(e===I.IQ2_XS)return Be(e,t,ut,74);if(e===I.IQ2_S)return Be(e,t,ut,82);if(e===I.IQ3_XXS)return Be(e,t,ut,98);if(e===I.IQ3_S)return Be(e,t,ut,110);if(e===I.TQ2_0)return Be(e,t,v0,_0);if(e===I.TQ1_0)return Be(e,t,ut,y0);if(e===I.MXFP4)return Be(e,t,A0,k0);if(e===I.NVFP4)return Be(e,t,S0,x0);if(e===I.IQ1_S)return Be(e,t,ut,T0);if(e===I.IQ1_M)return Be(e,t,ut,C0);throw new Re(`Unsupported GGML tensor type ${Vr[e]??e}`)}function Be(e,t,r,n){if(t%r!==0)throw new Re(`${Vr[e]??e} tensor element count ${t} must be divisible by ${r}`);return t/r*n}function H2(e){if(e.byteLength%mr!==0)throw new Re(`Q8_0 byte length ${e.byteLength} is not a multiple of ${mr}`);let t=e.byteLength/mr,r=new Float32Array(t*En),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let a=0;a<t;++a){let s=a*mr,o=fr(n.getUint16(s,!0)),i=a*En;for(let u=0;u<32;++u)r[i+u]=n.getInt8(s+2+u)*o}return r}function V2(e){if(e.byteLength%us!==0)throw new Re(`Q6_K byte length ${e.byteLength} is not a multiple of ${us}`);let t=e.byteLength/us,r=new Float32Array(t*Hl),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let a=0;a<t;++a){let s=a*us,o=fr(n.getUint16(s+208,!0)),i=a*Hl;for(let u=0;u<2;++u){let l=s+u*64,c=s+128+u*32,d=s+192+u*8,p=i+u*128;for(let f=0;f<32;++f){let m=f>>4,h=e[c+f],g=(e[l+f]&15|(h>>0&3)<<4)-32,w=(e[l+f+32]&15|(h>>2&3)<<4)-32,b=(e[l+f]>>4|(h>>4&3)<<4)-32,v=(e[l+f+32]>>4|(h>>6&3)<<4)-32;r[p+f]=o*n.getInt8(d+m+0)*g,r[p+f+32]=o*n.getInt8(d+m+2)*w,r[p+f+64]=o*n.getInt8(d+m+4)*b,r[p+f+96]=o*n.getInt8(d+m+6)*v}}}return r}function Zl(e,t,r){if(e<4)return[t[r+e]&63,t[r+e+4]&63];let n=t[r+e+4]&15|t[r+e-4]>>6<<4,a=t[r+e+4]>>4|t[r+e-0]>>6<<4;return[n,a]}function Z2(e){if(e.byteLength%ma!==0)throw new Re(`Q4_K byte length ${e.byteLength} is not a multiple of ${ma}`);let t=e.byteLength/ma,r=new Float32Array(t*Vl),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let a=0;a<t;++a){let s=a*ma,o=fr(n.getUint16(s,!0)),i=fr(n.getUint16(s+2,!0)),u=s+4,l=s+16,c=a*Vl,d=0;for(let p=0;p<256;p+=64){let[f,m]=Zl(d+0,e,u),h=o*f,g=i*m,[w,b]=Zl(d+1,e,u),v=o*w,y=i*b;for(let S=0;S<32;++S)r[c++]=h*(e[l+S]&15)-g;for(let S=0;S<32;++S)r[c++]=v*(e[l+S]>>4)-y;l+=32,d+=2}}return r}var e5=Int8Array.from([-127,-104,-83,-65,-49,-35,-22,-10,1,13,25,38,53,69,89,113]);var t5=Int8Array.from([0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12]),r5=new DataView(new ArrayBuffer(4));var n5=new Set([I.Q2_0,I.TQ1_0,I.PRISM_PQ2_0,I.PRISM_PTQ1_0,I.IQ2_XXS,I.IQ2_XS,I.IQ2_S,I.IQ3_XXS,I.IQ3_S,I.TQ2_0,I.IQ1_S,I.IQ1_M]);var X2={8:H2,12:Z2,14:V2},Xl=new Set([2,3,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,29,34,35,39,40,41,42,142,143]);var eg=1179993927,I0=32,B0=new TextDecoder("utf-8",{fatal:!0}),Ee=Object.freeze({UINT8:0,INT8:1,UINT16:2,INT16:3,UINT32:4,INT32:5,FLOAT32:6,BOOL:7,STRING:8,ARRAY:9,UINT64:10,INT64:11,FLOAT64:12});function qt(e,t,r,n){let a=e[t];if(typeof a=="number")return a;if(n!==void 0)return n;throw new Error(`${r} GGUF: missing numeric metadata "${t}"`)}function Zo(e){return typeof e=="number"?e:void 0}function E0(e){if(typeof e=="number"||Array.isArray(e)&&e.every(t=>typeof t=="number"))return e}function cs(e){let t=[];for(let a of["tokenizer.ggml.eos_token_id","tokenizer.ggml.eot_token_id","tokenizer.ggml.eog_token_ids"]){let s=E0(e[a]);for(let o of typeof s=="number"?[s]:s??[])t.includes(o)||t.push(o)}let r=e["tokenizer.ggml.tokens"],n=Array.isArray(r)?r.indexOf("<|im_end|>"):-1;return n>=0&&!t.includes(n)&&t.push(n),t}function tg(e,t){let r=e["tokenizer.ggml.tokens"],n=Array.isArray(r)?r.indexOf(t):-1;if(n<0)throw new Error(`the GGUF tokenizer has no ${t} token`);return n}var Y2={[Ee.UINT8]:1,[Ee.INT8]:1,[Ee.BOOL]:1,[Ee.UINT16]:2,[Ee.INT16]:2,[Ee.UINT32]:4,[Ee.INT32]:4,[Ee.FLOAT32]:4,[Ee.UINT64]:8,[Ee.INT64]:8,[Ee.FLOAT64]:8};function rg(e){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),n=e.byteLength,a=0,s=c=>a+c<=n,o=()=>{let c=r.getUint32(a+4,!0)*4294967296+r.getUint32(a,!0);return a+=8,c};if(!s(24))return{need:262168};if(r.getUint32(0,!0)!==eg)return{end:0};a=8;let i=o(),u=o(),l=()=>{if(!s(8))return!1;let c=o();return s(c)?(a+=c,!0):!1};for(let c=0;c<u;c++){if(!l()||!s(4))return{need:a+64+262144};let d=r.getUint32(a,!0);if(a+=4,d===Ee.STRING){if(!l())return{need:a+64+262144}}else if(d===Ee.ARRAY){if(!s(12))return{need:a+12+262144};let p=r.getUint32(a,!0);a+=4;let f=o(),m=Y2[p];if(m!==void 0){if(!s(f*m))return{need:a+f*m+262144};a+=f*m}else if(p===Ee.STRING){let h=a;for(let g=0;g<f;g++)if(!l()){let w=g>0?(a-h)/g:16;return{need:Math.ceil(a+(f-g)*w*1.25)+262144}}}else return{end:a}}else{let p=Y2[d];if(p===void 0)return{end:a};if(!s(p))return{need:a+p+262144};a+=p}}for(let c=0;c<i;c++){if(!l()||!s(4))return{need:a+(i-c)*96+262144};let d=r.getUint32(a,!0);if(a+=4,!s(d*8+12))return{need:a+(i-c)*96+262144};a+=d*8+12}return{end:a}}function Jl(e){if(e.has("token_embd.weight")){let r=e.info("token_embd.weight").shape[0];if(Number.isInteger(r)&&r>0)return r}let t=e.metadata["tokenizer.ggml.tokens"];if(Array.isArray(t)&&t.length>0)return t.length}var gr=class{#e;#t;version;tensorCount;metadataCount;metadata;alignment;dataStart;constructor(t,r={}){this.#e=L0(t);let n=new Yl(this.#e);if(n.u32()!==eg)throw new Re("Invalid GGUF magic");this.version=n.u32(),this.tensorCount=ls(n.u64(),"tensor count"),this.metadataCount=ls(n.u64(),"metadata count"),this.metadata={};for(let i=0;i<this.metadataCount;++i){let u=n.string();this.metadata[u]=n.metadataValue()}let s=new Map;for(let i=0;i<this.tensorCount;++i){let u=n.string(),l=Array.from({length:n.u32()},()=>ls(n.u64(),`${u} dimension`)),c=n.u32(),d=ls(n.u64(),`${u} offset`),p=l.reduce((f,m)=>f*m,1);s.set(u,{name:u,dims:l,shape:[...l].reverse(),type:c,typeName:Vr[c]??`UNKNOWN_${c}`,offset:d,elementCount:p,byteLength:Rn(c,p),dataOffsets:[0,0]})}let o=this.metadata["general.alignment"]??I0;if(typeof o!="number"||!Number.isInteger(o)||o<=0)throw new Re(`Invalid general.alignment: ${o}`);if(this.alignment=o,this.dataStart=at(n.offset,o),!r.headerOnly){if(this.dataStart>this.#e.byteLength)throw new Re("GGUF header exceeds file size");for(let i of s.values()){let u=this.dataStart+i.offset,l=u+i.byteLength;if(l>this.#e.byteLength)throw new Re(`Tensor ${i.name} exceeds file size`);i.dataOffsets=[u,l]}}this.#t=s}names(){return[...this.#t.keys()]}has(t){return this.#t.has(t)}info(t){return G0(this.#n(t))}tensorBytes(t){let r=this.#n(t),[n,a]=r.dataOffsets;return this.#e.subarray(n,a)}tensorByteRange(t,r,n){let[a]=this.#n(t).dataOffsets;return this.#e.subarray(a+r,a+n)}async streamTensorRange(t,r,n,a,s={}){let o=this.#n(t);tc(o,r,n),J2(s.signal);let[i]=o.dataOffsets;await a(this.#e.subarray(i+r,i+n)),J2(s.signal)}async streamTensors(t,r){for(let n of t)await r(n,this.tensorBytes(n))}#n(t){let r=this.#t.get(t);if(!r)throw new Re(`Unknown tensor: ${t}`);return r}};function ha(e,t){let r=X2[e];if(r)return r(t);if(Xl.has(e))throw new Re(`CPU dequantizer for ${Vr[e]??e} is not in this bundle (built for the shipped model's weight formats \u2014 load that model's own GGUF, or rebuild)`);return ng(e)?R0(e,t):null}function ng(e){return e===I.I8||e===I.I16||e===I.I32||e===I.I64||e===I.F64}function R0(e,t){let r=e===I.I8?1:e===I.I16?2:e===I.I32?4:8;if(t.byteLength%r!==0)throw new Re(`${Vr[e]} byte length ${t.byteLength} is not divisible by ${r}`);let n=new Float32Array(t.byteLength/r),a=new DataView(t.buffer,t.byteOffset,t.byteLength);for(let s=0;s<n.length;++s){let o=s*r;e===I.I8?n[s]=a.getInt8(o):e===I.I16?n[s]=a.getInt16(o,!0):e===I.I32?n[s]=a.getInt32(o,!0):e===I.I64?n[s]=Number(a.getBigInt64(o,!0)):n[s]=a.getFloat64(o,!0)}return n}function Xo(e){return Xl.has(e)}function ec(e){return Xo(e)||e===I.F32||e===I.F16||e===I.BF16||e===I.I8||e===I.I16||e===I.I32||e===I.I64||e===I.F64}function ag(e){if(e===I.F32)return"F32";if(e===I.F16)return"F16";if(e===I.BF16)return"BF16";throw new Re(`Unsupported GGUF scalar type ${Vr[e]??e}; quantized types dequantize separately`)}function ds(e){return Xo(e)||ng(e)?"F32":ag(e)}function Zr(e,t){return ha(e,t)??is(ag(e),t)}function G0(e){return{name:e.name,dims:[...e.dims],shape:[...e.shape],type:e.type,typeName:e.typeName,offset:e.offset,elementCount:e.elementCount,byteLength:e.byteLength,dataOffsets:[...e.dataOffsets]}}function tc(e,t,r){if(!Number.isSafeInteger(t)||!Number.isSafeInteger(r)||t<0||r<t)throw new RangeError(`Invalid byte range [${t}, ${r}) for tensor ${e.name}`);if(r>e.byteLength)throw new RangeError(`Byte range [${t}, ${r}) exceeds tensor ${e.name} size ${e.byteLength}`)}function J2(e){if(e?.aborted)throw e.reason??new Error("aborted")}var Yl=class{bytes;view;offset;constructor(t){this.bytes=t,this.view=new DataView(t.buffer,t.byteOffset,t.byteLength),this.offset=0}u8(){this.#t(1);let t=this.view.getUint8(this.offset);return this.offset+=1,t}i8(){this.#t(1);let t=this.view.getInt8(this.offset);return this.offset+=1,t}u16(){this.#t(2);let t=this.view.getUint16(this.offset,!0);return this.offset+=2,t}i16(){this.#t(2);let t=this.view.getInt16(this.offset,!0);return this.offset+=2,t}u32(){this.#t(4);let t=this.view.getUint32(this.offset,!0);return this.offset+=4,t}i32(){this.#t(4);let t=this.view.getInt32(this.offset,!0);return this.offset+=4,t}u64(){this.#t(8);let t=this.view.getBigUint64(this.offset,!0);return this.offset+=8,t}i64(){this.#t(8);let t=this.view.getBigInt64(this.offset,!0);return this.offset+=8,t}f32(){this.#t(4);let t=this.view.getFloat32(this.offset,!0);return this.offset+=4,t}f64(){this.#t(8);let t=this.view.getFloat64(this.offset,!0);return this.offset+=8,t}length(t){this.#t(8);let r=this.view.getUint32(this.offset+4,!0);if(r>2097151)throw new Re(`${t} ${this.view.getBigUint64(this.offset,!0)} exceeds JavaScript safe integer range`);let n=r*2**32+this.view.getUint32(this.offset,!0);return this.offset+=8,n}string(){let t=this.length("string length");this.#t(t);let r=this.bytes.subarray(this.offset,this.offset+t);return this.offset+=t,B0.decode(r)}metadataValue(){let t=this.u32();return this.#e(t)}#e(t){switch(t){case Ee.UINT8:return this.u8();case Ee.INT8:return this.i8();case Ee.UINT16:return this.u16();case Ee.INT16:return this.i16();case Ee.UINT32:return this.u32();case Ee.INT32:return this.i32();case Ee.FLOAT32:return this.f32();case Ee.BOOL:return!!this.u8();case Ee.STRING:return this.string();case Ee.UINT64:return ls(this.u64(),"metadata uint64");case Ee.INT64:return q0(this.i64(),"metadata int64");case Ee.FLOAT64:return this.f64();case Ee.ARRAY:{let r=this.u32();if(r===Ee.ARRAY)throw new Re("Nested metadata arrays are not supported");let n=this.length("metadata array length"),a=new Array(n);for(let s=0;s<n;s++)a[s]=this.#e(r);return a}default:throw new Re(`Unsupported GGUF metadata type ${t}`)}}#t(t){if(this.offset+t>this.bytes.byteLength)throw new Re("Unexpected end of GGUF file")}};function L0(e){if(e instanceof Uint8Array)return e;if(ArrayBuffer.isView(e))return new Uint8Array(e.buffer,e.byteOffset,e.byteLength);if(e instanceof ArrayBuffer)return new Uint8Array(e);throw new Re("GGUF input must be an ArrayBuffer or Uint8Array")}function ls(e,t){if(e>BigInt(Number.MAX_SAFE_INTEGER))throw new Re(`${t} ${e} exceeds JavaScript safe integer range`);return Number(e)}function q0(e,t){if(e>BigInt(Number.MAX_SAFE_INTEGER)||e<BigInt(Number.MIN_SAFE_INTEGER))throw new Re(`${t} ${e} exceeds JavaScript safe integer range`);return Number(e)}var Xt=class extends Error{constructor(t){super(t),this.name="DirectByteReadUnavailableError"}};var O0=1e4,sg=3,sc=class extends Error{};async function og(e,t,r,n,a){for(let s=1;;s++){Rr(e);let o=new AbortController,i=new sc(`${n} delivered nothing for ${r} ms`),u,l=()=>{clearTimeout(u),u=setTimeout(()=>o.abort(i),r)},c=()=>o.abort(e?.reason);e?.addEventListener("abort",c,{once:!0});let d=0;l();try{return await a(o.signal,t&&(p=>{d+=p,t(p)}),l)}catch(p){if(o.signal.reason!==i||e?.aborted)throw p;if(s>=sg)throw new Error(`${n} stalled ${sg} times`,{cause:i});d!==0&&t?.(-d)}finally{clearTimeout(u),e?.removeEventListener("abort",c)}}}function D0(e,t){let r=e.url||t;try{let n=new URL(r);return n.hash="",n.search="",n.toString()}catch{return r}}function ig(e){try{let t=new URL(e);if(t.hostname!=="huggingface.co")return;let r=t.pathname.split("/").filter(Boolean),n=r.indexOf("resolve");if(n>=0&&/^[0-9a-f]{40,64}$/i.test(r[n+1]??""))return r[n+1].toLowerCase();let a=r.indexOf("resolve-cache");return a>=0?r.slice(a+1).find(s=>/^[0-9a-f]{40,64}$/i.test(s))?.toLowerCase():void 0}catch{return}}function rc(e,t){let r=D0(e,t),n=e.headers.get("etag")?.trim();if(n&&!/^W\//i.test(n))return`http|${r}|etag=${n}`;let a=ig(r)??ig(t);return a?`http|${r}|revision=${a}`:void 0}async function dg(e,t,r){let n=null;try{n=await t(e,{method:"HEAD",signal:r})}catch{}if(n?.ok){let s=n.headers.get("content-length"),o=s===null?null:Number(s);if(s!==null&&!Number.isFinite(o))throw new Error(`Invalid content-length header: ${s}`);return{size:o,acceptsRanges:(n.headers.get("accept-ranges")??"").toLowerCase().includes("bytes"),resourceIdentity:rc(n,e)}}let a=await t(e,{headers:{Range:"bytes=0-0"},signal:r});if(a.status===206){await a.arrayBuffer();let s=Number(a.headers.get("content-range")?.split("/").pop());return{size:Number.isFinite(s)?s:null,acceptsRanges:!0,resourceIdentity:rc(a,e)}}if(a.ok){await a.body?.cancel().catch(()=>{});let s=a.headers.get("content-length"),o=s===null?null:Number(s);return{size:Number.isFinite(o??NaN)?o:null,acceptsRanges:!1,resourceIdentity:rc(a,e)}}throw new Error(`HEAD ${e} failed${n?` (${n.status} ${n.statusText})`:""} and the range probe returned ${a.status}`)}async function ei(e,t={}){return(await oc(e,t))?.size??null}async function oc(e,t={}){let r=t.fetch??((s,o)=>fetch(s,o)),n=AbortSignal.timeout(t.timeoutMs??4e3),a=t.signal?AbortSignal.any([t.signal,n]):n;try{let{size:s,resourceIdentity:o}=await dg(e,r,a);return{size:s,resourceIdentity:o}}catch{return null}}async function ic(e,t,{requireRangeRequests:r=!1,knownSize:n=void 0,knownAcceptsRanges:a=void 0,knownResourceIdentity:s=void 0}={}){let o=t.fetch??globalThis.fetch;if(typeof o!="function")throw new Error("No fetch implementation available; pass options.fetch");let i=n,u=a,l=s;if(i===void 0){let g=await dg(e,o,t.signal);i=g.size,u=g.acceptsRanges,l=g.resourceIdentity}else u=u!==!1;if(r&&!u)throw new Error(`Range requests are required for ${e}, but the server did not advertise Accept-Ranges: bytes`);let c=t.stallTimeoutMs??O0,d=e,p=async(g,w,b)=>{let v={headers:{Range:`bytes=${g}-${w-1}`},signal:b};if(d!==e){let S=await o(d,v);if(S.status!==401&&S.status!==403&&S.status!==410)return S;await Yo(S),d=e}let y=await o(e,v);return y.status===206&&y.redirected&&y.url&&y.url!==e&&(d=y.url),y},f=(g,w)=>`Range ${g}-${w-1} of ${e}`,m=!0,h={url:e,size:i??null,acceptsRanges:u,resourceIdentity:l,writeTensorAcceptsEphemeralBytes:!0,async readRange(g,w,b={}){if(g===w)return new Uint8Array(0);pg(g,w);let v=b.signal??t.signal;return og(v,b.onByteProgress??null,c,f(g,w),async(y,S,C)=>{if(u){let P=await p(g,w,y);if(P.status!==206&&P.status!==200)throw new Error(`${f(g,w)} failed: ${P.status}`);if(P.status===200&&r)throw new Error(`${f(g,w)} returned 200 instead of 206; refusing full-response fallback`);let _=await nc(P,{expectedLength:P.status===206?w-g:null,onByteProgress:P.status===200?lg(g,w,S):S,onChunk:C});if(P.status===200)return _.subarray(g,w);if(_.byteLength!==w-g)throw new Error(`Range ${g}-${w-1} returned ${_.byteLength} bytes`);return _}let k=await o(e,{signal:y});if(!k.ok)throw new Error(`GET ${e} failed: ${k.status}`);return(await nc(k,{onByteProgress:lg(g,w,S),onChunk:C})).subarray(g,w)})},async readRangeInto(g,w,b,v={}){if(W0(g,w,b),b.byteLength===0)return;let y=v.signal??t.signal;if(Rr(y),!m)throw new Xt("Direct response-body reads were disabled after a non-streaming response");try{await og(y,v.onByteProgress??null,c,f(g,w),async(S,C,k)=>{if(u){let P=await p(g,w,S);if(P.status!==206&&P.status!==200)throw new Error(`${f(g,w)} failed: ${P.status}`);if(P.status===200){if(r)throw await Yo(P),new Error(`${f(g,w)} returned 200 instead of 206; refusing full-response fallback`);await ug(P,g,w,b,{signal:S,onByteProgress:C,onChunk:k});return}await M0(P,b,{signal:S,onByteProgress:C,onChunk:k});return}let B=await o(e,{signal:S});if(!B.ok)throw new Error(`GET ${e} failed: ${B.status}`);await ug(B,g,w,b,{signal:S,onByteProgress:C,onChunk:k})})}catch(S){throw S instanceof Xt&&(m=!1,delete h.readRangeInto),S}},async readAll(g={}){let w=g.signal??t.signal,b=g.onProgress??t.onProgress,v=await o(e,{signal:w});if(!v.ok)throw new Error(`GET ${e} failed: ${v.status}`);return nc(v,{expectedLength:Number.isFinite(i)?i:null,onProgress:b,progressTotal:Number.isFinite(i)?i:null,fromCache:!1})},async readTensor(){return null},async writeTensor(){},async close(){}};return h}async function nc(e,{expectedLength:t=null,onByteProgress:r=null,onProgress:n=null,progressTotal:a=null,fromCache:s=!1,yieldEveryBytes:o=null,onChunk:i=null}={}){if(!e.body?.getReader){let f=await e.arrayBuffer(),m=new Uint8Array(f);return ac({onByteProgress:r,onProgress:n,loaded:m.byteLength,delta:m.byteLength,total:a??t,fromCache:s}),m}let u=e.body.getReader(),l=a??t??null,c=0;if(typeof t=="number"&&Number.isFinite(t)&&t>0){let f=new Uint8Array(t),m=[],h=0;for(;;){let{done:g,value:w}=await u.read();if(g)break;w&&(i?.(),c+w.byteLength<=f.byteLength&&m.length===0?f.set(w,c):(m.length===0&&m.push(f.subarray(0,c)),m.push(w)),c+=w.byteLength,ac({onByteProgress:r,onProgress:n,loaded:c,delta:w.byteLength,total:l,fromCache:s}),o&&(h+=w.byteLength)>=o&&(h=0,await Jo()))}return m.length===0?c===t?f:f.subarray(0,c):Ma(m,c)}let d=[],p=0;for(;;){let{done:f,value:m}=await u.read();if(f)break;m&&(i?.(),d.push(m),c+=m.byteLength,ac({onByteProgress:r,onProgress:n,loaded:c,delta:m.byteLength,total:l,fromCache:s}),o&&(p+=m.byteLength)>=o&&(p=0,await Jo()))}return Ma(d,c)}async function M0(e,t,{signal:r,onByteProgress:n=null,onChunk:a=null}={}){if(Rr(r),!e.body?.getReader)throw await Yo(e),new Xt("Direct response-body reads require a streaming response body");let s=e.body.getReader(),o=0;try{for(;;){Rr(r);let{done:i,value:u}=await s.read();if(Rr(r),i)break;if(u){if(a?.(),o+u.byteLength>t.byteLength)throw cg(t.byteLength,o+u.byteLength);t.set(u,o),o+=u.byteLength,n?.(u.byteLength),Rr(r)}}}catch(i){throw await s.cancel(i).catch(()=>{}),i}if(o!==t.byteLength)throw cg(t.byteLength,o)}async function ug(e,t,r,n,{signal:a,onByteProgress:s=null,onChunk:o=null}={}){if(Rr(a),!e.body?.getReader)throw await Yo(e),new Xt("Direct response-range reads require a streaming response body");let i=e.body.getReader(),u=0,l=0;try{for(;;){Rr(a);let{done:c,value:d}=await i.read();if(Rr(a),c)break;if(!d)continue;o?.();let p=u,f=u+d.byteLength,m=Math.max(t,p),h=Math.min(r,f);if(h>m){let g=m-p,w=h-m;n.set(d.subarray(g,g+w),m-t),l+=w,s?.(w)}u=f}}catch(c){throw await i.cancel(c).catch(()=>{}),c}if(l!==n.byteLength)throw new Error(`Full response has ${u} bytes; range [${t}, ${r}) is unavailable`)}function lg(e,t,r){if(!r)return null;let n=0;return a=>{let s=n,o=n+a;n=o;let i=Math.max(0,Math.min(t,o)-Math.max(e,s));i>0&&r(i)}}async function Yo(e){try{await e.body?.cancel()}catch{}}function Jo(){return new Promise(e=>setTimeout(e,0))}function ac({onByteProgress:e,onProgress:t,loaded:r,delta:n,total:a,fromCache:s}){e?.(n),t?.({loaded:r,total:a??null,fromCache:s})}function W0(e,t,r){pg(e,t);let n=t-e;if(r.byteLength!==n)throw new RangeError(`Destination has ${r.byteLength} bytes; expected ${n} for range [${e}, ${t})`)}function pg(e,t){if(!Number.isSafeInteger(e)||!Number.isSafeInteger(t)||e<0||t<e)throw new RangeError(`Invalid byte range [${e}, ${t})`)}function cg(e,t){return new Error(`Response returned ${t} bytes; expected ${e}`)}function Rr(e){if(e?.aborted)throw e.reason??new Error("aborted")}function fg(e){throw new Error(`Cannot use local path ${String(e)}: local files need a Node runtime`)}async function uc(e){return fg(e)}async function ti(e,t={}){return fg(e)}async function wa(e,t={},r={}){if(t.source)return t.source;if(typeof e=="string"&&/^https?:/i.test(e))return ic(e,t,r);if(typeof e=="string"&&e.startsWith("file:"))return ti(new URL(e),t);if(e instanceof URL){if(e.protocol==="http:"||e.protocol==="https:")return ic(e.toString(),t,r);if(e.protocol==="file:")return ti(e,t);throw new Error(`Unsupported URL protocol: ${e.protocol}`)}if(typeof e=="string")return ti(e,t);throw new Error("url must be a string or URL")}async function mg(e,t,r=""){let n=e.cacheStorage??globalThis.caches;return n?.open?n.open((e.cacheName??t)+r):null}var F0="gguf-v1";async function lc(e,t={}){let r=Q0(e),n=await N0(r,t);if(n)return n;let a=await wa(e,{fetch:t.fetch,signal:t.signal});try{let{gguf:s,bytes:o}=await cc(a,t.signal);return await z0(r,o,t,a.size??null),s}finally{await a.close()}}async function cc(e,t){if(!e.acceptsRanges||!e.readRange){if(!e.readAll)throw new Error(`GGUF source ${e.url} supports neither ranges nor readAll()`);let o=await e.readAll({signal:t});return{gguf:new gr(o,{headerOnly:!0}),bytes:o}}let r=64*1024*1024,n=e.size,a=n!=null?Math.min(n,r):r,s=await e.readRange(0,Math.min(4*1024*1024,a),{signal:t});for(;;){let o=rg(s);if("end"in o||s.byteLength>=a)break;let i=Math.min(a,Math.max(o.need,Math.ceil(s.byteLength*1.25),s.byteLength+1024*1024)),u=await e.readRange(s.byteLength,i,{signal:t});if(u.byteLength===0)break;s=Ma([s,u],s.byteLength+u.byteLength)}return{gguf:new gr(s,{headerOnly:!0}),bytes:s}}function Q0(e){return e instanceof URL?e.toString():String(e)}var j0="-headers";async function gg(e){return mg(e,F0,j0)}async function N0(e,t){if(t.force||t.cache===!1)return null;let r=await gg(t);if(!r)return null;let n=await r.match(e);if(!n)return null;let a=Number(n.headers.get("x-gguf-source-size")),s=Number.isFinite(a)?await ei(e,{fetch:t.fetch,signal:t.signal}):void 0;if(s===void 0||s!==null&&s!==a)return await r.delete(e).catch(()=>{}),null;try{return new gr(new Uint8Array(await n.arrayBuffer()),{headerOnly:!0})}catch{return null}}async function z0(e,t,r,n){if(r.cache===!1||t.byteLength>32*1024*1024)return;let a=await gg(r);if(!a)return;let s=new Response(Ws(t),{headers:{"content-type":"application/octet-stream","content-length":String(t.byteLength),...n!==null&&Number.isFinite(n)?{"x-gguf-source-size":String(n)}:{}}});try{await a.put(e,s)}catch(o){typeof console<"u"&&console.warn(`gguf header cache write failed: ${Sr(o)}`)}}var hr="chunks",Xr="meta";function ri(e,{cache:t,cacheKey:r=e.url,force:n=!1}){let a={...e,writeTensorAcceptsEphemeralBytes:t.acceptsEphemeralBytes===!0?!0:void 0,retainsWrittenTensors:!0,async readTensor(s,o){if(n)return null;try{return await t.get(r,s,o)}catch(i){return typeof console<"u"&&console.warn(`byte-range cache read failed: ${Sr(i)}`),null}},async writeTensor(s,o,i){try{let u=i.byteOffset===0&&i.byteLength===i.buffer.byteLength?i:new Uint8Array(i);await t.put(r,s,o,u)}catch(u){typeof console<"u"&&console.warn(`byte-range cache write failed: ${Sr(u)}`)}}};return e.readRangeInto?a.readRangeInto=async(s,o,i,u)=>{let l=e.readRangeInto;if(!l)throw delete a.readRangeInto,new Xt("The wrapped source disabled direct response-body reads");try{await l.call(e,s,o,i,u)}finally{e.readRangeInto||delete a.readRangeInto}}:delete a.readRangeInto,a}async function K0(e,t,r,n,a={}){let s=typeof r=="string"?r:r.toString();if(!/^https?:/i.test(s))return!1;let o=await ei(s,a);if(o===null||o===n)return!1;try{await e.delete?.(t)}catch(i){typeof console<"u"&&console.warn(`stale byte-range cache purge failed: ${Sr(i)}`)}return!0}var hg=new Map;function U0(e,t,r){let n=0,a=e.length-1,s=-1;for(;n<=a;){let o=n+a>>1;e[o][0]<=t?(s=o,n=o+1):a=o-1}return s>=0&&e[s][1]>=r?e[s]:null}function fs(e){if(typeof indexedDB>"u")return null;let t=new Map,r=null,n=new Set,a=async(o,i,u,l)=>new Promise((c,d)=>{let p=o.transaction(hr,"readonly").objectStore(hr).get([i,u,l]);p.onsuccess=()=>{let f=p.result;if(!f){c(null);return}f.arrayBuffer().then(m=>c(new Uint8Array(m)),d)},p.onerror=()=>d(p.error)}),s=(o,i)=>{let u=t.get(i);return u||(u=new Promise((l,c)=>{let d=o.transaction(hr,"readonly").objectStore(hr).getAllKeys(IDBKeyRange.bound([i],[i,[],[]]));d.onsuccess=()=>l(d.result.map(([,p,f])=>[p,f]).sort((p,f)=>p[0]-f[0])),d.onerror=()=>c(d.error)}),t.set(i,u)),u};return{acceptsEphemeralBytes:!0,async get(o,i,u){let l=await ps(e),c=await a(l,o,i,u);if(c)return c;if(r&&r.key===o&&r.begin<=i&&u<=r.end)return r.bytes.subarray(i-r.begin,u-r.begin);let d=U0(await s(l,o),i,u);if(!d)return null;let p=`${o}:${d[0]}:${d[1]}`;if(n.has(p))return null;let f;try{f=await a(l,o,d[0],d[1])}catch{return n.add(p),null}return f?(r={key:o,begin:d[0],end:d[1],bytes:f},f.subarray(i-d[0],u-d[0])):null},async put(o,i,u,l){let c=await ps(e);return t.delete(o),new Promise((d,p)=>{let f=c.transaction(hr,"readwrite").objectStore(hr).put(new Blob([Ws(l)]),[o,i,u]);f.onsuccess=()=>d(),f.onerror=()=>p(f.error)})},async getMeta(o){let i=await ps(e);return new Promise((u,l)=>{let c=i.transaction(Xr,"readonly").objectStore(Xr).get(o);c.onsuccess=()=>u(c.result??null),c.onerror=()=>l(c.error)})},async putMeta(o,i){let u=await ps(e);return new Promise((l,c)=>{let d=u.transaction(Xr,"readwrite").objectStore(Xr).put(i,o);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)})},async delete(o){let i=await ps(e);t.delete(o),r?.key===o&&(r=null);let u=IDBKeyRange.bound([o],[o,[],[]]);return await new Promise((l,c)=>{let d=i.transaction(hr,"readwrite").objectStore(hr).delete(u);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)}),new Promise((l,c)=>{let d=i.transaction(Xr,"readwrite").objectStore(Xr).delete(o);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)})}}}function ps(e){let t=hg.get(e);if(t)return t;let r=new Promise((n,a)=>{let s=indexedDB.open(e,2);s.onupgradeneeded=()=>{let o=s.result;o.objectStoreNames.contains(hr)||o.createObjectStore(hr),o.objectStoreNames.contains(Xr)||o.createObjectStore(Xr)},s.onsuccess=()=>n(s.result),s.onerror=()=>a(s.error),s.onblocked=()=>a(new Error("indexedDB open blocked"))});return hg.set(e,r),r}async function dc(e,t,r,n,a={}){try{let s=n(await e.getMeta(t));if(!s)return null;if(!a.verifyResourceIdentity)return await K0(e,t,r,s.size,a)?null:s;let o=typeof r=="string"?r:r.toString();if(!/^https?:/i.test(o))return{...s,resourceIdentity:void 0};let i=await oc(o,a);if(i&&(i.size!==null&&i.size!==s.size||i.resourceIdentity!==void 0&&i.resourceIdentity!==s.resourceIdentity)){try{await e.delete?.(t)}catch(u){typeof console<"u"&&console.warn(`stale byte-range cache purge failed: ${Sr(u)}`)}return null}return i?.resourceIdentity===s.resourceIdentity?s:{...s,resourceIdentity:void 0}}catch(s){return typeof console<"u"&&console.warn(`byte-range meta cache read failed: ${Sr(s)}`),null}}var ms=class{#e=[];covered(t,r){for(let n of this.#e){if(n.begin>t)break;if(r<=n.end)return!0}return!1}add(t,r){if(r<=t)return 0;let n=0;for(;n<this.#e.length&&this.#e[n].end<t;)n+=1;let a=n,s=t,o=r,i=0;for(;a<this.#e.length&&this.#e[a].begin<=r;){let u=this.#e[a];i+=Math.max(0,Math.min(r,u.end)-Math.max(t,u.begin)),s=Math.min(s,u.begin),o=Math.max(o,u.end),a+=1}return this.#e.splice(n,a-n,{begin:s,end:o}),r-t-i}read(t,r,n){let a=0,s=0;return{received:o=>{if(a=Math.min(r-t,a+o),a<=s)return;let i=this.add(t+s,t+a);s=a,i>0&&n?.(i)},finish:()=>{let o=this.add(t,r);o>0&&n?.(o)}}}};function gs(e,{maxBytes:t,maxGap:r}){let n=[...e].sort((o,i)=>o.begin-i.begin),a=[],s=null;for(let o of n){if(!s){s={begin:o.begin,end:o.end,tensors:[o]};continue}let i=o.begin-s.end,u=o.end-s.begin;i<=r&&u<=t?(s.end=Math.max(s.end,o.end),s.tensors.push(o)):(a.push(s),s={begin:o.begin,end:o.end,tensors:[o]})}return s&&a.push(s),a}var LOWMEM=/[?&]lowmem=1/.test(self.location?.search??""),$0=(LOWMEM?8:24)<<20,H0=1<<20,V0=(LOWMEM?32:96)<<20,Z0=LOWMEM?4:32;async function ni(e,t,r,{concurrency:n=Z0,chunkMaxBytes:a=$0,chunkMaxGap:s=H0,byteBudget:o=V0,signal:i,onProgress:u,onByteProgress:l,directDestinations:c,coalesceAgainst:d,countedChunks:p}={}){if(t.length===0)return;let f=e.readRangeInto&&e.writeTensorAcceptsEphemeralBytes===!0&&c?.size?c:void 0,m={maxBytes:a,maxGap:s},h=d?.length&&e.retainsWrittenTensors?d:void 0,g=h?X0(t,h,m):Y0(t,m,f),w=g.reduce((C,k)=>C+(k.end-k.begin),0),b=0,v=new Map,y=(C={})=>{if(!u)return;let k=b;for(let B of v.values())k+=B;u({loaded:k,total:w,...C})},S=async C=>{let{begin:k,end:B,tensors:P}=g[C],_=B-k,x=p?.read(k,B,l),R=x?.received??(F=>l?.(F)),O=P.length===1?P[0]:void 0,N=e.readRangeInto,Q=O?.begin===k&&O.end===B&&N&&e.writeTensorAcceptsEphemeralBytes===!0?c?.get(O.name):void 0;if(Q){let F=Q();if(F){if(F.bytes.byteLength!==_)throw await wg(F),new Error(`Direct destination for ${O.name} has ${F.bytes.byteLength} bytes; expected ${_}`);let H=!1;try{let Y=await e.readTensor(k,B),fe=Y?.byteLength===_;bg(i),fe?(F.bytes.set(Y),H=!0):(v.set(C,0),await N.call(e,k,B,F.bytes,{signal:i,onByteProgress:ce=>{v.set(C,(v.get(C)??0)+ce),R(ce),y({fromCache:!1,range:[k,B],inFlight:!0})}}),H=!0,await Promise.resolve().then(()=>e.writeTensor(k,B,F.bytes)).catch(()=>{})),fe&&!p&&l?.(_),x?.finish(),bg(i),await r({begin:k,end:B,bytes:F.bytes,tensors:P,directDestination:F}),v.delete(C),b+=_,y({fromCache:fe,range:[k,B]});return}catch(Y){if(await wg(F),Y instanceof Xt&&!H)v.delete(C);else throw Y}}}let V=await J0(P.map(F=>Promise.resolve().then(()=>e.readTensor(F.begin,F.end)))),$=V.every((F,H)=>F!=null&&F.byteLength===P[H].end-P[H].begin),M=P[0].begin,j=P.reduce((F,H)=>Math.max(F,H.end),P[0].end),A=j-M,T,E=k,X=B;if($){E=M,X=j;let H=(P.length===1?P[0]:void 0)?V[0]:void 0;if(H?.byteLength===A)T=H;else{T=new Uint8Array(A);for(let Y=0;Y<P.length;++Y)T.set(V[Y],P[Y].begin-M)}}else v.set(C,0),T=await e.readRange(k,B,{signal:i,onByteProgress:F=>{v.set(C,(v.get(C)??0)+F),R(F),y({fromCache:!1,range:[k,B],inFlight:!0})}});$&&!p&&l?.(_),x?.finish();let z=Promise.resolve().then(()=>r({begin:E,end:X,bytes:T,tensors:P})),G=$?Promise.resolve():Promise.resolve().then(()=>e.writeTensor(k,B,T)).catch(()=>{}),L=!1,K,D=z.catch(F=>{L=!0,K=F});if(await Promise.all([D,G]),L)throw K;v.delete(C),b+=_,y({fromCache:$,range:[k,B]})};await new Promise((C,k)=>{let B=0,P=0,_=0,x=g.length,R=!1,O=!1,N,Q=new Set,V=()=>{},$=()=>{R||!O||Q.size!==0||(R=!0,V(),k(N))},M=T=>{O||(O=!0,N=T),$()},j=(T,E,X,z)=>{if(Q.delete(T),P--,_-=E,X){M(z);return}if(O){$();return}if(--x===0){R=!0,V(),C();return}A()},A=()=>{if(!(R||O)){if(i?.aborted){M(i.reason??new Error("aborted"));return}for(;B<g.length&&P<n;){let T=g[B].end-g[B].begin;if(P>0&&_+T>o)break;let E=B++;P++,_+=T;let X=S(E);Q.add(X),X.then(()=>j(X,T,!1),z=>j(X,T,!0,z))}}};if(i){let T=()=>M(i.reason??new Error("aborted"));i.aborted?T():(i.addEventListener("abort",T,{once:!0}),V=()=>{i.removeEventListener("abort",T)})}A()})}function X0(e,t,r){let n=gs(t,r),a=n.map(l=>l.begin),s=l=>{let c=0,d=a.length-1,p=-1;for(;c<=d;){let m=c+d>>1;a[m]<=l.begin?(p=m,c=m+1):d=m-1}let f=p===-1?void 0:n[p];return f&&l.end<=f.end?f:void 0},o=new Map,i=[];for(let l of e){let c=s(l);if(!c){i.push(l);continue}let d=o.get(c);d?d.push(l):o.set(c,[l])}let u=[...o].map(([l,c])=>({begin:l.begin,end:l.end,tensors:c.sort((d,p)=>d.begin-p.begin)}));return u.push(...gs(i,r)),u.sort((l,c)=>l.begin-c.begin)}function Y0(e,t,r){if(!r)return gs(e,t);let n=[...e].sort((i,u)=>i.begin-u.begin),a=[],s=[],o=()=>{s.length!==0&&(a.push(...gs(s,t)),s=[])};for(let i of n){if(!r.has(i.name)){s.push(i);continue}o(),a.push({begin:i.begin,end:i.end,tensors:[i]})}return o(),a}async function wg(e){try{await e.abort()}catch{}}function bg(e){if(e?.aborted)throw e.reason??new Error("aborted")}async function J0(e){let t=new Array(e.length),r=!1,n;if(await Promise.all(e.map((a,s)=>a.then(o=>{t[s]=o},o=>{r||(r=!0,n=o)}))),r)throw n;return t}var e4="gguf-cache-v1",t4=32<<20;function r4(e){let t=e;return t?.headerBytes&&Number.isFinite(t.size)&&(t.resourceIdentity===void 0||typeof t.resourceIdentity=="string")?t:null}var Yr=class e{#e;#t;onByteProgress=null;#n=null;#r=new ms;constructor(t,r){this.#e=t,this.#t=r}get supportsDirectDestinations(){return this.#t.readRangeInto&&this.#t.writeTensorAcceptsEphemeralBytes===!0?!0:void 0}get resourceIdentity(){return this.#t.resourceIdentity}static async open(t,r={}){let n=r.cacheKey??(typeof t=="string"?t:t.toString()),a=r.cache===!1||r.source?null:r.chunkCache??fs(r.cacheName??e4),s=!!r.force,o={source:r.source,fetch:r.fetch,signal:r.signal},i=a&&!s?await dc(a,n,t,r4,{fetch:r.fetch,signal:r.signal,verifyResourceIdentity:!0}):null;if(a&&i){let l=new gr(i.headerBytes,{headerOnly:!0}),c=await wa(t,o,{knownSize:i.size,knownAcceptsRanges:i.acceptsRanges??!0,knownResourceIdentity:i.resourceIdentity});return new e(l,ri(c,{cache:a,cacheKey:n,force:s}))}let u=await wa(t,o);try{let{gguf:l,bytes:c}=await cc(u,r.signal);return a&&u.size!=null&&u.acceptsRanges&&c.byteLength<=t4&&a.putMeta(n,{size:u.size,headerBytes:c,acceptsRanges:u.acceptsRanges,resourceIdentity:u.resourceIdentity}).catch(()=>{}),new e(l,a?ri(u,{cache:a,cacheKey:n,force:s}):u)}catch(l){throw await u.close(),l}}get metadata(){return this.#e.metadata}names(){return this.#e.names()}has(t){return this.#e.has(t)}info(t){return this.#e.info(t)}planReads(t=[]){let r=t.filter(n=>this.#e.has(n)).map(n=>{let a=this.#e.info(n),s=this.#e.dataStart+a.offset;return{name:n,begin:s,end:s+a.byteLength}});this.#n=r.length>0?r.sort((n,a)=>n.begin-a.begin):null,this.#r=new ms}async tensorBytes(t,r={}){let n=this.#e.info(t),a=this.#e.dataStart+n.offset;return this.#a(a,a+n.byteLength,r.signal)}async tensorByteRange(t,r,n,a={}){let s=this.#e.dataStart+this.#e.info(t).offset;return this.#a(s+r,s+n,a.signal)}async streamTensorRange(t,r,n,a,s={}){let o=this.#e.info(t);tc(o,r,n);let i=this.#e.dataStart+o.offset,u=i+r,l=i+n;if(!Number.isSafeInteger(u)||!Number.isSafeInteger(l))throw new RangeError(`Absolute byte range for tensor ${t} is outside the safe integer range`);await ni(this.#t,[{name:t,begin:u,end:l}],({bytes:c,directDestination:d})=>a(c,d),{signal:s.signal,onByteProgress:this.onByteProgress??void 0,countedChunks:this.#r,directDestinations:s.directDestination?new Map([[t,s.directDestination]]):void 0})}async#a(t,r,n){let a=this.#r.read(t,r,i=>this.onByteProgress?.(i)),s=await this.#t.readTensor(t,r);if(s)return a.finish(),s;let o=await this.#t.readRange(t,r,{signal:n,onByteProgress:a.received});return a.finish(),await this.#t.writeTensor(t,r,o),o}async streamTensors(t,r,n={}){let a=t.map(s=>{let o=this.#e.info(s),i=this.#e.dataStart+o.offset;return{name:s,begin:i,end:i+o.byteLength}});await ni(this.#t,a,async s=>{for(let o of s.tensors)await r(o.name,s.bytes.subarray(o.begin-s.begin,o.end-s.begin),s.directDestination)},{concurrency:n.concurrency,chunkMaxBytes:n.chunkMaxBytes,chunkMaxGap:n.chunkMaxGap,byteBudget:n.byteBudget,signal:n.signal,onByteProgress:this.onByteProgress??void 0,directDestinations:n.directDestinations,coalesceAgainst:this.#n??void 0,countedChunks:this.#r})}async close(){await this.#t.close()}};var vg="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",n4="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",a4="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",s4="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",o4="[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",i4=["\\p{N}{1,3}","[\u4E00-\u9FA5\u3040-\u309F\u30A0-\u30FF]+","[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"],Ag=new Set(["llama3","llama-v3","llama-bpe","falcon3","falcon-h1","pixtral","midm-2.0","lfm2","jina-v5-nano"]),u4=new Set(["gpt-4o","llama4","kanana2","talkie","minimax-m2"]),l4=new Set(["qwen2","deepseek-r1-qwen","kormo","f2llmv2","megrez","stablelm2","hunyuan","solar-open"]),c4=new Set(["qwen35"]),d4=new Set(["default","gpt-2","phi-2","jina-es","jina-de","gigachat","jina-v2-es","jina-v2-de","a.x-4.0","mellum","modern-bert","jina-v1-en","jina-v2-code","roberta-bpe","exaone4"]);function fc(e){let t=e["tokenizer.ggml.pre"];return typeof t=="string"&&t.length>0?t:null}function p4(e){let t=fc(e);return t!==null&&Ag.has(t)}function Gn(...e){return{type:"Sequence",pretokenizers:[...e.map(t=>({type:"Split",pattern:{Regex:t},behavior:"Isolated",invert:!1})),{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}]}}var f4=`{% for message in messages %}<|im_start|>{{ message['role'] }}
{{ message['content'] }}<|im_end|>
{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}`;function kg(e){let t=xg(e,"tokenizer.ggml.tokens"),r=e["tokenizer.ggml.merges"]??[],n={};if(t.forEach((s,o)=>{n[s]=o}),m4(e))return{version:"1.0",truncation:null,padding:null,added_tokens:pc(e,t),normalizer:{type:"Replace",pattern:{String:" "},content:"\u2581"},pre_tokenizer:{type:"Split",pattern:{String:" "},behavior:"MergedWithPrevious",invert:!1},post_processor:null,decoder:{type:"Sequence",decoders:[{type:"Replace",pattern:{String:"\u2581"},content:" "},{type:"ByteFallback"},{type:"Fuse"}]},model:{type:"BPE",dropout:null,unk_token:ba(t,e["tokenizer.ggml.unknown_token_id"])??null,continuing_subword_prefix:null,end_of_word_suffix:null,fuse_unk:!0,byte_fallback:!0,ignore_merges:!1,vocab:n,merges:yg(r)}};let a=p4(e)||fc(e)==="minicpm5";return{version:"1.0",truncation:null,padding:null,added_tokens:pc(e,t),normalizer:null,pre_tokenizer:g4(e),post_processor:w4(),decoder:h4(),model:{type:"BPE",dropout:null,unk_token:ba(t,e["tokenizer.ggml.unknown_token_id"])??null,continuing_subword_prefix:null,end_of_word_suffix:null,fuse_unk:!1,byte_fallback:!!e["tokenizer.ggml.byte_fallback"],ignore_merges:a,vocab:n,merges:yg(r)}}}function m4(e){let t=e["tokenizer.ggml.model"];return typeof t!="string"?!1:t==="llama"||t==="spm"||t.startsWith("gemma")||t.startsWith("llama")}function Sg(e){let t=xg(e,"tokenizer.ggml.tokens"),r=e["tokenizer.ggml.bos_token_id"],n=e["tokenizer.ggml.eos_token_id"],a=e["tokenizer.ggml.unknown_token_id"],s=e["tokenizer.ggml.padding_token_id"],o=Object.fromEntries(pc(e,t).map(i=>[i.id,{content:i.content,single_word:i.single_word,lstrip:i.lstrip,rstrip:i.rstrip,normalized:i.normalized,special:i.special}]));return{tokenizer_class:"Qwen2Tokenizer",model_max_length:b4(e)??e["general.context_length"]??32768,bos_token:ba(t,r),eos_token:ba(t,n),unk_token:ba(t,a),pad_token:ba(t,s),bos_token_id:Number.isInteger(r)?r:void 0,eos_token_id:Number.isInteger(n)?n:void 0,eos_token_ids:cs(e),unk_token_id:Number.isInteger(a)?a:void 0,pad_token_id:Number.isInteger(s)?s:void 0,chat_template:mc(e,t),added_tokens_decoder:o}}function mc(e,t=_4(e)){let r=e["tokenizer.chat_template"];if(typeof r=="string")return r;let n=e["tokenizer.ggml.chat_template"];return typeof n=="string"?n:t.includes("<|im_start|>")&&t.includes("<|im_end|>")?f4:null}function g4(e){let t=fc(e);if(t===null)return e["general.architecture"]==="qwen3"?Gn(vg):_g();if(c4.has(t))return Gn(n4);if(t==="minicpm5")return Gn("\\p{N}{1,3}",s4);if(t==="flint")return Gn(...i4);if(l4.has(t)||t==="qwen3")return Gn(vg);if(Ag.has(t))return Gn(a4);if(u4.has(t))return Gn(o4);if(d4.has(t))return _g();throw new Error(`Unsupported tokenizer.ggml.pre "${t}": this build has no split regex for it, and the GPT-2 fallback would tokenize the prompt differently from the reference implementation.`)}function _g(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!0,use_regex:!0}}function h4(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}}function w4(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}}function pc(e,t){let r=Array.isArray(e["tokenizer.ggml.token_type"])?e["tokenizer.ggml.token_type"]:[],n=new Set;for(let a of[e["tokenizer.ggml.bos_token_id"],e["tokenizer.ggml.eos_token_id"],e["tokenizer.ggml.unknown_token_id"],e["tokenizer.ggml.padding_token_id"]])Number.isInteger(a)&&n.add(a);for(let a=0;a<r.length;++a)v4(r[a])&&n.add(a);return[...n].filter(a=>a>=0&&a<t.length).sort((a,s)=>a-s).map(a=>({id:a,content:t[a],single_word:!1,lstrip:!1,rstrip:!1,normalized:!1,special:!0}))}function b4(e){let t=e["general.architecture"];return typeof t=="string"?e[`${t}.context_length`]:void 0}function v4(e){return Number.isInteger(e)&&e!==1&&e!==6}function yg(e){return Array.isArray(e)?e.map(t=>Array.isArray(t)?t.join(" "):String(t)):[]}function ba(e,t){return typeof t=="number"&&Number.isInteger(t)&&t>=0&&t<e.length?e[t]:void 0}function xg(e,t){let r=e[t];if(!Array.isArray(r)||r.some(n=>typeof n!="string"))throw new Error(`GGUF metadata ${t} must be a string array`);return r}function _4(e){let t=e["tokenizer.ggml.tokens"];return Array.isArray(t)&&t.every(r=>typeof r=="string")?t:[]}var y4="webgpu-prefix-snapshots",Tg="prefix-snapshot-slot",Cg=3,A4=0;function k4(e){if(typeof e!="object"||e===null)return!1;let t=e;return t.version===Cg&&typeof t.chunkKey=="string"&&typeof t.identity=="string"&&typeof t.length=="number"&&typeof t.layout=="string"&&Array.isArray(t.sizes)&&t.sizes.every(r=>Number.isInteger(r)&&r>0)}function S4(){let e=globalThis.crypto?.randomUUID?.();return e?`prefix-snapshot:${e}`:`prefix-snapshot:${Date.now().toString(36)}:${(++A4).toString(36)}:${Math.random().toString(36).slice(2)}`}function x4(e){let t=async()=>{let r=await e.getMeta(Tg);return k4(r)?r:null};return{async load(r){try{let n=await t();if(!n||n.identity!==r)return null;let a=[],s=0;for(let o of n.sizes){let i=await e.get(n.chunkKey,s,s+o);if(!i||i.byteLength!==o)return null;a.push(i),s+=o}return{identity:r,length:n.length,layout:n.layout,chunks:a}}catch(n){return ai("prefix snapshot read failed",n),null}},async save(r){let n=S4();try{let a=await t(),s=0,o=[];for(let u of r.chunks)await e.put(n,s,s+u.byteLength,u),o.push(u.byteLength),s+=u.byteLength;let i={version:Cg,chunkKey:n,identity:r.identity,length:r.length,layout:r.layout,sizes:o};if(await e.putMeta(Tg,i),a&&a.chunkKey!==n)try{await e.delete?.(a.chunkKey)}catch(u){ai("previous prefix snapshot cleanup failed",u)}}catch(a){ai("prefix snapshot write failed",a);try{await e.delete?.(n)}catch(s){ai("prefix snapshot cleanup failed",s)}}}}}function Pg(e=y4){let t=fs(e);return t?x4(t):null}function ai(e,t){typeof console<"u"&&console.warn(`${e}: ${t instanceof Error?t.message:String(t)}`)}function hc(e,t,r,n="main"){let a=Ig(e,t);if(a.toLowerCase().endsWith(".gguf"))return!/^https?:/i.test(a)&&typeof location<"u"?new URL(a,location.href).href:a;let s=r??e.defaultFile?.(a);if(s===void 0)throw new Error(`name the GGUF \`file\` to load from ${a}`);return Cl(a,{revision:n,file:s})}function Ig(e,t){let r=t??e.defaultModelId;if(r===void 0)throw new Error("name the model to load: a repo id, or a .gguf URL");return r}function Bg(e,t,r={}){let n=Ig(e,t),a=n.toLowerCase().endsWith(".gguf"),s=r.mmproj!==void 0?r.mmproj:a?null:e.defaultMmproj?.(n)??null;if(s===null)return null;if(!e.vision&&!e.speech)throw new Error(`this model takes no projector (mmproj "${s}")`);if(/^[a-z][a-z0-9+.-]*:/i.test(s))return s;if(a){let o=hc(e,n,r.file,r.revision??"main");return o.slice(0,o.lastIndexOf("/")+1)+s}return Cl(n,{revision:r.revision??"main",file:s})}function T4(e){let t=e.metadata,r=Sg(t),n=kg(t),a=new g2(n,r),s=typeof r.chat_template=="string"?r.chat_template:mc(t);if(!s)throw new Error("GGUF metadata is missing a chat_template");let o=C4(r,a);return{tokenizer:a,chatTemplate:new Xp(ap(s)),tokenizerConfig:r,eosTokenIds:o}}function C4(e,t){let r=new Set,n=e.eos_token_id;mn(n)&&r.add(n);let a=e.eos_token_ids;if(Array.isArray(a))for(let s of a)mn(s)&&r.add(s);if(typeof e.eos_token=="string"){let s=t.token_to_id(e.eos_token);mn(s)&&r.add(s)}if(r.size===0)throw new Error("GGUF metadata has no eos token id");return[...r]}function P4(e){let t=a=>{let s=e[`general.sampling.${a}`];return mn(s)?s:void 0},r={temperature:t("temp"),top_k:t("top_k"),top_p:t("top_p"),min_p:t("min_p")},n=Object.fromEntries(Object.entries(r).filter(([,a])=>a!==void 0));return Object.freeze({do_sample:Object.keys(n).length>0,...n})}var I4=Object.freeze({do_sample:!1});function gc(){return typeof requestAnimationFrame=="function"?new Promise(e=>requestAnimationFrame(()=>e())):new Promise(e=>setTimeout(e,0))}function Eg(e,t=null,r={}){return Xm(async n=>{let a=hc(e,t,r.file,r.revision??"main"),s=Bg(e,t,r),o={fetch:Pl(r),signal:r.signal},[i,u]=await Promise.all([lc(a,o),s===null?null:lc(s,o)]);return Rg([i,u])??e.unsupportedReason(i,n,u)})}function Rg(e){let t=new Set;for(let r of e)for(let n of r?.names()??[]){let{type:a,typeName:s}=r.info(n);[0,1,8,12,14].includes(a)||t.add(s??String(a))}return t.size===0?null:`This model uses weight formats this build does not include: ${[...t].join(", ")}`}function B4(e,t){if(!e)return null;let{features:r}=Oo(t.device),n=[r.shaderF16?"f16":"no-f16",r.subgroups?"sg":"no-sg",r.subgroupMatrix?"sgmat":"no-sgmat"].join("+");return`gguf|${e}|${n}`}async function Gg(e,t,r=null,n={}){let a=n.onProgress??(()=>{}),s=hc(e,r,n.file,n.revision??"main"),o=Bg(e,r,n),i=Pl(n);a({status:"init",message:"Requesting WebGPU device"});let u=n.runtime??await L2(n.runtimeOptions??{}),l=!n.runtime;u.captureShaders=!0;let c=async()=>{},d=null,p=null,f=null,m=(h,g)=>{e.disposeGenerationState?e.disposeGenerationState(h,g):g.cache.dispose?.()};try{a({status:"init",message:"Reading model headers"});let h={fetch:i,signal:n.signal,cache:n.cache,force:n.force,cacheName:n.cacheName},[g,w]=await Promise.allSettled([Yr.open(s,h),o===null?Promise.resolve(null):Yr.open(o,h)]);if(g.status==="rejected"||w.status==="rejected")throw g.status==="fulfilled"&&await g.value.close().catch(()=>{}),w.status==="fulfilled"&&await w.value?.close().catch(()=>{}),g.status==="rejected"?g.reason:w.reason;let b=g.value,v=w.value;c=v?async()=>{await Promise.all([b.close(),v.close()])}:()=>b.close(),a({status:"init",message:"Checking device support"});let y=Rg([b,v])??await e.unsupportedReason(b,u.device,v);if(y)throw new Error(y);let S=v?[b,v]:[b],C=S.reduce((E,X)=>E+X.names().reduce((z,G)=>z+X.info(G).byteLength,0),0),k=0,B=E=>{k+=E,a({status:"weights",kind:"bytes",loaded:k,total:C||null,fraction:C?Math.min(1,k/C):void 0,message:"Streaming weights"})};for(let E of S)E.onByteProgress=B;a({status:"weights",message:"Preparing weight upload"});let P=e.loadModel(u,b,E=>{!mn(E.total)||E.total<=0||a({status:"weights",kind:"tensors",loaded:E.processed,total:E.total,message:"Uploading to GPU"})},v,n.signal);a({status:"tokenizer",message:"Loading tokenizer"});let[_,x]=await Promise.allSettled([gc().then(()=>T4(b)),P]);if(x.status==="fulfilled"&&(d=x.value),_.status==="rejected")throw _.reason;if(x.status==="rejected")throw x.reason;let{tokenizer:R,chatTemplate:O,tokenizerConfig:N,eosTokenIds:Q}=_.value,V=x.value;await c(),n.signal?.throwIfAborted();let $=p=e.createGenerationState(V,n.maxLength),M=n.primeSystemPrompt,j=f=t({runtime:u,ownsRuntime:l,model:V,tokenizer:R,chatTemplate:O,tokenizerConfig:N,eosTokenIds:Q,generationState:$,defaultMaxNewTokens:e.defaultMaxNewTokens,prefixSnapshotStore:n.prefixSnapshotStore===void 0?Pg():n.prefixSnapshotStore,prefixSnapshotIdentity:B4(b.resourceIdentity&&(e.vision||e.speech?`${b.resourceIdentity}|multimodal`:b.resourceIdentity),u),vision:e.vision?.(V)??null,speech:e.speech?.(V)??null,generationConfig:V.canSample?.()?P4(b.metadata):I4,createGenerationState:()=>e.createGenerationState(V,n.maxLength),disposeGenerationState:E=>m(V,E)});n.chatTemplateArgs&&(j.chatTemplateArgs={...n.chatTemplateArgs}),a({status:"weights",kind:"tensors",message:M?"Warming up kernels and caching system prompt":"Compiling kernels (warmup)"}),await gc();let A=typeof V.warmRemainingPrefill=="function";await V.warmup($.cache,A?{deferRemainingPrefill:!0}:void 0),n.signal?.throwIfAborted();let T=n.decodePipelineDepth??e.decodePipelineDepth;if(T===void 0&&a({status:"weights",kind:"tensors",message:"Tuning decode pipeline"}),await j.tuneDecodePipeline({decodePipelineDepth:T}),n.signal?.throwIfAborted(),M!==void 0){a({status:"weights",kind:"tensors",message:"Caching system prompt"}),await gc();let E=j.create_session();try{let X=[{role:"system",content:M}];await E.prime(X,n.signal?{signal:n.signal}:{}).result}finally{await E.dispose()}n.signal?.throwIfAborted()}return a({status:"ready",message:"Ready",fraction:1}),A&&V.warmRemainingPrefill($.cache).catch(()=>{}),j}catch(h){if(await c().catch(()=>{}),f)await f.dispose().catch(()=>{});else if(d){let g=d,w=p;for(let b of[()=>w&&m(g,w),()=>g.dispose()])try{b()}catch{}}throw l&&await u.destroy().catch(()=>{}),h}}var si=class{#e;constructor(t){this.#e=t}create_session(t={}){return this.#e.create_session(t)}get generation_config(){return this.#e.generation_config}get contextLength(){return this.#e.contextLength}get lost(){return this.#e.lost}deviceInfo(){return this.#e.deviceInfo()}memoryStats(){return this.#e.runtime.memoryStats()}renderedShaders(){return this.#e.runtime.getRenderedShaders()}dispose(){return this.#e.dispose()}};var Lg=new Map;function Jr(e,t={}){let r=Lg.get(e);r||(r=ns(e),Lg.set(e,r));let n=r.manifest.conformance,a=t.conformance;if(n==="internal-lowering"&&a!==n)throw new Error(`${e}: this package is an internal tensor lowering, not the standard operator; graph callers must pass { conformance: "internal-lowering" } after performing the lowering explicitly`);if(a!==void 0&&n!==a)throw new Error(`${e}: graph caller expects conformance ${a}, but the manifest declares ${n??"standard"}`);return r}var E4=new Set;function qg(e){E4.add(e)}var R4=new Set(["intermediate","output"]),lt=class{tensors=[];nodes=[];aliases=[];boundNames=new Map;ownedHandles=new WeakSet;nodesByTensor=new Map;viewChildren=new Map;get nodeCount(){return this.nodes.length}input(t,r,n){return this.declareTensor("input",t,r,n)}weight(t,r,n){return this.declareTensor("weight",t,r,n)}state(t,r,n){return this.declareTensor("state",t,r,n)}stepInput(t,r,n){return this.declareTensor("stepInput",t,r,n)}uniform(t,r){if(r<=0||r%4!==0)throw new Error(`graph uniform "${t}" needs a positive multiple-of-4 byte length (got ${r})`);return this.declareTensor("uniform",t,"uint32",[r/4])}scratch(t,r,n){return this.declareTensor("intermediate",t,r,n)}view(t,r,n,a,s){if(t=this.currentTensor(t),t.logicalDtype!==void 0&&!bn(t.logicalDtype,n))throw new Error(`graph view of "${t.name}" cannot reinterpret logical dtype ${t.logicalDtype} as ${n} storage`);if(!Number.isInteger(r)||r<0)throw new Error(`graph view of "${t.name}" needs a non-negative integer elementOffset (got ${r})`);if(!a.every(p=>Number.isInteger(p)&&p>=0))throw new Error(`graph view "${s??`${t.name}.view@${r}`}" has a non-integer shape [${a}]`);let o=Ae(n),i=r*o,u=ke(a)*o,l=ke(t.shape)*Ae(t.dtype);if(i+u>l)throw new Error(`graph view "${s??`${t.name}.view@${r}`}" exceeds its parent "${t.name}" range (${i}+${u} > ${l} bytes)`);let c=t.logicalDtype,d=t.id;if(t.view){let f=t.view.elementOffset*Ae(t.dtype)+i;if(f%o!==0)throw new Error(`graph view "${s??`${t.name}.view@${r}`}" cannot represent byte offset ${f} in ${n} elements`);t=this.currentTensor(this.tensors[t.view.parent]),r=f/o}return this.addView(t,r,n,a,s,c,d)}storageView(t,r={}){if(t=this.currentTensor(t),t.logicalDtype===void 0||t.logicalDtype===t.dtype||!bn(t.logicalDtype,t.dtype))throw new Error(`graph tensor "${t.name}" has no logical/storage projection to expose`);let n=r.dtype??t.dtype;if(!bn(t.logicalDtype,n))throw new Error(`storage view of "${t.name}" cannot represent logical dtype ${t.logicalDtype} as ${n} storage`);if(Ae(n)!==Ae(t.dtype))throw new Error(`storage view of "${t.name}" cannot change its element size (${t.dtype} to ${n})`);let a=r.shape??t.shape;if(ke(a)!==ke(t.shape))throw new Error(`storage view of "${t.name}" cannot change its element count (${ke(t.shape)} to ${ke(a)})`);let s=t.id,o=0;if(t.view){let i=t.view.elementOffset*Ae(t.dtype),u=Ae(n);if(i%u!==0)throw new Error(`storage view of "${t.name}" cannot represent byte offset ${i} in ${n} elements`);t=this.currentTensor(this.tensors[t.view.parent]),o=i/u}return this.addView(t,o,n,a,r.name,void 0,s)}addView(t,r,n,a,s,o,i){if(t.view)throw new Error(`internal error: graph view "${t.name}" was not flattened to its storage parent`);if(t.kind==="uniform")throw new Error(`graph uniform "${t.name}" cannot be viewed`);if(!Number.isInteger(r)||r<0)throw new Error(`graph view of "${t.name}" needs a non-negative integer elementOffset (got ${r})`);let u={id:this.tensors.length,kind:"intermediate",dtype:n,...o===void 0?{}:{logicalDtype:o},shape:[...a],name:s??`${t.name}.view@${r}`,view:{parent:t.id,elementOffset:r}};this.tensors.push(u),this.ownedHandles.add(u);let l=this.viewChildren.get(i);return l?l.push(u.id):this.viewChildren.set(i,[u.id]),u}op(t,r,n={}){let a=this.opFor(t,n.conformance),s={},o={},i={};for(let[p,f]of Object.entries(r)){if(f===void 0)continue;if(!G4(f))throw new Error(`${t}: binding "${p}" is not a graph tensor \u2014 scalars belong in options.args`);let m=this.currentTensor(f);i[p]=m,s[p]=m,o[p]=m.id}let u=a.inferOutputs(s,{args:n.args,attrs:n.attrs}),l={},c={};for(let[p,f]of Object.entries(u)){let m=i[p],h=Le(f.dtype);if(m!==void 0){if(!Vs(m.shape,f.shape))throw new Error(`${t}: output "${p}" bound to tensor "${m.name}" of shape [${m.shape}] but the manifest infers [${f.shape}]`);l[p]=this.withLogicalDtype(m,f.logicalDtype),c[p]=m.id;continue}let g=this.withLogicalDtype(this.allocate("intermediate",`${t.split(".").pop()}.${p}#${this.nodes.length}`,h,f.shape),f.logicalDtype);l[p]=g,c[p]=g.id}let d=this.nodes.length;this.nodes.push({op:t,...n.conformance?{conformance:n.conformance}:{},inputs:o,outputs:c,...n.args?{args:{...n.args}}:{},...n.attrs?{attrs:{...n.attrs}}:{},...n.tunables?{tunables:{...n.tunables}}:{}});for(let p of new Set([...Object.values(o),...Object.values(c)])){let f=this.nodesByTensor.get(p);f||(f=new Set,this.nodesByTensor.set(p,f)),f.add(d)}return l}alias(t,r){if(t=this.currentTensor(t),r=this.currentTensor(r),t.id===r.id)throw new Error(`alias: a tensor cannot alias itself ("${t.name}")`);for(let[s,o]of[["from",t],["to",r]])if(!R4.has(o.kind))throw new Error(`alias: ${s} tensor "${o.name}" is ${o.kind}; only intermediate/output tensors can share a buffer`);if(t.dtype!==r.dtype)throw new Error(`alias("${t.name}","${r.name}"): dtype mismatch (${t.dtype} vs ${r.dtype})`);let n=ke(t.shape)*Ae(t.dtype),a=ke(r.shape)*Ae(r.dtype);if(n!==a)throw new Error(`alias("${t.name}","${r.name}"): byte-length mismatch (${n} vs ${a}); a shared buffer needs an exact-size fit`);this.aliases.push({from:t.id,to:r.id})}output(t,r){let n=this.currentTensor(t),a={...n,kind:"output",name:r??n.name};return this.tensors[n.id]=a,this.ownedHandles.add(a),a}finish(t){return Object.freeze({name:t.name,params:Object.freeze({...t.params??{}}),tensors:Object.freeze([...this.tensors]),nodes:Object.freeze([...this.nodes]),...this.aliases.length>0?{aliases:Object.freeze([...this.aliases])}:{}})}opFor(t,r){return Jr(t,{conformance:r})}declareTensor(t,r,n,a){let s=this.boundNames.get(r);if(s)throw new Error(`graph tensor name "${r}" already declared (${s.kind})`);let o=this.allocate(t,r,n,a);return this.boundNames.set(r,o),o}allocate(t,r,n,a){if(!a.every(o=>Number.isInteger(o)&&o>=0))throw new Error(`graph tensor "${r}" has a non-integer shape [${a}]`);let s={id:this.tensors.length,kind:t,dtype:n,shape:[...a],name:r};return this.tensors.push(s),this.ownedHandles.add(s),s}withLogicalDtype(t,r){if(t=this.currentTensor(t),t.logicalDtype===r)return t;let{logicalDtype:n,...a}=t,s={...a,...r===void 0?{}:{logicalDtype:r}},o=new Map([[t.id,s]]),i=[{parent:t,nextLogicalDtype:r}];for(;i.length>0;){let{parent:u,nextLogicalDtype:l}=i.pop();for(let c of this.viewChildren.get(u.id)??[]){let d=this.tensors[c];if(d.logicalDtype!==u.logicalDtype)continue;if(l!==void 0&&!bn(l,d.dtype))throw new Error(`graph view "${d.name}" cannot represent logical dtype ${l} as ${d.dtype} storage`);let{logicalDtype:p,...f}=d,m={...f,...l===void 0?{}:{logicalDtype:l}};o.set(d.id,m),i.push({parent:d,nextLogicalDtype:l})}}this.revalidateNodesReferencing([...o.keys()],o);for(let[u,l]of o)this.tensors[u]=l,this.ownedHandles.add(l);return s}revalidateNodesReferencing(t,r){let n=new Set;for(let a of t)for(let s of this.nodesByTensor.get(a)??[])n.add(s);for(let a of n){let s=this.nodes[a],o={};for(let[i,u]of Object.entries({...s.inputs,...s.outputs}))o[i]=r.get(u)??this.tensors[u];try{this.opFor(s.op,s.conformance).inferOutputs(o,{args:s.args,attrs:s.attrs})}catch(i){let u=i instanceof Error?i.message:String(i);throw new Error(`graph node ${a} (${s.op}) became invalid after a logical dtype refinement: ${u}`,{cause:i})}}}currentTensor(t){if(!this.ownedHandles.has(t))throw new Error(`graph tensor "${t.name}" (id ${t.id}) does not belong to this builder`);let r=this.tensors[t.id];if(r===void 0)throw new Error(`graph tensor "${t.name}" (id ${t.id}) does not belong to this builder`);return r}};function vt(e){let t={},r={},n=new Map;return{w:(o,i)=>{let u=n.get(o);if(u)return u;t[o]=i;let l=e.weight(o,i.dtype,i.shape);return n.set(o,l),l},state:(o,i)=>(r[o]=i,e.state(o,i.dtype,i.shape)),boundWeights:t,states:r}}function G4(e){return e!==null&&typeof e=="object"&&typeof e.id=="number"&&typeof e.dtype=="string"&&Array.isArray(e.shape)}var oi=class{#e=!1;#t;constructor(t){this.#t=t}get config(){return this.#t.config}async checkDeviceSupport(t){return await this.#t.checkSupport(t)}async unsupportedReason(t){return(await this.#t.checkSupport(t)).reason}async load(t,r={}){if(this.#e)throw new Error("prepared model has already been loaded or closed");this.#e=!0;try{let n=this.#t.loadWeights(t,r),a=Promise.resolve(this.#t.checkSupport(t.device)),[s,o]=await Promise.allSettled([a,n]);if(s.status==="fulfilled"&&s.value.reason)throw o.status==="fulfilled"&&o.value?.dispose?.(),new Error(s.value.reason);if(o.status==="rejected")throw o.reason;if(s.status==="rejected")throw o.value?.dispose?.(),s.reason;return o.value}finally{await this.#t.closeSource()}}async close(){this.#e||(this.#e=!0,await this.#t.closeSource())}};async function Og(e,t){let r=t.configTensors?.(e)??[];return new Map(await Promise.all(r.map(async n=>[n,await e.tensorBytes(n)])))}function Dg(e,t,r,n,a){let s=t.configFromGGUF(e,a);return new oi({config:s,checkSupport:o=>t.checkSupport(s,o,e),loadWeights:(o,{signal:i})=>t.loadWeights(o,e,s,{onProgress:r,signal:i}),closeSource:n})}async function L4(e,t,r){let n=e instanceof Yr?e:await Yr.open(e,t);try{return Dg(n,r,t.onProgress??null,()=>n.close(),await Og(n,r))}catch(a){throw await n.close(),a}}async function q4(e,t,r){return Dg(e,r,t.onProgress??null,async()=>{},await Og(e,r))}async function O4(e){return new gr(await uc(e))}function Yt(e){let t=(n,a={})=>L4(n,a,e(a)),r=async(n,a)=>(await q4(a.gguf,{onProgress:a.onProgress},e(a))).load(n);return{prepare:t,async load(n,a,s={}){return(await t(a,s)).load(n,{signal:s.signal})},async loadGGUFFile(n,a,s={}){return r(n,{...s,gguf:await O4(a)})},loadGGUF:r}}function Mg(e){let t=new Map;for(let _ of e.tensors)t.set(_.id,_);let r=_=>t.get(_)?.view?.parent??_,n=new Map;for(let _ of e.tensors)_.kind==="intermediate"&&!_.view&&n.set(_.id,_);let a=new Map,s=new Map,o=new Map,i=M4(e,t),u=new Map,l=new Map,c=new Map;e.nodes.forEach((_,x)=>{for(let R of Object.values(_.inputs)){let O=r(R);n.has(O)&&(l.has(O)||l.set(O,x),c.set(O,x))}for(let R of Object.values(_.outputs)){let O=r(R);n.has(O)&&(u.has(O)||u.set(O,x),c.set(O,x))}});let d=_=>i.rootOf.get(_)??_,p=new Map,f=new Map,m=new Map;for(let _ of n.values()){let x=d(_.id),R=u.get(_.id),O=c.get(_.id),N=l.get(_.id);if(R!==void 0){let Q=p.get(x);(Q===void 0||R<Q)&&p.set(x,R)}if(O!==void 0){let Q=f.get(x);(Q===void 0||O>Q)&&f.set(x,O)}if(N!==void 0){let Q=m.get(x);(Q===void 0||N<Q)&&m.set(x,N)}}let h=new Set;for(let _ of e.tensors){if(!_.view||_.kind==="intermediate")continue;let x=r(_.id);n.has(x)&&h.add(d(x))}for(let[_,x]of i.byRoot){let R=p.get(_),O=R!==void 0&&(m.get(_)??1/0)<R;(x.some(N=>!n.has(N))||R===void 0||O)&&h.add(_)}for(let _ of h){let x=i.byRoot.get(_)??[_],R=t.get(x[0]),O=Wg(s,Ln(R));o.set(O,R);for(let N of x)a.set(N,O)}let g=[],w=0,b=new Map,v=new Set;for(let _ of n.values()){w+=Ln(_);let x=u.get(_.id),R=l.get(_.id);if(x===void 0&&R!==void 0)throw new Error(`graph "${e.name}": intermediate "${_.name}" (id ${_.id}) is read by node ${R} but no node produces it`);if(!i.rootOf.has(_.id)){if(x===void 0){g.push(_.name);continue}if(R!==void 0&&R<x)throw new Error(`graph "${e.name}": intermediate "${_.name}" (id ${_.id}) is read by node ${R} before it is produced by node ${x}; pooling cannot order that lifetime`)}let N=d(_.id);if(h.has(N)||v.has(N))continue;v.add(N);let Q=f.get(N),V=b.get(Q);V?V.push(N):b.set(Q,[N])}let y=[],S=new Map;for(let _=0;_<e.nodes.length;++_){let x=b.get(_-1);if(x)for(let R of x)y.push(S.get(R));for(let R of Object.values(e.nodes[_].outputs)){let O=r(R),N=n.get(O);if(!N||a.has(O))continue;let Q=d(O),V=S.get(Q);V===void 0&&(V=D4(y,s,o,N),S.set(Q,V));for(let $ of i.byRoot.get(Q)??[O])a.set($,V)}}let C=0;for(let _ of s.values())C+=_;let k=new Map;for(let _ of v){let x=p.get(_),R=f.get(_);k.set(x,(k.get(x)??0)+1),k.set(R+1,(k.get(R+1)??0)-1)}let B=0,P=0;for(let _ of[...k.keys()].sort((x,R)=>x-R))B+=k.get(_),B>P&&(P=B);return{slots:a,slotByteLengths:s,slotAllocations:o,diagnostics:{intermediateCount:n.size,slotCount:s.size,peakLiveSlots:h.size+P,naiveBytes:w,pooledBytes:C,deadTensors:g}}}function Wg(e,t){let r=e.size;return e.set(r,t),r}function D4(e,t,r,n){let a=Ln(n),s=-1,o=1/0,i=-1,u=-1;for(let d=0;d<e.length;++d){let p=t.get(e[d]);p>=a&&p<o&&(o=p,s=d),p>u&&(u=p,i=d)}let l=s>=0?s:i;if(l<0){let d=Wg(t,a);return r.set(d,n),d}let c=e[l];return e.splice(l,1),t.get(c)<a&&(t.set(c,a),r.set(c,n)),c}function M4(e,t){let r=new Map,n=new Map,a=e.aliases??[];if(a.length===0)return{rootOf:r,byRoot:n};let s=new Map,o=l=>{let c=l;for(;s.get(c)!==c;)c=s.get(c);let d=l;for(;s.get(d)!==c;){let p=s.get(d);s.set(d,c),d=p}return c},i=l=>{s.has(l)||s.set(l,l)};for(let{from:l,to:c}of a){for(let[d,p]of[["from",l],["to",c]]){let f=t.get(p);if(!f)throw new Error(`graph "${e.name}": alias ${d} tensor id ${p} does not exist`);if(f.kind!=="intermediate"&&f.kind!=="output")throw new Error(`graph "${e.name}": alias ${d} tensor "${f.name}" is ${f.kind}; only intermediate/output tensors can share a buffer`)}i(l),i(c),s.set(o(l),o(c))}let u=new Map;for(let l of s.keys()){let c=o(l),d=u.get(c);d?d.push(l):u.set(c,[l])}for(let l of u.values()){let c=t.get(l[0]);for(let p of l){let f=t.get(p);if(f.dtype!==c.dtype)throw new Error(`graph "${e.name}": aliased tensors "${c.name}" (${c.dtype}) and "${f.name}" (${f.dtype}) must be the same dtype to share a buffer`);if(Ln(f)!==Ln(c))throw new Error(`graph "${e.name}": aliased tensors "${c.name}" (${Ln(c)} B) and "${f.name}" (${Ln(f)} B) must be the same byte length to share a buffer`)}let d=o(l[0]);n.set(d,l);for(let p of l)r.set(p,d)}return{rootOf:r,byRoot:n}}function Ln(e){return Ye(e.dtype,ke(e.shape))}function _c(e,t,r){let n=(o,i)=>[...new Set([...o,...i])],a=e.reason??(t.reason===null?null:`${r}: ${t.reason}`),s=e.footprint&&t.footprint?{totalBytes:e.footprint.totalBytes+t.footprint.totalBytes,buffers:[...e.footprint.buffers,...t.footprint.buffers].sort((o,i)=>i.bytes-o.bytes)}:e.footprint??t.footprint;return{ok:e.ok&&t.ok,problems:[...e.problems,...t.problems],reason:a,opIds:n(e.opIds,t.opIds),selectableVariants:n(e.selectableVariants,t.selectableVariants),footprint:s}}function W4(e,t){let r=he(e.dtype,e.shape);return t?{...r,layout:t,byteLength:Ye(e.dtype,r.size,t)}:r}function F4(e,t,r){let n={};for(let[a,s]of Object.entries({...e.inputs,...e.outputs})){let o=t.get(s);o&&(n[a]=W4(o,r.has(s)?"packed":void 0))}return{resources:n,...e.args?{args:{...e.args}}:{},...e.attrs?{attrs:{...e.attrs}}:{},...e.tunables?{tunables:{...e.tunables}}:{}}}function wc(e,t,r){return Fg(e,t,r).selected!==null}function Fg(e,t,r){try{let n=e.explain({device:t},r);return{selected:n.selected,runnable:n.candidates.filter(a=>a.ok).map(a=>a.id)}}catch(n){return{selected:null,runnable:[],error:n instanceof Error?n.message:String(n)}}}function bc(e,t){return{...e,features:[...e.features,...t]}}function Q4(e,t,r){let n=new Set(t.features),a=jr.filter(o=>!n.has(o));for(let o of a)if(wc(e,bc(t,[o]),r))return[o];let s=[];for(let o of a)if(s.push(o),wc(e,bc(t,s),r))return j4(e,t,r,s);return[]}function j4(e,t,r,n){let a=[...n];for(let s of n){let o=a.filter(i=>i!==s);o.length<a.length&&wc(e,bc(t,o),r)&&(a=o)}return a}function ii(e){let t=e/1048576;return`${t.toFixed(t<1e4?1:0)} MB`}var N4=null;function hs(e,t){let r=Ue(t),n=Nt(r),a=[],s=[...e.tensors??[]],o=new Set,i=new Set,u=new Set,l=new Set;el(()=>{for(let{op:p,request:f,id:m}of e.opRequests??[]){let h=m??p.manifest.id;u.add(h);let g=`${h}|${Tr(f)}`;if(o.has(g))continue;o.add(g),N4?.({op:p,request:f,id:h},g);let{selected:w,runnable:b,error:v}=Fg(p,n,f);for(let k of b)l.add(`${h}#${k}`);if(w!==null)continue;if(v!==void 0){let k=`${h}|fault`;if(i.has(k))continue;i.add(k),a.push({kind:"feature",op:h,missingFeatures:[],message:`${h} could not be evaluated for this device: ${v}`});continue}let y=Q4(p,n,f),S=`${h}|${y.join(",")}`;if(i.has(S))continue;i.add(S);let C=y.map(k=>`"${k}"`).join(", ");a.push({kind:"feature",op:h,missingFeatures:y,message:y.length>0?`${h} needs WebGPU ${lr(y.length,"feature")} ${C}, which this device lacks`:`${h} has no runnable variant on this device`})}});let c=r.limits.maxBufferSize;if(typeof c=="number"){let p=new Set;for(let f of s){let m=vc(f);m<=c||p.has(m)||(p.add(m),a.push({kind:"buffer",tensor:f.name,bytes:m,limitBytes:c,message:`tensor ${f.name} needs a ${ii(m)} buffer, over this device's ${ii(c)} maxBufferSize`}))}}let d=r.limits.maxStorageBufferBindingSize;if(typeof d=="number"){let p=new Set;for(let f of s){if(!f.boundWhole)continue;let m=vc(f);m<=d||p.has(m)||typeof c=="number"&&m>c||(p.add(m),a.push({kind:"binding",tensor:f.name,bytes:m,limitBytes:d,message:`tensor ${f.name} needs a ${ii(m)} storage binding, over this device's ${ii(d)} maxStorageBufferBindingSize`}))}}return{ok:a.length===0,problems:a,reason:a.length>0?K4(a):null,opIds:[...u],selectableVariants:[...l],footprint:z4(s)}}function z4(e){let t=e.filter(n=>n.allocation);if(t.length===0)return null;let r=t.filter(n=>!n.released).map(n=>({name:n.name,bytes:vc(n)})).sort((n,a)=>a.bytes-n.bytes);return{totalBytes:r.reduce((n,a)=>n+a.bytes,0),buffers:r}}function vc(e){let t=e.byteLength??Ye(Le(e.dtype),ke(e.shape));if(!Number.isSafeInteger(t)||t<0)throw new Error(`support tensor ${e.name} has invalid byteLength ${t}`);let r=ct(t);if(!Number.isSafeInteger(r))throw new Error(`support tensor ${e.name} byte length exceeds safe integer storage`);return r}function Qg(e){let t=new Map(e.tensors.map(u=>[u.id,u])),r=e.nodes.map(u=>Jr(u.op,{conformance:u.conformance})),n=u=>t.get(u)?.view?.parent??u,a=new Map;e.nodes.forEach((u,l)=>{for(let c of r[l].manifest.inputs){let d=u.inputs[c.role];if(d===void 0||t.get(d)?.dtype!=="uint8")continue;let p=n(d);if(a.has(p)&&a.get(p)!==c.layout)throw new Error(`graph "${e.name}": tensor ${p} is required in both packed and widened storage layouts`);a.set(p,c.layout)}});let s=new Set(e.tensors.filter(u=>a.get(n(u.id))==="packed").map(u=>u.id));for(let u of e.tensors){if(!s.has(u.id))continue;let l=t.get(n(u.id));if(!["weight","input","state"].includes(l.kind))throw new Error(`graph "${e.name}": packed storage requires an externally bound tensor, not ${l.kind}`);if(u.dtype!=="uint8"||u.view&&u.view.elementOffset%4!==0)throw new Error(`graph "${e.name}": packed views require uint8 dtype and word-aligned offsets`)}let o=e.nodes.map((u,l)=>({op:r[l],request:F4(u,t,s),id:u.op})),i=e.tensors.map(u=>({name:u.name,dtype:u.dtype,shape:u.shape,boundWhole:u.kind==="weight",...s.has(u.id)?{byteLength:Ye(u.dtype,ke(u.shape),"packed")}:{}}));return{opRequests:o,tensors:i}}function K4(e){let t=[...new Set(e.flatMap(s=>s.missingFeatures??[]))],r=e.filter(s=>s.kind==="buffer"||s.kind==="binding");if(t.length===0&&r.length===0)return`${e[0].message} \u2014 this build has no kernel for the quantization format combination this checkpoint uses. That is a coverage gap in the library, not a limitation of your GPU or browser, so switching either will not help.`;let n=t.length>0?`This GPU/browser is missing WebGPU ${lr(t.length,"feature")} ${t.map(s=>`"${s}"`).join(", ")} that the model needs`:r[0].message,a=t.length>0&&r.length>0?" Some weights also exceed the device's buffer limits.":"";return`${n} \u2014 it can't run here.${a} Try Chrome or Edge with up-to-date GPU drivers.`}var RE=BigInt(1e8),GE=new TextDecoder("utf-8",{fatal:!0});var yc=class{tensors=[];opRequests=[];#e=0;#t(t,r,n,a={},s){let o=he(Le(t),r),i=Ye(o.dtype,o.size,a.layout),u=s??i;if(!Number.isSafeInteger(u)||u<i)throw new Error(`support weights tensor requires ${i} bytes, but byteLength is ${u}`);let l=this.tensors.push({name:n??`buffer#${this.#e}`,dtype:t,shape:[...r],byteLength:ct(u),boundWhole:!0,allocation:!0})-1;this.#e+=1;let c=()=>{let d=this.tensors[l];d&&!d.released&&(this.tensors[l]={...d,released:!0})};return{...o,byteLength:i,...a.layout?{layout:a.layout}:{},buffer:{destroy:c},destroy:c}}recordSupportOp(t,r){this.opRequests.push({op:t,request:r})}asRuntime(t=null){let r=this;return{device:t,metadataOnly:!0,recordSupportOp:(n,a)=>r.recordSupportOp(n,a),empty:(n,a,s,o)=>r.#t(n,a,s,o),tensorFromTypedArray:(n,a,s,o)=>r.#t(n,a,void 0,o),allocateWeightsBuffer:({byteLength:n,dtype:a,shape:s,label:o})=>r.#t(a,s,o,void 0,n),writeWeightsRange:()=>{},readTensor:async n=>new Float32Array(n.size??1),createUniformU32:()=>({destroy(){}}),host:{writeBuffer:()=>{}},runProgram:async()=>{},runProgramSequence:async()=>{},prepareProgramSequence:async()=>[],enqueuePreparedProgramSequence:()=>{},queueIdle:async()=>{},copyBufferToBuffer:()=>{},clearBuffers:()=>{}}}};function pt(e,t,r,n){let a=Le(t);return e.metadataOnly?e.empty(a,r):e.tensorFromTypedArray(a,r,n())}function jg(e,t,r){let n=e.recordSupportOp;return n?(n.call(e,t,r),!0):!1}function Ng(e){let t=new Uint8Array(0),r=n=>(t.length<n&&(t=new Uint8Array(n)),t.subarray(0,n));return new Proxy(e,{get(n,a){if(a==="tensorBytes")return o=>r(n.info(o).byteLength);if(a==="tensorByteRange"&&typeof n.tensorByteRange=="function")return(o,i,u)=>r(u-i);if(a==="streamTensorRange"&&typeof n.streamTensorRange=="function")return async(o,i,u,l)=>{await l(r(u-i))};if(a==="streamTensors"&&typeof n.streamTensors=="function")return async(o,i)=>{for(let u of o)await i(u,r(n.info(u).byteLength))};if(a==="supportsDirectDestinations")return;if(a==="planReads")return()=>{};let s=Reflect.get(n,a,n);return typeof s=="function"?s.bind(n):s}})}async function zg(e){let t=new yc,r=t.asRuntime(e.device??null),n=await e.loadWeights(r),a=e.outsideGraphRequests?.(r,n)??[],s=await e.emitGraphs(r,n),i=(Array.isArray(s)?s:[s]).map(Qg);if(a.length>0&&i.length>0&&(i[0]={...i[0],opRequests:[...a,...i[0].opRequests]}),i.length===0||t.tensors.length===0)return i;let u=t.tensors.map(l=>({...l,boundWhole:!1}));return i.map((l,c)=>c===0?{...l,tensors:[...u,...l.tensors]}:l)}function Kg(e){let t={tensors:e.flatMap(r=>r.tensors),opRequests:e.flatMap(r=>r.opRequests)};return r=>{let n=hs(t,r);if(n.ok||e.length<=1)return n;let a=e.map(i=>hs(i,r)),s=new Set,o=[];for(let i of a)for(let u of i.problems){let l=u.kind==="feature"?`feature:${u.op??""}:${(u.missingFeatures??[]).join(",")}`:`${u.kind}:${u.bytes??""}`;s.has(l)||(s.add(l),o.push(u))}return{...n,problems:o,reason:a.find(i=>i.reason)?.reason??n.reason}}}var Ac=new WeakMap;qg(()=>{Ac=new WeakMap});var kc=class{slots=new Map;capacities=null;capacitySlots=new Map;persistent=new Map;rt;constructor(t){this.rt=t}reserve(t){if(this.slots.size>0||this.capacitySlots.size>0)throw new Error("Collector scratch capacity must be planned before materialization");let r=new Map;for(let n of t){let a=new Map;for(let s of n){let o=a.get(s.dtype)??0;a.set(s.dtype,o+1);let i=r.get(s.dtype)??[];i[o]=Math.max(i[o]??0,s.byteLength),r.set(s.dtype,i)}}this.capacities=r}resourcesForPlan(t){if(t.length===0)return{};if(this.capacities){let a=new Map,s={};for(let o of t){let i=a.get(o.dtype)??0;a.set(o.dtype,i+1);let u=this.capacities.get(o.dtype)?.[i];if(u===void 0||u<o.byteLength)throw new Error(`No capacity for scratch ${o.id}`);let l=this.capacitySlots.get(o.dtype);l||(l=[],this.capacitySlots.set(o.dtype,l));let c=l[i];c||(c=this.rt.empty(o.dtype,[u/Ae(o.dtype)],`webgpu-scratch-${o.id}`),l[i]=c),s[o.id]={runtime:c.runtime,dtype:o.dtype,shape:o.shape,size:ke(o.shape),byteLength:o.byteLength,byteOffset:c.byteOffset??0,buffer:c.buffer,destroy:vn}}return s}let r=new Map,n={};for(let a of t){let s=JSON.stringify([a.dtype,a.shape]),o=r.get(s)??0;r.set(s,o+1);let i=this.slots.get(s);i||(i=[],this.slots.set(s,i));let u=i[o];u===void 0&&(u=this.rt.empty(a.dtype,a.shape,`webgpu-scratch-${a.id}`),i.push(u)),n[a.id]=u}return n}persistentResourcesForPlan(t,r){let n={};for(let a of r){let s=`${t}\0${a.id}`,o=this.persistent.get(s);o===void 0&&(o=this.rt.empty(a.dtype,a.shape,`webgpu-const-scratch-${a.id}`),this.persistent.set(s,o)),n[a.id]=o}return n}clearPersistent(){let t=[];for(let[r,n]of this.persistent)try{n.destroy(),this.persistent.delete(r)}catch(a){t.push(a)}xt(t,"Collector constant-scratch cleanup failed")}clear(){let t=[];for(let[r,n]of this.capacitySlots){for(let a=n.length-1;a>=0;--a){let s=n[a];try{s.destroy(),n.splice(a,1)}catch(o){t.push(o)}}n.length===0&&this.capacitySlots.delete(r)}for(let[r,n]of this.slots){for(let a=n.length-1;a>=0;--a){let s=n[a];try{s.destroy(),n.splice(a,1)}catch(o){t.push(o)}}n.length===0&&this.slots.delete(r)}xt(t,"Collector scratch cleanup failed")}};function U4(e,t){let r=[e.manifest.id];for(let n of Object.keys(t.resources).sort()){let a=t.resources[n];if(a!==null&&typeof a=="object"&&"shape"in a&&"dtype"in a){let s=a;r.push(`${n}:[${s.shape.join(",")}]:${s.dtype}`)}else a==null?r.push(`${n}:-`):r.push(`${n}=${String(a)}`)}return r.push(`a:${JSON.stringify(t.args??{})}`),r.push(`t:${JSON.stringify(t.attrs??{})}`),r.push(`u:${JSON.stringify(t.tunables??{})}`),r.join("|")}var ui=class{programs=[];constantPrograms=[];constantProgramsReady=null;cleanups=[];scratchPool;rt;timing;preparedCache;collectMs=0;flushMs=0;constructor(t,r={}){if(this.rt=t,this.scratchPool=new kc(t),this.timing=r.timing===!0,r.cachePrepared){let n=Ac.get(t);n||(n=new Map,Ac.set(t,n)),this.preparedCache=n}else this.preparedCache=null}reserveScratchCapacities(t){this.scratchPool.reserve(t)}prepare(t,r){if(!this.preparedCache)return t.prepare(this.rt,r);let n=U4(t,r),a=this.preparedCache.get(n);if(a)return{program:a.program,plan:a.plan,request:r};let s=t.prepare(this.rt,r);return this.preparedCache.set(n,{program:s.program,plan:s.plan}),s}add(t,r,n={}){if(jg(this.rt,t,r))return;let a=this.timing?performance.now():0,s=this.prepare(t,r),o=this.materialize(s,n);return this.pushMaterialized(o),this.cleanups.push(o.cleanup),this.timing&&(this.collectMs+=performance.now()-a),s}get programCount(){return this.programs.length}programSlice(t,r){return this.programs.slice(t,r)}addPrepared(t,r,n,a={}){let s=this.materialize({program:t,plan:r,request:n},a);this.pushMaterialized(s),this.cleanups.push(s.cleanup)}adoptMaterialized(t){for(let r of t)this.programs.push(r)}async buildSteps(){return this.rt.prepareProgramSequence(this.programs)}enqueue(t){this.rt.enqueuePreparedProgramSequence(t)}disposeBuild(){let t=[];try{this.clearBuildResources()}catch(r){t.push(r)}try{this.scratchPool.clearPersistent()}catch(r){t.push(r)}xt(t,"Program collector disposal failed")}async runConstantPrograms(){if(this.constantProgramsReady===null){if(this.constantPrograms.length===0)return;let t=[...this.constantPrograms];this.constantProgramsReady=this.rt.runProgramSequence(t,{wait:!0}).then(()=>{this.constantPrograms.splice(0,t.length),this.constantProgramsReady=null})}await this.constantProgramsReady,this.constantPrograms.length>0&&await this.runConstantPrograms()}async flush(t=!0){if(this.programs.length===0){this.clearBuildResources();return}let r=this.timing?performance.now():0;try{await this.rt.runProgramSequence(this.programs,{wait:t})}finally{this.timing&&(this.flushMs+=performance.now()-r),this.clearBuildResources()}}materialize(t,r){let n=r.constantPasses?.(t);if(n===void 0||n.size===0){let u=this.scratchPool.resourcesForPlan(t.plan.scratches);return Ja(this.rt,t,{scratchResources:u})}let a=Ya(t,n),s=t.plan.scratches.filter(u=>!a.has(u.id)),o=t.plan.scratches.filter(u=>a.has(u.id)),i={...this.scratchPool.resourcesForPlan(s),...this.scratchPool.persistentResourcesForPlan(r.constantScratchKey??"",o)};return Ja(this.rt,t,{scratchResources:i,constantPassIds:n})}pushMaterialized(t){for(let r of t.zeroInitClears)this.programs.push(r);for(let r of t.programs)this.programs.push(r);for(let r of t.constantPrograms??[])this.constantPrograms.push(r)}clearBuildResources(){this.programs=[],this.constantPrograms=[],this.constantProgramsReady=null;let t=[],r=0;for(let n of this.cleanups)try{n()}catch(a){this.cleanups[r++]=n,t.push(a)}this.cleanups.length=r;try{this.scratchPool.clear()}catch(n){t.push(n)}xt(t,"Program collector cleanup failed")}};function ws(e,t,r){let n=e.tensors.find(s=>s.kind==="uniform"&&s.name===t);if(!n)throw new Error(`graphUniformPacker: graph "${e.name}" has no uniform "${t}"`);let a=[];if(e.nodes.forEach((s,o)=>{for(let[i,u]of Object.entries(s.inputs))u===n.id&&a.push(...$4(s.op,i,r?.[o]??null))}),a.length===0)throw new Error(`graphUniformPacker: no op reads uniform "${t}" in graph "${e.name}"`);return H4(a,`${e.name}.${t}`,n.shape[0])}var Ug=new Map;function $4(e,t,r=null){let n=`${e}\0${t}\0${r??""}`,a=Ug.get(n);if(a)return a;let s=ns(e),o=[];for(let i of s.manifest.variants)if(!(r!==null&&i.id!==r))for(let u of i.passes)for(let l of u.bindings){if(l.buffer?.type!=="uniform"||!l.struct||l.arg!==t&&l.name!==t)continue;let c=l.struct.fields.map(d=>{if(d.type!=="u32"&&d.type!=="i32"&&d.type!=="f32")throw new Error(`${e}#${i.id}: uniform field ${d.name} has non-scalar type ${d.type} \u2014 step packers cover scalar structs only`);return{name:d.name,kind:d.type}});o.push({source:`${e}#${i.id}/${u.id}`,fields:c})}return Ug.set(n,o),o}function H4(e,t,r){let n=e[0];for(let f of e)f.fields.length>n.fields.length&&(n=f);for(let f of e)f.fields.forEach((m,h)=>{let g=n.fields[h];if(m.name!==g.name||m.kind!==g.kind)throw new Error(`stepUniformPacker(${t}): ${f.source} declares field ${h} "${m.name}: ${m.kind}" but ${n.source} declares "${g.name}: ${g.kind}" \u2014 ops sharing one override buffer must agree on a common field prefix`)});let a=Math.max(4,Math.ceil(n.fields.length/4)*4);if(r!==void 0&&r!==a)throw new Error(`stepUniformPacker(${t}): g.uniform declares ${r*4} bytes but the struct (${n.fields.map(f=>f.name).join(", ")}) packs to ${a*4}`);let s=new ArrayBuffer(a*4),o=new Uint32Array(s),i=new Int32Array(s),u=new Float32Array(s),l=new Map(n.fields.map((f,m)=>[f.name,{...f,index:m}])),c=n.fields.filter(f=>!f.name.startsWith("_")).length,d=()=>n.fields.map(f=>f.name).join(", ");return Object.assign(f=>{let m=0;for(let h of Object.keys(f)){let g=l.get(h);if(!g)throw new Error(`stepUniformPacker(${t}): unknown field "${h}" (struct: ${d()})`);g.kind==="f32"?u[g.index]=f[h]:g.kind==="i32"?i[g.index]=f[h]:o[g.index]=f[h],m+=1}if(m!==c){let h=n.fields.filter(g=>!g.name.startsWith("_")&&!Object.hasOwn(f,g.name)).map(g=>g.name);throw new Error(`stepUniformPacker(${t}): missing field(s) ${h.join(", ")} (struct: ${d()})`)}return o},{fields:new Set(n.fields.map(f=>f.name))})}var $g={passIds:new Set};function Sc(e,t){let r=new Set(e.plan.scratches.map(i=>i.id));if(r.size===0)return $g;let n=new Set(e.plan.scratches.filter(i=>i.zeroInit===!0).map(i=>i.id)),a=new Set,s=new Set,o;for(let i=0;i<e.program.passes.length;i+=1){let u=e.program.passes[i];if(e.plan.passes[i].dispatchWorkgroups.some(p=>p===0))continue;if(u.submitAfter||u.viewAlias!==void 0){o=u.id;break}let c=[],d=!0;for(let p of u.bindingSpecs){let f=Ur(p,r);if(p.buffer.type==="uniform"){t.get(p.arg??p.name)==="uniform"&&(d=!1);continue}if(p.buffer.type==="storage"){if(f===void 0||n.has(f)){d=!1;break}c.push(f);continue}if(f!==void 0){if(!s.has(f)){d=!1;break}continue}if(t.get(p.arg??p.name)!=="weight"){d=!1;break}}if(!d){o=u.id;break}a.add(u.id);for(let p of c)s.add(p)}if(a.size===0)return o===void 0?$g:{passIds:new Set,stoppedBy:o};for(let i of e.program.passes)if(!a.has(i.id))for(let u of i.bindingSpecs){if(u.buffer.type!=="storage")continue;let l=Ur(u,r);if(l!==void 0&&s.has(l))return{passIds:new Set,stoppedBy:i.id}}return{passIds:a,...o===void 0?{}:{stoppedBy:o}}}var Tc=new WeakMap,xc=new Set;function V4(e,t){let r=Object.keys(e);if(r.length!==Object.keys(t).length)return!1;for(let n of r)if(e[n]!==t[n])return!1;return!0}function Z4(e,t){if(!e)return null;let r=Tc.get(e);if(!r)throw new Error(`graph "${t.name}": sibling compile has no reusable plans (support-record compiles cannot seed siblings)`);if(r.graph!==t)throw new Error(`graph "${t.name}": sibling compiled a DIFFERENT graph object ("${r.graph.name}") \u2014 siblings must share one emitted graph`);return r}function en(e,t,r={},n={}){let a=n.pooling!==!1,s=n.constantInputPasses!==!1,o=Z4(n.sibling,e),i=o?.plan??Mg(e),u=a?{plan:i,slotTensors:new Map,shared:n.scratch??null}:null,l=new Map,c=new Set,d=new Map,p=new Map,f=new Map,m=[],h=[];try{for(let _ of e.tensors){if(_.view){let R=l.get(_.view.parent);if(!R){if(c.has(_.view.parent)){c.add(_.id);continue}throw new Error(`graph view "${_.name}" references unrealized parent id ${_.view.parent}`)}let O=o&&o.realized.get(_.view.parent)===R?o.realized.get(_.id):eS(R,_);l.set(_.id,O),_.kind!=="intermediate"&&d.set(_.name,O);continue}if(_.kind==="uniform"){let R=o?o.uniformById.get(_.id):t.createUniformU32(new Uint32Array(ke(_.shape)),`graph-uniform-${_.name}`);o||h.push(R),p.set(_.id,R),f.set(_.name,R);continue}let x=o&&_.kind!=="weight"&&_.kind!=="state"?o.realized.get(_.id)??null:J4(_,e.name,t,r,m,u);if(x===null){c.add(_.id);continue}l.set(_.id,x),_.kind!=="intermediate"&&d.set(_.name,x)}}catch(_){li(_,null,m,h)}let g;try{g=new ui(t,{cachePrepared:!0})}catch(_){li(_,null,m,h)}let w=new Map(e.tensors.map(_=>[_.id,_])),b=null;if(n.opScratchCapacityPooling&&!t.recordSupportOp)try{b=o?.nodePlans??e.nodes.map(x=>{let R=Jr(x.op,{conformance:x.conformance}),O={};for(let[Q,V]of Object.entries(x.inputs))O[Q]=l.get(V)??p.get(V);for(let[Q,V]of Object.entries(x.outputs))O[Q]=l.get(V);let N=g.prepare(R,{resources:O,...x.args?{args:{...x.args}}:{},...x.attrs?{attrs:{...x.attrs}}:{},...x.tunables?{tunables:{...x.tunables}}:{}});return{program:N.program,plan:N.plan}});let _=b.map((x,R)=>{if(!s)return x.plan.scratches;let O=e.nodes[R],N=new Map;for(let[$,M]of Object.entries(O.inputs)){let j=w.get(M)?.kind;j!==void 0&&N.set($,j)}let Q=Sc({...x,request:{resources:{}}},N).passIds;if(Q.size===0)return x.plan.scratches;let V=Ya({...x,request:{resources:{}}},Q);return x.plan.scratches.filter($=>!V.has($.id))});g.reserveScratchCapacities(_)}catch(_){li(_,g,m,h)}let v=[],y=[],S=[],C=[],k=[];try{e.nodes.forEach((_,x)=>{let R=Jr(_.op,{conformance:_.conformance}),O={};for(let[T,E]of Object.entries(_.inputs))O[T]=l.get(E)??p.get(E);for(let[T,E]of Object.entries(_.outputs))O[T]=l.get(E);let N={resources:O,..._.args?{args:{..._.args}}:{},..._.attrs?{attrs:{..._.attrs}}:{},..._.tunables?{tunables:{..._.tunables}}:{}},Q=g.programCount,V=new Map;for(let[T,E]of Object.entries(_.inputs)){let X=w.get(E)?.kind;X!==void 0&&V.set(T,X)}let $=xc,M={constantPasses:T=>($=s?Sc(T,V).passIds:xc,$),constantScratchKey:String(x)};try{if(o){let T=o.nodePlans[x];$=M.constantPasses({...T,request:N});let E=o.nodeConstantPasses[x]??xc;$.size===0&&E.size===0&&V4(o.nodeResources[x],O)?g.adoptMaterialized(o.nodePrograms[x]):g.addPrepared(T.program,T.plan,N,{constantPasses:()=>$,constantScratchKey:String(x)}),y.push(T),v.push(T.program.variant)}else{let T=b?.[x],E=T?{...T,request:N}:g.add(R,N,M);T&&g.addPrepared(T.program,T.plan,N,M),y.push(E?{program:E.program,plan:E.plan}:null),v.push(E?.program.variant??null)}}catch(T){let E=Object.entries(O).map(([X,z])=>`${X}=[${z?.shape??"?"}]`).join(" ");throw new Error(`graph node ${_.op} (args ${JSON.stringify(_.args??{})}; ${E}): ${T.message}`,{cause:T})}C.push(O),S.push($);let j=g.programSlice(Q,g.programCount),A=Y4(e.name,_.op,_,w,x);for(let T of j){if(!("source"in T))continue;let E=T;E.profile={...E.profile,...A}}k.push(j)}),X4(e,w,y)}catch(_){li(_,g,m,h)}let B=a?i.diagnostics:{...i.diagnostics,slotCount:i.diagnostics.intermediateCount,pooledBytes:i.diagnostics.naiveBytes},P={collector:g,tensors:l,allocation:B,nodeVariants:v,nodeConstantPasses:S,tensor(_){let x=d.get(_);if(!x)throw new Error(`graph "${e.name}" has no bound tensor named "${_}"`);return x},uniformBuffer(_){let x=f.get(_);if(!x)throw new Error(`graph "${e.name}" has no uniform named "${_}"`);return x},async runEager(){await g.runConstantPrograms(),await g.flush(!0)},buildSteps(_){if(!_)return g.runConstantPrograms().then(()=>g.buildSteps());let x=_.startNode??0,R=_.endNode??e.nodes.length;if(!Number.isInteger(x)||!Number.isInteger(R)||x<0||R<x||R>e.nodes.length)throw new Error(`graph "${e.name}": invalid node range [${x}, ${R}) for ${e.nodes.length} nodes`);let O=[];for(let N=x;N<R;++N)O.push(...k[N]);return g.runConstantPrograms().then(()=>O.length>0?t.prepareProgramSequence(O):Promise.resolve([]))},dispose(){Tc.delete(P),Vg(g,m,h)}};return y.every(_=>_!==null)&&Tc.set(P,{graph:e,plan:i,nodePlans:y,nodeConstantPasses:S,nodeResources:C,nodePrograms:k,realized:l,uniformById:p,uniformByName:f}),P}function X4(e,t,r){let n=[],a=[];if(e.nodes.forEach((i,u)=>{let l=r[u];l&&(n.push(i),a.push(l.program.variant))}),n.length===0)return;let s=n.length===e.nodes.length?e:{...e,nodes:n},o=new Set;for(let i of n)for(let u of Object.values(i.inputs))t.get(u)?.kind==="uniform"&&o.add(u);for(let i of e.tensors)i.kind==="uniform"&&o.has(i.id)&&ws(s,i.name,a);e.nodes.forEach((i,u)=>{let l=Object.entries(i.inputs).map(([m,h])=>({port:m,tensor:t.get(h)})).filter(m=>m.tensor?.kind==="uniform");if(l.length===0)return;let c=r[u];if(!c)return;let{program:d,plan:p}=c,f=new Map;d.passes.forEach((m,h)=>{if(m.repeatBaseId===void 0)return;let g=p.passes[h];g.dispatchWorkgroups.some(w=>w===0)||m.bindingSpecs.forEach((w,b)=>{if(w.buffer.type!=="uniform"||!w.struct)return;let v=l.find(({port:C})=>w.arg===C||w.name===C);if(!v)return;let y=m.bindings[b];if(y?.kind!=="uniform")return;let S=Xs(y.struct,g.uniforms[w.name]);w.struct.fields.forEach((C,k)=>{if(C.internal)return;let B=`${v.tensor.id}\0${m.repeatBaseId}\0${w.name}\0${C.name}`,P=S[k],_=f.get(B);if(!_)f.set(B,{word:P,passId:m.id});else if(_.word!==P)throw new Error(`graph "${e.name}" node ${u} (${i.op}#${d.variant}) repeats pass "${m.repeatBaseId}" with different generated values for graph uniform "${v.tensor.name}" field ${C.name}:${C.type} (${_.passId} != ${m.id}); one caller-owned buffer cannot carry repeat-varying state. Move that field to a separate generated uniform binding.`)})})})})}function li(e,t,r,n){try{Vg(t,r,n)}catch(a){throw new AggregateError([e,a],"Model graph compilation and cleanup both failed")}throw e}function Vg(e,t,r){let n=[];try{e?.disposeBuild()}catch(a){n.push(a)}Hg(t,a=>a.destroy(),n),Hg(r,a=>a.destroy?.(),n),xt(n,"Model graph resource cleanup failed")}function Hg(e,t,r){for(let n=e.length-1;n>=0;--n)try{t(e[n]),e.splice(n,1)}catch(a){r.push(a)}}function Y4(e,t,r,n,a){let s=null;for(let l of Object.values(r.inputs)){let c=n.get(l);if(c?.kind==="weight"){s=c.name.replace(/\.[^.]+$/,"");break}}if(!s){let l=Object.values(r.outputs)[0];s=l!==void 0?n.get(l)?.name??t:t}let o=s.match(/(?:^|\.)(\d+)(?:\.|$)/),i=s.replace(/#\d+$/,""),u={phase:e,node:s,part:o?i.replace(/(^|\.)\d+(\.|$)/,(l,c,d)=>c&&d?".":""):i,graphNode:a};return o&&(u.layer=Number(o[1])),u}function J4(e,t,r,n,a,s){if(s){let i=s.plan.slots.get(e.id);if(i!==void 0){let u=s.slotTensors.get(i);if(!u){let l=s.plan.slotByteLengths.get(i),c=s.plan.slotAllocations.get(i),d=()=>r.empty(c.dtype,c.shape,`graph-slot-${i}`);s.shared?u=s.shared.acquire(i,l,t,d):(u=d(),a.push(u)),s.slotTensors.set(i,u)}return tS(u,e,s.plan.slotByteLengths.get(i))}if(e.kind==="intermediate")return null}switch(e.kind){case"weight":case"state":{let u=(e.kind==="weight"?n.weights:n.states)?.[e.name];if(!u)throw new Error(`graph ${e.kind} "${e.name}" has no binding`);return u}case"input":{let i=n.inputs?.[e.name];if(i===void 0)throw new Error(`graph input "${e.name}" has no binding`);if(wt(i))return i;let u=r.tensorFromTypedArray(e.dtype,e.shape,i);return a.push(u),u}case"intermediate":case"stepInput":case"output":break}let o=r.empty(e.dtype,e.shape,`graph-${e.name}`);return a.push(o),o}function eS(e,t){let r=ke(t.shape),n=t.view.elementOffset;if(!Number.isSafeInteger(n)||n<0)throw new Error(`graph view "${t.name}" needs a non-negative safe integer elementOffset`);if(e.layout==="packed"){if(t.dtype!=="uint8")throw new Error(`graph view "${t.name}": packed storage must retain uint8 dtype`);if(n%4!==0)throw new Error(`graph view "${t.name}": packed offsets must be word-aligned`);if(n+r>e.size)throw new Error(`graph view "${t.name}" exceeds its packed parent's logical range`)}let a=e.layout==="packed"?1:Ae(t.dtype),s=(e.byteOffset??0)+n*a,o=Ye(t.dtype,r,e.layout);if(s+o>(e.byteOffset??0)+e.byteLength)throw new Error(`graph view "${t.name}" exceeds its parent's range (${s}+${o} > ${e.byteLength})`);return{runtime:e.runtime,dtype:t.dtype,shape:t.shape,size:r,byteLength:o,byteOffset:s,...e.layout?{layout:e.layout}:{},buffer:e.buffer,destroy:vn}}function tS(e,t,r){let n=ke(t.shape),a=Ye(t.dtype,n);if(a>r)throw new Error(`graph tensor "${t.name}" (${a} bytes) does not fit slot of ${r} bytes`);return{runtime:e.runtime,dtype:t.dtype,shape:t.shape,size:n,byteLength:a,byteOffset:e.byteOffset??0,buffer:e.buffer,destroy:vn}}var tn=class{#e;#t;#n;#r;#a=new Map;constructor(t,r={}){this.#e=t,this.#t=Math.max(1,r.maxSessions??Number.POSITIVE_INFINITY),this.#n=r.constantInputPasses,this.#r=r.opScratchCapacityPooling===!0}async acquire(t,r){let n=this.#a.get(t);if(n)return this.#a.delete(t),this.#a.set(t,n),n;let a=r(),s=en(a.graph,this.#e,{weights:a.weights,...a.states?{states:a.states}:{}},{constantInputPasses:this.#n,opScratchCapacityPooling:this.#r}),o=await s.buildSteps(),i={compiled:s,steps:o,meta:a.meta};for(;this.#a.size>=this.#t;){let u=this.#a.keys().next().value;this.#a.get(u)?.compiled.dispose(),this.#a.delete(u)}return this.#a.set(t,i),i}write(t,r,n){this.#e.host.writeBuffer(t.compiled.tensor(r).buffer,0,n)}enqueue(t){this.#e.enqueuePreparedProgramSequence(t.steps)}async read(t,r){return Float32Array.from(_n(await this.#e.readTensor(t.compiled.tensor(r))))}dispose(){for(let t of this.#a.values())t.compiled.dispose();this.#a.clear()}};var rS=64*1024,bs=class{runtime;tensor;constructor({runtime:t,tensor:r}){this.runtime=t,this.tensor=r}writeTensor({sourceDtype:t,targetDtype:r,sourceBytes:n,targetElementOffset:a}){M2(t,r);let s=W2(t),o=F2(r);if(n.byteLength%s!==0)throw new Error(`source byte length ${n.byteLength} not divisible by ${s}`);let i=n.byteLength/s,u=a*o;if(t==="F32"&&r==="float32")return this.runtime.writeWeightsRange(this.tensor,u,n),a+i;if(t==="F16"&&r==="float16")return this.runtime.writeWeightsRange(this.tensor,u,n),a+i;if(r==="bfloat16"){if(a%2!==0||i%2!==0)throw new Error(`bfloat16 storage needs an even element offset and count, got ${a} + ${i}`);return this.runtime.writeWeightsRange(this.tensor,u,n),a+i}let l=Math.max(1,Math.floor(rS/o)),c=j2(t,r);for(let d=0;d<i;d+=l){let p=Math.min(l,i-d),f=n.subarray(d*s,(d+p)*s),m=c(f),h=u+d*o;this.runtime.writeWeightsRange(this.tensor,h,Q2(m))}return a+i}};function nh(e){e&&e.depth++===0&&(e.since=performance.now())}function ah(e){e&&--e.depth===0&&(e.hostMs+=performance.now()-e.since)}var ci=class{specs;handlers;pending;#e=new Map;#t=new Map;#n=!1;constructor(t=[]){this.specs=t.map(ih),this.handlers=new Map,this.pending=new Set}prepare({reader:t,context:r={}}){for(let n of this.#e.values())if(n.state==="running"||n.state==="committing"||n.state==="rolling-back")throw new Error(`Cannot prepare WeightPlan while derived task ${n.spec.name} is ${n.state}`);this.handlers.clear(),this.pending.clear(),this.#e.clear(),this.#t.clear(),this.#n=!1;for(let n of this.specs)n.type==="tensor"?this.#r(n,t,r):n.type==="group"&&this.#a(n,t,r);for(let n of this.specs)n.type==="derived"&&this.#s(n,t,r);return this}names(){return[...this.handlers.keys()]}derivedTaskNames(){return[...this.#e.keys()]}directDestinations(){let t=new Map;for(let[r,n]of this.handlers)n.openDirect&&t.set(r,n.openDirect);return t}async onChunk({bytes:t,tensors:r,directDestination:n},a={}){if(n){let o=r.length===1?r[0]:void 0;if(!o||o.offset!==0||o.length!==t.byteLength||t!==n.bytes)throw await Cc(n),new Error("A direct weight destination requires one exact, unsliced tensor chunk");let i=this.handlers.get(o.name);if(!i?.receiveDirect)throw await Cc(n),new Error(`WeightPlan has no direct handler for tensor: ${o.name}`);await nS(i,n,a);return}let s=[];for(let o of r){let i=this.handlers.get(o.name);if(!i)continue;let u=t.subarray(o.offset,o.offset+o.length);s.push(Yg(i,u,a))}if(s.length){let o=!1,i;if(await Promise.all(s.map(u=>u.catch(l=>{o||(o=!0,i=l)}))),o)throw i}}async readSequential(t,r={}){for(let n of this.names()){let a=await t.read(n),s=this.handlers.get(n);s&&await Yg(s,a,r)}}assertComplete(){if(this.pending.size>0){let t=[...this.pending].slice(0,5).join(", ");throw new Error(`WeightPlan incomplete - ${this.pending.size} tensor(s) never arrived (first: ${t})`)}}async finishDerivedTasks(){let t=[...this.#e.values()],r=!1,n;if(await Promise.all(t.map(async a=>{try{if(a.state==="metadata")return;if(a.state==="waiting")throw new Error(`Derived weight task ${a.spec.name} is missing completed dependencies: ${[...a.remaining].join(", ")}`);if(await a.promise,a.state==="failed")throw new Error(`Derived weight task ${a.spec.name} failed`);if(a.state==="aborted")throw new Error(`Derived weight task ${a.spec.name} was aborted`)}catch(s){r||(r=!0,n=s)}})),r)throw n}async abortDerivedTasks(){this.#n=!0;let t=[...this.#e.values()],r=!1,n,a=s=>{r||(r=!0,n=s)};if(await Promise.all(t.map(async s=>{if(!(s.state==="metadata"||s.state==="published"||s.state==="retained-fallback"||s.state==="aborted")){if(s.state==="waiting"){s.state="aborted";return}if(s.state==="running"&&!s.abortStarted){s.abortStarted=!0;try{await s.spec.abort?.(s.context)}catch(o){a(o)}}try{await s.promise}catch(o){a(o)}}})),r)throw n}#r(t,r,n){let a=eh(r,t.name);if(!a){rh(t,n);return}th(t,a);let s=this.#o(t.name,async(o,i)=>{await iS(t,o,{...n,...i,description:a}),this.pending.delete(t.name),this.#l([t.name],i)},t.progressLabel??t.name);t.directDestination&&!di(n)&&(s.openDirect=()=>t.directDestination?.({...n,description:a})??null,s.receiveDirect=async(o,i)=>{await uS(t,o,{...n,...i,description:a}),this.pending.delete(t.name),this.#l([t.name],i)})}#a(t,r,n){let a={},s=t.names.filter(u=>{let l=eh(r,u);return l&&(a[u]=l),!l});if(s.length>0){rh(t,n,s);return}for(let u of t.names)th(t,a[u],u);let o={},i=t.names.length;for(let u of t.names)this.#o(u,async(l,c)=>{if(this.pending.delete(u),o[u]=l,--i===0)try{await lS(t,o,{...n,...c,descriptions:a}),this.#l(t.names,c)}finally{for(let d of t.names)delete o[d]}},t.progressLabel??u)}#s(t,r,n){if(this.#e.has(t.name))throw new Error(`Duplicate derived weight task: ${t.name}`);if(new Set(t.dependencies).size!==t.dependencies.length)throw new Error(`Derived weight task ${t.name} has duplicate dependencies`);let s={},o=t.dependencies.filter(u=>!this.handlers.has(u)||!r.has(u)?!0:(s[u]=r.describe(u),!1));if(o.length>0)throw new Error(`Derived weight task ${t.name} has missing dependencies: ${o.join(", ")}`);let i={spec:t,context:{...n,descriptions:s},descriptions:s,remaining:new Set(t.dependencies),state:di(n)?"metadata":"waiting",promise:null,transaction:null,abortStarted:!1};if(this.#e.set(t.name,i),i.state!=="metadata")for(let u of t.dependencies){let l=this.#t.get(u);l?l.push(i):this.#t.set(u,[i])}}#l(t,r){let n=new Set;for(let a of t)for(let s of this.#t.get(a)??[])s.state==="waiting"&&(s.remaining.delete(a),n.add(s));for(let a of n)a.remaining.size===0&&this.#u(a,r)}#u(t,r){if(t.state!=="waiting")return;if(this.#n){t.state="aborted";return}t.state="running";let n={...t.context,...r,descriptions:t.descriptions},a;try{a=t.spec.start(n)}catch(o){a=Promise.reject(o)}let s=(async()=>{try{let o=await a;if(o===null){t.state="retained-fallback";return}if(oS(t.spec.name,o),t.transaction=o,this.#n){t.state="rolling-back",await uh(t),t.state="aborted";return}t.state="committing";try{await o.commit()}catch(i){throw t.state="rolling-back",await Jg(t,i),t.state="failed",i}t.transaction=null,t.state="published"}catch(o){throw t.transaction&&t.state!=="published"&&t.state!=="aborted"&&(t.state="rolling-back",await Jg(t,o)),t.state="failed",o}})();t.promise=s,s.catch(()=>{})}#o(t,r,n=t){if(this.handlers.has(t))throw new Error(`Duplicate WeightPlan handler for tensor: ${t}`);let a={name:t,progressLabel:n,received:!1,receive:r};return this.handlers.set(t,a),this.pending.add(t),a}};function sh(e){return ih({type:"tensor",required:!0,...e})}async function oh(e,{source:t=null,reader:r,context:n={},concurrency:a,chunkMaxBytes:s,onProgress:o=null,signal:i}={}){let u=e instanceof ci?e:new ci(e);if(!r)throw new Error("WeightPlan execution requires a reader");let l=[],c={hostMs:0,depth:0,since:0},d=performance.now(),p=null;try{u.prepare({reader:r,context:n});let f={...n,timing:c,onTensorProgress:aS(u,o)};t?await t.streamAll(m=>u.onChunk(m,f),{concurrency:a,chunkMaxBytes:s,names:u.names(),onProgress:m=>{p=m.loaded,o?.({...m,phase:"bytes"})},signal:i,directDestinations:t.supportsDirectDestinations===!0?u.directDestinations():void 0}):await u.readSequential(r,f),u.assertComplete(),await u.finishDerivedTasks(),o?.({phase:"loaded",bytes:p,wallMs:performance.now()-d,hostMs:c.hostMs})}catch(f){l.push(f);try{await u.abortDerivedTasks()}catch{}}try{await sS(t,r)}catch(f){l.push(f)}if(l.length===1)throw l[0];if(l.length>1)throw new AggregateError(l,"WeightPlan execution and I/O cleanup both failed");return u}async function Yg(e,t,r){if(e.received)throw new Error(`WeightPlan received tensor ${e.name} more than once`);e.received=!0,nh(r.timing);try{await e.receive(t,r)}finally{ah(r.timing)}r.onTensorProgress?.({name:e.name,label:e.progressLabel})}async function nS(e,t,r){if(e.received)throw await Cc(t),new Error(`WeightPlan received tensor ${e.name} more than once`);e.received=!0,nh(r.timing);try{await e.receiveDirect?.(t,r)}finally{ah(r.timing)}r.onTensorProgress?.({name:e.name,label:e.progressLabel})}async function Cc(e){try{await e.abort()}catch{}}function aS(e,t){if(!t)return null;let r=0,n=e.names().length;return({name:a,label:s})=>{t({phase:"tensor",processed:++r,total:n,name:a,label:s})}}async function sS(e,t){let r=new Set,n=e?[t,e]:[t],a=[];for(let s of n){let o=s.closeTarget??s;if(!r.has(o)){r.add(o);try{await s.close()}catch(i){a.push(i)}}}if(a.length===1)throw a[0];if(a.length>1)throw new AggregateError(a,"Multiple WeightPlan I/O handles failed to close")}function ih(e){if(!e||typeof e!="object")throw new Error("WeightPlan spec must be an object");if(e.type==="tensor"){if(typeof e.name!="string"||e.name.length===0)throw new Error("WeightPlan tensor spec requires a name");return{required:!0,...e}}if(e.type==="group"){if(!Array.isArray(e.names)||e.names.length===0)throw new Error("WeightPlan group spec requires a non-empty names array");for(let t of e.names)if(typeof t!="string"||t.length===0)throw new Error("WeightPlan group tensor names must be non-empty strings");return{required:!0,...e}}if(e.type==="derived"){if(typeof e.name!="string"||e.name.length===0)throw new Error("Derived weight task requires a name");if(!Array.isArray(e.dependencies)||e.dependencies.length===0)throw new Error(`Derived weight task ${e.name} requires non-empty dependencies`);for(let t of e.dependencies)if(typeof t!="string"||t.length===0)throw new Error(`Derived weight task ${e.name} dependencies must be non-empty strings`);if(typeof e.start!="function")throw new Error(`Derived weight task ${e.name} requires start()`);return{...e,type:"derived"}}throw new Error(`Unknown WeightPlan spec type: ${e.type}`)}function oS(e,t){if(!t||typeof t!="object"||typeof t.commit!="function"||typeof t.rollback!="function")throw new Error(`Derived weight task ${e} returned an invalid transaction`)}async function uh(e){let t=e.transaction;e.transaction=null,t&&await t.rollback()}async function Jg(e,t){try{await uh(e)}catch{}}function di(e){return e.rt?.metadataOnly===!0}async function iS(e,t,r){let n=e.transform&&!di(r)?await e.transform(t,r):t,a=e.upload?await e.upload(n,r):n;e.run&&await e.run(a,r),await e.assign?.(a,r),await e.cleanup?.(a,r)}async function uS(e,t,r){let n=!1,a;try{a=await t.commit(),n=!0,e.run&&await e.run(a,r),await e.assign?.(a,r),await e.cleanup?.(a,r)}catch(s){if(n)try{await t.rollback(a)}catch{}else try{await t.abort()}catch{}throw s}}async function lS(e,t,r){let n=e.transform&&!di(r)?await e.transform(t,r):t,a=e.upload?await e.upload(n,r):n;e.run&&await e.run(a,r),await e.assign?.(a,r),await e.cleanup?.(a,r)}function eh(e,t){return e.has(t)?e.describe(t):null}function th(e,t,r=e.type==="tensor"?e.name:"group"){if(e.sourceDtype&&t.dtype!==e.sourceDtype)throw new Error(`${r} has dtype ${t.dtype}; expected ${e.sourceDtype}`);if(e.shape){let n=cS(e.shape,t.elementCount,t);if(e.reshape){let a=n.reduce((s,o)=>s*o,1);if(a!==t.elementCount)throw new Error(`${r} has ${t.elementCount} elements; expected ${a} for reshape [${n}]`)}else if(n.length!==t.shape.length||n.some((a,s)=>a!==t.shape[s]))throw new Error(`${r} has shape [${t.shape}]; expected [${n}]`)}}function rh(e,t,r=e.type==="tensor"?[e.name]:[...e.names]){if(e.required===!1){e.missing?.(r,t);return}throw new Error(`Missing required ${lr(r.length,"tensor")}: ${r.join(", ")}`)}function cS(e,t,r){return typeof e=="function"?e({valueLength:t,description:r}):Array.isArray(e)?e:r.shape?r.shape:[t]}function wr(e){return Math.max(0,Math.floor(Math.min(e.maxBufferSize,e.maxStorageBufferBindingSize)))}function qn({byteLimit:e,maxBlocks:t,bytesForBlocks:r}){if(!Number.isFinite(e)||e<0)throw new Error(`pack shard byte limit must be a finite non-negative number, got ${e}`);if(!Number.isSafeInteger(t)||t<0)throw new Error(`pack shard block count must be a non-negative safe integer, got ${t}`);if(r(t)<=e)return t;let n=0,a=t;for(;a-n>1;){let s=n+Math.floor((a-n)/2);r(s)<=e?n=s:a=s}return n}function On(e,t,r){if(!Number.isSafeInteger(t)||t<0)throw new Error(`${r} shard capacity must be a non-negative safe integer, got ${t}`);let n=[],a=new Map,s=new Set;for(let c of e){if(s.has(c.id))throw new Error(`Duplicate ${r} packed weight id: ${c.id}`);if(s.add(c.id),!Number.isSafeInteger(c.blockCount)||c.blockCount<=0)throw new Error(`${r} packed weight ${c.id} has invalid block count ${c.blockCount}`);let d=c.unitKey===void 0?`id:${c.id}`:`unit:${c.unitKey}`,p=a.get(d);if(p||(p={entries:[],blockCount:0},a.set(d,p),n.push(p)),p.entries.push(c),p.blockCount+=c.blockCount,!Number.isSafeInteger(p.blockCount))throw new Error(`${r} allocation unit containing ${c.id} exceeds safe integer range`)}let o=[],i=new Map,u=0,l=()=>{u!==0&&(o.push(u),u=0)};for(let c=0;c<n.length;++c){let d=n[c];u>0&&u+d.blockCount>t&&l();let p=o.length;for(let f of d.entries)i.set(f.id,{shardIndex:p,unitIndex:c,blockOffset:u,blockCount:f.blockCount}),u+=f.blockCount;u>t&&l()}return l(),{shardBlockCounts:o,placements:i}}var ch="top";function lh(e){let t=/^layers\.(\d+)\./.exec(e);return t?`layers.${t[1]}`:ch}function dS(e){return e==="top"?ch:`layers.${e}`}function pi({entries:e,elementBytes:t,limits:r}){if(!Number.isSafeInteger(t)||t<=0)throw new Error(`dense weight element size must be a positive integer, got ${t}`);let n=[],a=[],s=0;for(let p of e){if(!Number.isSafeInteger(p.length)||p.length<0)throw new Error(`dense weight ${p.id} has invalid element count ${p.length}`);if(p.length===0){a.push(p.id);continue}n.push({id:p.id,blockCount:p.length,unitKey:lh(p.id)}),s+=p.length}let o=wr(r),i=o>0?qn({byteLimit:o,maxBlocks:s,bytesForBlocks:p=>p*t}):s,u=On(n,i,"dense weights"),l=u.shardBlockCounts.length>0?[...u.shardBlockCounts]:[0],c=new Map,d=new Map;for(let p of n){let f=u.placements.get(p.id);if(!f)throw new Error(`dense weight ${p.id} is missing from its own shard plan`);c.set(p.id,{shardIndex:f.shardIndex,elementOffset:f.blockOffset}),d.set(p.unitKey,f.shardIndex)}for(let p of a){let f=lh(p),m=d.get(f)??0;d.set(f,m),c.set(p,{shardIndex:m,elementOffset:0})}return{shardElementCounts:l,placements:c,shardIndexByUnit:d}}function dh(e,t){let r=e.denseShards;if(!r||r.length<=1)return 0;let n=dS(t),a=e.denseShardIndex?.[n];if(a===void 0)throw new Error(`no dense weight shard holds ${n}; the loader and the emit disagree on the dense layout`);if(a<0||a>=r.length)throw new Error(`dense weight shard ${a} for ${n} is outside the ${r.length}-shard plan`);return a}function va(e,t){let r=e.find(n=>n.id==="lm_head");return r?!t(r.name):!1}function fh({runtime:e,entries:t,stateDict:r,tieWordEmbeddings:n=!1,modelName:a="model"}){let s=Jt(),o=[],i=0;for(let l of t){let c=r[l.name];if(c===void 0&&l.id==="lm_head"&&n){if(s.top.embed_tokens===void 0)throw new Error(`${a}: tied lm_head requires embed_tokens to come first in entries`);_t(s,l.id,s.top.embed_tokens);continue}if(c===void 0)throw new Error(`Missing ${a} state dict weight: ${l.name}`);let d=fS(c,l.name);if(l.transform&&(d=l.transform(d,l.shape)),d.length!==l.length)throw new Error(`Weight ${l.name} length ${d.length} does not match expected ${l.length}`);_t(s,l.id,i),o.push(d),i+=d.length}let u=hS(o,i,Float32Array);return{weights:e.tensorFromTypedArray("float32",[u.length],u),offsets:s}}function Pc(e=8*1024*1024){let t=0;return r=>(t+=r,t<e?Promise.resolve():(t=0,Jo()))}async function Ic({runtime:e,config:t,entries:r,reader:n,source:a=null,canUseFloat16:s=Lr,canUseQ8LmHead:o=Ec,canUseQ4LmHead:i=vs,modelName:u="model",onProgress:l=null,signal:c,skipDenseIds:d,keepBf16:p=!1}){let f=va(r,z=>n.has(z)),m=Bc({entries:r.filter(z=>!d?.has(z.id)&&!(z.id==="lm_head"&&f)),describe:z=>n.describe(z).dtype,keepBf16:p,float16:e.device.features.has("shader-f16")&&s(t)}),h=m==="float16",g=m==="float32"?4:2,w=Jt(),b=[],v=[],y=new Set,S=null;for(let z of r){if(d?.has(z.id))continue;if(z.id==="lm_head"&&f){if(!y.has("embed_tokens"))throw new Error(`${u}: tied lm_head requires embed_tokens to come first in entries`);S=z,b.push({entry:z,tied:!0});continue}let G=gi(z).map(F=>({name:F,meta:n.describe(F)})),L=G[0].meta.dtype;Vo(L);let K=G.find(F=>F.meta.dtype!==L);if(K)throw new Error(`${u} tensor ${K.name} is ${K.meta.dtype} but ${G[0].name} is ${L}; the parts of ${z.name} must share one dtype`);if(z.parts&&z.transform)throw new Error(`${u} tensor ${z.name}: a stacked entry takes no transform`);let D=G.reduce((F,H)=>F+H.meta.elementCount,0);if(D!==z.length)throw new Error(`${u} tensor ${z.name} length ${D} does not match expected ${z.length}`);v.push({id:z.id,length:z.length}),y.add(z.id),b.push({entry:z,tied:!1,sourceDtype:L,sourceShape:z.parts?[G.length,...G[0].meta.shape]:G[0].meta.shape,sources:G.map(F=>({name:F.name,elementCount:F.meta.elementCount})),elementOffset:0,shardIndex:0})}let C=pi({entries:v,elementBytes:g,limits:e.device.limits});for(let z of b){if(z.tied)continue;let G=C.placements.get(z.entry.id);if(!G)throw new Error(`${u}: dense weight ${z.entry.id} has no place in the shard plan`);z.elementOffset=G.elementOffset,z.shardIndex=G.shardIndex,_t(w,z.entry.id,G.elementOffset)}S&&_t(w,S.id,w.top.embed_tokens);let k=C.shardElementCounts.length>1,B=C.shardElementCounts.map((z,G)=>e.allocateWeightsBuffer({byteLength:z*g,...m==="bfloat16"?{dtype:"uint32",shape:[z/2]}:{dtype:m,shape:[z]},label:k?`${u}-weights.${G}`:`${u}-weights`})),P=B[0],_=B.map(z=>new bs({runtime:e,tensor:z})),x=h&&i(t),R=h&&!x&&o(t),O=x||R,N=null,Q=null,V=b.filter(z=>!z.tied),$=0,M=Pc(),j=[],A=V.reduce((z,G)=>z+G.sources.length,0);for(let z of V){let G=0;for(let L of z.sources){let K=z.elementOffset+G;G+=L.elementCount,j.push(sh({name:L.name,sourceDtype:z.sourceDtype,run:async D=>{let F=z.sourceDtype,H=D;if(z.entry.transform){let Y=is(z.sourceDtype,D),fe=z.entry.transform(Y,z.sourceShape);F="F32",H=new Uint8Array(fe.buffer,fe.byteOffset,fe.byteLength)}if(_[z.shardIndex].writeTensor({sourceDtype:F,targetDtype:m,sourceBytes:H,targetElementOffset:K}),O&&(z.entry.id==="embed_tokens"||z.entry.id==="lm_head")){let Y=is(z.sourceDtype,D);z.entry.id==="embed_tokens"?N=Y:Q=Y}$+=1,l?.({processed:$,total:A,name:L.name}),await M(D.byteLength)}}))}}try{await oh(j,{reader:n,source:a,signal:c})}catch(z){throw await cr(e,B),z}for(let z of b)z.tied&&O&&z.entry.id==="lm_head"&&(Q=N);O&&Q&&await M(1/0);let T=x&&Q?mS(Q,t.vocab_size,t.hidden_size):null,E=R&&Q?gS(Q,t.vocab_size,t.hidden_size):null,X=e.device.features.has("shader-f16");return{weights:P,offsets:w,denseShards:B,denseShardIndex:Object.fromEntries(C.shardIndexByUnit),lmHeadQ4:T?e.tensorFromTypedArray("uint32",[T.bits.length],T.bits):null,lmHeadQ4Scales:T?X?e.tensorFromTypedArray("float16",[T.scales.length],Bn(T.scales)):e.tensorFromTypedArray("float32",[T.scales.length],T.scales):null,lmHeadQ8:E?e.tensorFromTypedArray("uint32",[E.packed.length],E.packed):null,lmHeadQ8Scales:E?e.tensorFromTypedArray("float32",[E.scales.length],E.scales):null,packs:{},cacheDtype:m==="bfloat16"?"float32":m}}function Bc({entries:e,describe:t,keepBf16:r,float16:n}){return r&&e.length>0&&e.every(a=>!a.transform&&a.length%4===0&&gi(a).every(s=>t(s)==="BF16"))?"bfloat16":n?"float16":"float32"}function mi(e,t){let r=Bc({entries:e,describe:t,keepBf16:!0,float16:!1})==="bfloat16",n=e.length>0&&e.every(a=>gi(a).every(s=>t(s)==="F16"));return{canUseFloat16:()=>n,keepBf16:r}}function gi(e){return e.parts??[e.name]}function fS(e,t){if(e instanceof Float32Array)return e;if(wS(e))return Float32Array.from(e);if(Array.isArray(e))return Float32Array.from(e.flat(1/0));throw new Error(`Weight ${t} must be a Float32Array or array`)}function Jt(){return{top:{},layers:[]}}function _t(e,t,r){let n=/^layers\.(\d+)\.(.+)$/.exec(t);if(!n){e.top[t]=r;return}let a=Number(n[1]);(e.layers[a]??={})[n[2]]=r}function Gr(e,t){let r=/^layers\.(\d+)\.(.+)$/.exec(t);return r?e.layers[Number(r[1])]?.[r[2]]:e.top[t]}function Lr(e){return!e.prismHadamard&&e.hidden_size%4===0&&e.intermediate_size%4===0}function Ec(e){return!e.prismHadamard&&e.hidden_size%4===0}function vs(e){return!e.prismHadamard&&e.hidden_size%32===0}function mS(e,t,r){let n=r/32,a=t*n,s=new Uint32Array(a*4),o=new Float32Array(a);for(let i=0;i<t;++i)for(let u=0;u<n;++u){let l=i*r+u*32,c=0,d=0;for(let h=0;h<32;++h){let g=e[l+h];Math.abs(g)>c&&(c=Math.abs(g),d=g)}let p=d/-8,f=p!==0?1/p:0,m=i*n+u;o[m]=p;for(let h=0;h<4;++h){let g=0;for(let w=0;w<4;++w){let b=h*4+w,v=ph(Math.round(e[l+b]*f)+8),y=ph(Math.round(e[l+b+16]*f)+8);g|=((v|y<<4)&255)<<w*8}s[m*4+h]=g>>>0}}return{bits:s,scales:o}}function ph(e){return e<0?0:e>15?15:e}function gS(e,t,r){let n=r/4,a=new Uint32Array(t*n),s=new Float32Array(t);for(let o=0;o<t;++o){let i=o*r,u=0;for(let c=0;c<r;++c)u=Math.max(u,Math.abs(e[i+c]));let l=u>0?u/127:1;s[o]=l;for(let c=0;c<n;++c){let d=i+c*4,p=fi(e[d]/l),f=fi(e[d+1]/l),m=fi(e[d+2]/l),h=fi(e[d+3]/l);a[o*n+c]=p&255|(f&255)<<8|(m&255)<<16|(h&255)<<24}}return{packed:a,scales:s}}function mh(e,t,r){let n=null;return e.tensorByteRange?async(a,s)=>e.tensorByteRange(t,a*r,(a+s)*r):async(a,s)=>(n??=await e.tensorBytes(t),n.subarray(a*r,(a+s)*r))}function hS(e,t,r){let n=new r(t),a=0;for(let s of e)n.set(s,a),a+=s.length;return n}function fi(e){return Math.max(-127,Math.min(127,Math.round(e)))}function wS(e){return ArrayBuffer.isView(e)&&!(e instanceof DataView)&&"length"in e}var hi=128,bS=4*Uint32Array.BYTES_PER_ELEMENT;function Rc(e){return wr(e)}function vS(e){let t=e.device;return t.wgslLanguageFeatures?.has("immediate_address_space")===!0&&(t.limits?.maxImmediateSize??0)>=gh}var gh=16;function Gc(e){return e.replace(/[ \t]*\/\/.*$/gm,"").replace(/\n{2,}/g,`
`).trim()}function wi(e,t,r,n,a,s,o,i,u,l=[0,0]){if(i&&t.metadataStorage!==void 0&&t.metadataStorage!=="float")throw new Error(`transcode ${t.id} requires uint32 metadata storage`);if(i&&t.supportsF16Scales===!1)throw new Error(`transcode ${t.id} requires float32 scale storage`);let c=Math.ceil(o/t.workgroupSize),d=e.device.limits.maxComputeWorkgroupsPerDimension,p=!t.bakedWgsl&&vS(e),f=[o,s,l[0],l[1]],m=p?null:e.createUniformU32(f,`${t.id}-params`),h=[{tensor:r,type:"read-only-storage",binding:0},{tensor:n,type:"storage",binding:1},{tensor:a,type:"storage",binding:2},...m?[{buffer:m,type:"uniform",binding:3}]:[]];u&&h.push({tensor:u,type:"read-only-storage",binding:4});let g={name:`com.xenova.transcode.${t.id}`,source:t.buildWgsl({f16Scales:i,immediateParams:p}),entryPoint:"main",cacheKey:`com.xenova.transcode:${t.id}:${t.metadataStorage==="atomicU32"?"au32":t.metadataStorage==="uint32"?"u32":i?"f16":"f32"}`+(p?":imm":""),bindings:h,...p?{immediateSize:gh,immediateData:new Uint32Array(f)}:{},dispatchWorkgroups:[Math.min(c,d),Math.ceil(c/d),1]},w=!1;return{program:g,release(){w||(w=!0,m?.destroy?.())}}}async function Lc(e,t){if(t.length!==0)try{await e.runProgramSequence(t.map(r=>r.program))}finally{for(let r of t)r.release()}}function bi(e,t,r){return t.grid?_S(e,t.grid,r):void 0}function _S(e,t,r){let n=r.get(t.name);if(!n){let a=new Uint8Array(Math.ceil(t.data.byteLength/4)*4);a.set(t.data),n=e.tensorFromTypedArray("uint32",[a.byteLength/4],new Uint32Array(a.buffer)),r.set(t.name,n)}return n}var Dn=class{#e=[];track(t){this.#e.push(t)}destroyAll(){for(let t of this.#e)t.destroy();this.#e.length=0}};function vi(e,t,r,n,a,s,o,i,u,l,c=[0,0]){let d=_i(e,t,r,n,a,s,o,i,u,l,c);return e.runProgram(d.program).finally(()=>d.release())}function _i(e,t,r,n,a,s,o,i,u,l,c=[0,0]){if(!t)throw new Error("transcode kernel for this weight format is not in this bundle (built for the shipped model's weight formats \u2014 load that model's own GGUF, or rebuild)");let d=r.byteLength%4===0&&r.byteOffset%4===0,p;if(d)p=new Uint32Array(r.buffer,r.byteOffset,r.byteLength/4);else{let h=new Uint8Array(Math.ceil(r.byteLength/4)*4);h.set(r),p=new Uint32Array(h.buffer)}let f=e.tensorFromTypedArray("uint32",[p.length],p);l?.track(f);let m=bi(e,t,u);return wi(e,t,f,n,a,s,o,i,m,c)}var ft={Q8_0:{id:"q8_0_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  ss(ob, f16at(base));
  var q: array<u32, 8>; load32(base + 2u, &q);
  for (var w = 0u; w < 8u; w = w + 1u) { out_bits[ob * 8u + w] = q[w]; }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  ss(ob, f16at(base));
  var q: array<u32, 8>; load32(base + 2u, &q);
  for (var w = 0u; w < 8u; w = w + 1u) { out_bits[ob * 8u + w] = q[w]; }
}`},Q4_KNC:{id:"q4_k_to_q4knc",workgroupSize:64,metadataStorage:"uint32",supportsF16Scales:!1,bakedWgsl:!0,buildWgsl:e=>{if(e.f16Scales)throw new Error("transcode q4_k_to_q4knc requires uint32 metadata storage");return`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<u32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = bitcast<u32>(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let j = bi % 8u; let base = sb * 36u;
  let og = (ob / 8u) * 32u;
  let hi = (j & 1u) == 1u;
  let qsWordBase = base + 4u + (j >> 1u) * 8u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let loByte = b8(&qs, 4u * w + k); let hiByte = b8(&qs, 4u * w + k + 16u);
      let nLo = select(loByte & 0x0Fu, loByte >> 4u, hi);
      let nHi = select(hiByte & 0x0Fu, hiByte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[og + 4u * j + w] = word;
  }
  if (j == 0u) {
    out_scales[(ob / 8u) * 5u] = src[base];
  }
  if ((j & 1u) == 0u) {
    let c = j >> 1u;
    let sm0 = getSM(base * 4u, j);
    let sm1 = getSM(base * 4u, j + 1u);
    let carrier = sm0.x | (sm0.y << 6u) | (sm1.x << 12u) | (sm1.y << 18u);
    out_scales[(ob / 8u) * 5u + 1u + c] = carrier;
  }
}`}},Q6_KN:{id:"q6_k_to_q6k",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn q6code(base: u32, e: u32) -> u32 {
  let h = e >> 7u; let l = e & 127u;
  let r = l >> 5u; let n = l & 31u;
  let qlb = gbyte(base + 64u * h + n + 32u * (r & 1u));
  let nib = select(qlb >> 4u, qlb & 15u, r < 2u);
  let hi2 = (gbyte(base + 128u + 32u * h + n) >> (2u * r)) & 3u;
  return nib | (hi2 << 4u);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = (bi / 8u) * 210u;
  let j = bi % 8u;
  let og = (ob / 8u) * 52u;
  var hiLo = 0u;
  var hiHi = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let cLo = q6code(base, 32u * j + 4u * w + k);
      let cHi = q6code(base, 32u * j + 16u + 4u * w + k);
      word = word | (((cLo & 15u) | ((cHi & 15u) << 4u)) << (8u * k));
      hiLo = hiLo | ((cLo >> 4u) << (8u * k + 2u * w));
      hiHi = hiHi | ((cHi >> 4u) << (8u * k + 2u * w));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + 2u * j] = hiLo;
  out_bits[og + 32u + 2u * j + 1u] = hiHi;
  if (j < 4u) { out_bits[og + 48u + j] = word32at(base + 192u + 4u * j); }
  if (j == 0u) { ss(ob / 8u, f16at(base + 208u)); }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn q6code(base: u32, e: u32) -> u32 {
  let h = e >> 7u; let l = e & 127u;
  let r = l >> 5u; let n = l & 31u;
  let qlb = gbyte(base + 64u * h + n + 32u * (r & 1u));
  let nib = select(qlb >> 4u, qlb & 15u, r < 2u);
  let hi2 = (gbyte(base + 128u + 32u * h + n) >> (2u * r)) & 3u;
  return nib | (hi2 << 4u);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = (bi / 8u) * 210u;
  let j = bi % 8u;
  let og = (ob / 8u) * 52u;
  var hiLo = 0u;
  var hiHi = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let cLo = q6code(base, 32u * j + 4u * w + k);
      let cHi = q6code(base, 32u * j + 16u + 4u * w + k);
      word = word | (((cLo & 15u) | ((cHi & 15u) << 4u)) << (8u * k));
      hiLo = hiLo | ((cLo >> 4u) << (8u * k + 2u * w));
      hiHi = hiHi | ((cHi >> 4u) << (8u * k + 2u * w));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + 2u * j] = hiLo;
  out_bits[og + 32u + 2u * j + 1u] = hiHi;
  if (j < 4u) { out_bits[og + 48u + j] = word32at(base + 192u + 4u * j); }
  if (j == 0u) { ss(ob / 8u, f16at(base + 208u)); }
}`},Q4_K:{id:"q4_k_to_kq4",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let s = bi % 8u; let base = sb * 36u;
  let dd = unpack2x16float(src[base]);
  let scm = getSM(base * 4u, s);
  let d1 = dd.x * f32(scm.x); let m1 = dd.y * f32(scm.y);
  ss(ob * 2u, d1); ss(ob * 2u + 1u, 8.0 * d1 - m1);
  let qsWordBase = base + 4u + (s >> 1u) * 8u; let hi = (s & 1u) == 1u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let lo_byte = b8(&qs, k); let hi_byte = b8(&qs, k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (bb * 8u));
    }
    out_bits[ob * 4u + w] = word;
  }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let s = bi % 8u; let base = sb * 36u;
  let dd = unpack2x16float(src[base]);
  let scm = getSM(base * 4u, s);
  let d1 = dd.x * f32(scm.x); let m1 = dd.y * f32(scm.y);
  ss(ob * 2u, d1); ss(ob * 2u + 1u, 8.0 * d1 - m1);
  let qsWordBase = base + 4u + (s >> 1u) * 8u; let hi = (s & 1u) == 1u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let lo_byte = b8(&qs, k); let hi_byte = b8(&qs, k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (bb * 8u));
    }
    out_bits[ob * 4u + w] = word;
  }
}`}},er={Q4_K:{id:"q4_k_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`},Q6_K:{id:"q6_k_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`}},qr={Q4_K:{id:"q4_k_to_f32",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`},Q6_K:{id:"q6_k_to_f32",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`}};var yt={},Ct={},rn={};var hh=`
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }`;var wh=`${hh}
fn r_half_up(x: f32) -> f32 { return floor(x + 0.5); }`;function yS(e){return Gc(`${e?`enable f16;
`:""}@group(0) @binding(0) var<storage, read> emb: array<f32>;
@group(0) @binding(1) var<storage, read_write> packed: array<u32>;
@group(0) @binding(2) var<storage, read_write> scales: array<${e?"f16":"f32"}>;
struct P { chunkRows: u32, cols: u32, rowOffset: u32 }
@group(0) @binding(3) var<uniform> params: P;
${wh}
fn qi8(x: f32) -> i32 { return clamp(i32(r_half_up(x)), -127, 127); }
var<workgroup> wmax: array<f32, 256>;
@compute @workgroup_size(256)
fn main(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id) lid3: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let localRow = wid.x + wid.y * nwg.x;
  if (localRow >= params.chunkRows) { return; }
  let row = params.rowOffset + localRow;
  let lid = lid3.x; let cols = params.cols; let rowOff = localRow * cols;
  var m = 0.0;
  for (var c = lid; c < cols; c = c + 256u) { m = max(m, abs(emb[rowOff + c])); }
  wmax[lid] = m; workgroupBarrier();
  for (var s = 128u; s > 0u; s = s >> 1u) { if (lid < s) { wmax[lid] = max(wmax[lid], wmax[lid + s]); } workgroupBarrier(); }
  let maxAbs = wmax[0];
  let scale = select(1.0, div_exact(maxAbs, 127.0, recip_exact(127.0)), maxAbs > 0.0);
  let inv = recip_exact(scale);
  if (lid == 0u) { scales[row] = ${e?"f16(scale)":"scale"}; }
  let cols4 = cols >> 2u;
  for (var c4 = lid; c4 < cols4; c4 = c4 + 256u) {
    let base = rowOff + c4 * 4u;
    let q0 = qi8(emb[base] * inv); let q1 = qi8(emb[base + 1u] * inv);
    let q2 = qi8(emb[base + 2u] * inv); let q3 = qi8(emb[base + 3u] * inv);
    packed[row * cols4 + c4] = (u32(q0) & 0xFFu) | ((u32(q1) & 0xFFu) << 8u) | ((u32(q2) & 0xFFu) << 16u) | ((u32(q3) & 0xFFu) << 24u);
  }
}`)}function AS(e){return Gc(`${e?`enable f16;
`:""}@group(0) @binding(0) var<storage, read> emb: array<f32>;
@group(0) @binding(1) var<storage, read_write> bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> scales: array<${e?"f16":"f32"}>;
struct P { chunkBlocks: u32, cols: u32, blkOffset: u32 }
@group(0) @binding(3) var<uniform> params: P;
${wh}
fn nib(q: i32) -> u32 { return u32(clamp(q, 0, 15)); }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let localBlk = gid.x + gid.y * nwg.x * 64u;
  if (localBlk >= params.chunkBlocks) { return; }
  let blk = params.blkOffset + localBlk;
  let base = localBlk * 32u;
  var amax = 0.0; var vmax = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) { let v = emb[base + j]; let a = abs(v); if (a > amax) { amax = a; vmax = v; } }
  let d = vmax * -0.125;
  let id = select(0.0, recip_exact(d), d != 0.0);
  scales[blk] = ${e?"f16(d)":"d"};
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let xlo = emb[base + k] * id; let xhi = emb[base + k + 16u] * id;
      let qlo = nib(i32(r_half_up(xlo)) + 8);
      let qhi = nib(i32(r_half_up(xhi)) + 8);
      word = word | (((qlo | (qhi << 4u)) & 0xFFu) << (bb * 8u));
    }
    bits[blk * 4u + w] = word;
  }
}`)}var kS=32*1024*1024;async function yi(e,t,r,n,a,s,o){let u=o.embed??!0,l=o.head??"q4_0",c=l==="q4_0";if(!u&&!l)throw new Error("transcodeEmbedRows: targets must select at least one of embed / head");let d=e.device.limits.maxComputeWorkgroupsPerDimension,p=n/4,f=n/32,m=u?e.empty("uint32",[r*p]):null,h=u?e.empty("float32",[r]):null,g=c?e.empty("uint32",[r*f*4]):null,w=c?e.empty(a?"float16":"float32",[r*f]):null,b=null,v=null,y=Math.max(1,Math.min(r,s??(Math.floor(kS/(n*4))||1)));for(let S=0;S<r;S+=y){let C=Math.min(y,r-S),k;try{k=await t(S,C)}catch(R){throw await cr(e,[m,h,g,w,b,v]),R}let B=k instanceof Float32Array?e.tensorFromTypedArray("float32",[C*n],k):k,P=m&&h?e.createUniformU32([C,n,S,0],"embedQ8-params"):null,_=g&&w?e.createUniformU32([C*f,n,S*f,0],"embedQ4-params"):null,x=b&&v?e.createUniformU32([C*f,n,S*f,0],"embedQ2-params"):null;if(m&&h&&P&&await e.runProgram({name:"com.xenova.transcode.embedQ8Rows",source:yS(!1),entryPoint:"main",cacheKey:"com.xenova.transcode:embedQ8Rows:f32",bindings:[{tensor:B,type:"read-only-storage",binding:0},{tensor:m,type:"storage",binding:1},{tensor:h,type:"storage",binding:2},{buffer:P,type:"uniform",binding:3}],dispatchWorkgroups:[Math.min(C,d),Math.ceil(C/d),1]}),g&&w&&_){let R=C*f,O=Math.ceil(R/64);await e.runProgram({name:"com.xenova.transcode.embedQ4Rows",source:AS(a),entryPoint:"main",cacheKey:`com.xenova.transcode:embedQ4Rows:${a?"f16":"f32"}`,bindings:[{tensor:B,type:"read-only-storage",binding:0},{tensor:g,type:"storage",binding:1},{tensor:w,type:"storage",binding:2},{buffer:_,type:"uniform",binding:3}],dispatchWorkgroups:[Math.min(O,d),Math.ceil(O/d),1]})}await e.queueIdle(),B.destroy(),P?.destroy?.(),_?.destroy?.(),x?.destroy?.()}return{embedQ8:m,embedQ8Scales:h,lmHeadQ4:g,lmHeadQ4Scales:w,lmHeadQ2:b,lmHeadQ2Scales:v}}function bh(e,t){let r=/^model\.layers\.(\d+)\.(.+)\.weight$/.exec(e);if(!r)return null;let[,n,a]=r,s=t[a];return s?`blk.${n}.${s}.weight`:null}function qc(e,t){return{has:r=>e.has(t(r)),describe(r){let n=e.info(t(r));return{dtype:ds(n.type),shape:[...n.shape],elementCount:n.elementCount}},async read(r){let n=t(r),a=e.info(n),s=await e.tensorBytes(n),o=ha(a.type,s);return o?new Uint8Array(o.buffer,o.byteOffset,o.byteLength):s},async close(){}}}function Oc(e,t){return{async streamAll(r,{concurrency:n,chunkMaxBytes:a,names:s=null,signal:o}={}){if(!s)throw new Error("makeGGUFWeightSource.streamAll requires the plan's tensor names");let i=new Map(s.map(u=>[t(u),u]));await e.streamTensors([...i.keys()],async(u,l)=>{let c=ha(e.info(u).type,l),d=c?new Uint8Array(c.buffer,c.byteOffset,c.byteLength):l;await r({bytes:d,tensors:[{name:i.get(u),offset:0,length:d.byteLength}]})},{concurrency:n,chunkMaxBytes:a,signal:o})},close(){}}}var vh={"self_attn.q_proj":"attn_q","self_attn.k_proj":"attn_k","self_attn.v_proj":"attn_v","self_attn.out_proj":"attn_output","self_attn.q_layernorm":"attn_q_norm","self_attn.k_layernorm":"attn_k_norm","conv.in_proj":"shortconv.in_proj","conv.conv":"shortconv.conv","conv.out_proj":"shortconv.out_proj","feed_forward.w1":"ffn_gate","feed_forward.w3":"ffn_up","feed_forward.w2":"ffn_down",operator_norm:"attn_norm",ffn_norm:"ffn_norm"};function _h(e){return e==="model.embed_tokens.weight"?"token_embd.weight":e==="model.embedding_norm.weight"?"token_embd_norm.weight":e==="lm_head.weight"?"output.weight":bh(e,vh)??e}var SS={...vh,"self_attn.o_proj":"attn_output","self_attn.q_norm":"attn_q_norm","self_attn.k_norm":"attn_k_norm","self_attn.gate_proj":"attn_gate",post_operator_norm:"post_attention_norm",post_ffn_norm:"post_ffw_norm"};function _s(e){return e==="model.embedding_norm.weight"?"output_norm.weight":bh(e,SS)??_h(e)}function br(e){return e.slice(e.lastIndexOf(".")+1)}var _a=64*1024*1024;function xS(e,t){for(;t!==0;)[e,t]=[t,e%t];return e}function TS(e){for(let t=1;t<=8;++t){let r=t*ga;try{let n=Rn(e,r);return{elementsPerSourceBlock:r,bytesPerSourceBlock:n,outputBlocksPerSourceBlock:t}}catch{}}throw new Error(`Cannot determine GGML source-block geometry for type ${e}`)}function yh(e,t,r,n=_a){if(!Number.isSafeInteger(t)||t<=0)throw new Error(`Invalid block-quant element count ${t}`);if(!Number.isSafeInteger(r)||r<=0)throw new Error(`Invalid block-quant byte length ${r}`);if(!Number.isSafeInteger(n)||n<=0)throw new Error(`Invalid block-quant chunk byte cap ${n}`);let a=Math.min(n,_a),s=TS(e);if(t%s.elementsPerSourceBlock!==0)throw new Error(`GGML type ${e} element count ${t} is not divisible by source block ${s.elementsPerSourceBlock}`);let o=t/s.elementsPerSourceBlock,i=o*s.bytesPerSourceBlock;if(i!==r)throw new Error(`GGML type ${e} byte length ${r} does not match expected ${i}`);let u=4/xS(s.bytesPerSourceBlock,4),l=Math.floor(a/s.bytesPerSourceBlock),c=l-l%u;if(c<=0&&o>l)throw new Error(`GGML type ${e} source block cannot form a 4-byte-aligned chunk within ${a} bytes`);let d=[],p=0;for(;p<o;){let f=o-p,m=f<=l?f:c,h=p*s.bytesPerSourceBlock,g=h+m*s.bytesPerSourceBlock;d.push({relBegin:h,relEnd:g,outputBlockBegin:p*s.outputBlocksPerSourceBlock,outputBlockCount:m*s.outputBlocksPerSourceBlock,directMappable:h%4===0&&g%4===0}),p+=m}return d}var CS=new Set([I.PRISM_PQ2_0,I.PRISM_PTQ1_0,I.Q1_0,I.Q2_0,I.TQ1_0,I.TQ2_0,I.IQ2_XXS,I.IQ2_XS,I.IQ2_S,I.IQ3_XXS,I.IQ3_S,I.IQ1_S,I.IQ1_M]);function Dc(e){return e===I.Q4_0?4:CS.has(e)?null:Xo(e)?8:null}var tr={q_proj:["q_proj","k_proj","v_proj"],k_proj:["q_proj","k_proj","v_proj"],v_proj:["q_proj","k_proj","v_proj"],gate_proj:["gate_proj","up_proj"],up_proj:["gate_proj","up_proj"],linear_in_proj_qkv:["linear_in_proj_qkv"],linear_in_proj_z:["linear_in_proj_z"],attn_q:["attn_q","attn_k","attn_v"],attn_k:["attn_q","attn_k","attn_v"],attn_v:["attn_q","attn_k","attn_v"],gate_exps:["gate_exps","up_exps"],up_exps:["gate_exps","up_exps"]},PS={...tr,q_proj:["q_proj"],k_proj:["k_proj"],v_proj:["v_proj"]},Ah={...PS,gate_proj:["gate_proj"],up_proj:["up_proj"]};var IS=new Set([I.Q4_K,I.Q4_1]);function Mc(e,t,r,n=tr){let a=new Set;for(let s of e){let o=br(s.id);if(!t.has(o))continue;let i=r(s.id);if(i===null)continue;let u=n[o]??[o],l=s.id.slice(0,s.id.length-o.length);u.every(c=>r(l+c)===i)&&a.add(s.id)}return a}function Wc(e,t,r,n,a){let s=new Map(e.map(o=>[o.id,o]));return o=>{let i=s.get(o);if(!i||a&&!a.has(br(o)))return null;let u=r(i.name);return t.has(u)&&n(t.info(u).type)?"1":null}}function kh(e,t,r,n,a){return Mc(e,n,Wc(e,t,r,s=>IS.has(s),n),a)}var Sh={linear_in_proj_qkv:["linear_in_proj_qkv","linear_in_proj_z"],linear_in_proj_z:["linear_in_proj_qkv","linear_in_proj_z"],q_proj:["q_proj","k_proj","v_proj"],k_proj:["q_proj","k_proj","v_proj"],v_proj:["q_proj","k_proj","v_proj"],gate_proj:["gate_proj","up_proj"],up_proj:["gate_proj","up_proj"]};function xh(e,t,r,n,a=tr){let s=new Map(e.map(u=>[u.id,u])),o=u=>{let l=s.get(u);if(!l||!n.has(br(u)))return!1;let c=r(l.name);return t.has(c)&&Dc(t.info(c).type)!==null},i=new Set;for(let u of e){let l=br(u.id),c=a[l];if(!c||!o(u.id))continue;let d=u.id.slice(0,u.id.length-l.length);c.every(p=>o(d+p))||i.add(u.id)}return i}function Mn(e,t,r,n,a,s){let o=br(e.id);if(!n.has(o)||a?.has(e.id)||s?.has(e.id))return!1;let i=r(e.name);return t.has(i)&&Dc(t.info(i).type)!==null}function Fc(e,t,r,n,a,s){let o=!1,i=!0;for(let u of e)Mn(u,t,r,n,a,s)&&(o=!0,Dc(t.info(r(u.name)).type)!==4&&(i=!1));return o?i?4:8:null}function ys(e,t){let r=new Map,n=i=>{let u=r.get(i);if(u===void 0)return r.set(i,i),i;if(u===i)return i;let l=n(u);return r.set(i,l),l},a=(i,u)=>{let l=n(i),c=n(u);l!==c&&r.set(c,l)},s=new Set(e.map(i=>i.id));for(let i of e){let u=br(i.id),l=i.id.slice(0,i.id.length-u.length);for(let c of t?[tr,t]:[tr])for(let d of c[u]??[]){let p=l+d;s.has(p)&&a(i.id,p)}}let o=new Map;for(let i of e){if(i.packedGroup===void 0)continue;let u=o.get(i.packedGroup);u===void 0?o.set(i.packedGroup,i.id):a(u,i.id)}return new Map(e.map(i=>[i.id,n(i.id)]))}function Ai(e,t){if(e.length===0)return null;if(e.length>1)throw new Error(`${t} split across ${e.length} device-sized buffers, but its consumer binds one pack whole. Pack a smaller group at a time, or teach the consumer to resolve a pack per weight id.`);return e[0]}var Ph={q6_k:"q6k",q5_k:"q5k",q3_k:"q3k",q2_k:"q2k",q4_k_native_compact:"q4knc"};function Qc(e,t){return e===I.Q4_K?t%256===0?"q4_k_native_compact":"q4_k":e===I.Q4_1?"q4_k":e===I.Q6_K&&t%256===0?"q6_k":e===I.Q5_K&&t%256===0?"q5_k":e===I.Q3_K&&t%256===0?"q3_k":e===I.Q2_K&&t%256===0?"q2_k":null}function BS(e,t){let n=Qc(e,t);return n!=="q3_k"&&n!=="q2_k"&&n!=="q6_k"&&n!=="q5_k"?null:n}function ES(e,t){return!1}function RS(e,t,r){return!1}function GS(e){return e==="q6_k"||e==="q5_k"||e==="q3_k"||e==="q2_k"}var LS=256;function ki(e,t,r,n,a,s,o=!1){let i={embedFormat:null,headPacked:!1,headFormat:null,headLut2:null,embedLut2:null,headRescoreQ8:!1,headScan2:!1,headAliasesEmbed:!1,headKSuperScan:!1};if(o)return MS(e,t,r,n,a,i);let u=r.hidden_size;if(!s||!Lr(r)||u%32!==0)return i;let l=n.find(y=>y.id==="embed_tokens");if(!l)return i;let c=t(l.name);if(!e.has(c))return i;let d=e.info(c).type,f=Qc(d,u)??(ec(d)?"q8_rows":null),m=a?d:(()=>{let y=n.find(C=>C.id==="lm_head"),S=y?t(y.name):null;return S&&e.has(S)?e.info(S).type:null})(),h=m!==null&&vs(r)&&ec(m),g=h?BS(m,u)??(ES(a,f)?"q8_rows":null):null;return{embedFormat:f,headPacked:h,headFormat:g,headLut2:null,embedLut2:null,headRescoreQ8:!1,headScan2:!1,headAliasesEmbed:a&&g!==null&&g===f,headKSuperScan:!1}}var qS=new Set(["q4_k_native_compact","q6_k","q5_k","q3_k","q2_k"]);function OS(e){return e!==null&&qS.has(e)}function DS(e,t){return e===I.Q8_0&&t%32===0?"q8_0":null}function MS(e,t,r,n,a,s){let o=r.hidden_size,i=m=>{let h=n.find(w=>w.id===m),g=h?t(h.name):null;return g&&e.has(g)?e.info(g).type:null},u=m=>{let h=m===null?null:Qc(m,o);return OS(h)?h:DS(m,o)},l=i("embed_tokens"),c=a?l:i("lm_head"),d=u(c),p=a&&d===null?null:u(l);return{...s,embedFormat:p,headPacked:d!==null,headFormat:d,headAliasesEmbed:a&&d!==null&&d===p,headKSuperScan:RS(a,d,r.vocab_size)}}function As(e,t,r,n,a=null){let s=t*(r/256),o=t*(r/32);if(e==="q4_0")return{bits:o*4,scales:o,scalesDtype:"float32"};if(e==="lut2_32"){if(a!==8&&a!==9)throw new Error(`lut2_32 gather needs codebook 8 or 9, got ${a}`);return{bits:o*2,scales:s*(a===9?1:3),scalesDtype:"uint32"}}return e==="q8_rows"?{bits:t*(r/4),scales:t,scalesDtype:"float32"}:e==="q8_0"?{bits:o*8,scales:o,scalesDtype:"float32"}:e==="q6_k"?{bits:s*52,scales:s,scalesDtype:"float32"}:e==="q5_k"?{bits:s*44,scales:s*2,scalesDtype:"float32"}:e==="q3_k"?{bits:s*28,scales:s,scalesDtype:"float32"}:e==="q2_k"?{bits:s*20,scales:s*2,scalesDtype:"float32"}:e==="q4_k_native_compact"?{bits:s*32,scales:s*5,scalesDtype:"uint32"}:{bits:o*4,scales:o*2,scalesDtype:n?"float16":"float32"}}function WS(e){return e==="q8_0"||Ph[e]!==void 0}async function Th(e,t,r,n){let a=Ph[r];if(r!=="q8_0"&&!a)throw new Error(`${t.name} resolved to the ${r} grid, which has no sole-table packer`);let s=a?await n.packKSuperGGUFWeights({...e,entries:[t]},{ids:new Set([t.id]),...n.kSuperRoutes[a],scalesF16:!1}):await n.packQ8_0GGUFTable(e,t),o=Ai(s,`the ${r} table for ${t.name}`);if(o&&Gr(o.offsets,t.id)!==0)throw new Error(`${t.name} must be packed at block 0 of its gather buffer`);return o}async function Ih(e,t,r,n){let a=new Set,s={skipDenseIds:a},o=[],i=u=>(o.push(u),u);try{return await FS(s,a,i,e,t,r,n)}catch(u){throw await cr(e.runtime,[s,o]),u}}async function FS(e,t,r,n,a,s,o){let{runtime:i,entries:u,gguf:l,nameToGGUF:c}=n,d=a.hidden_size,p=a.vocab_size,f=ki(l,c,a,u,s,i.device.features.has("shader-f16"),n.sourcePrecision);if(!f.embedFormat&&!f.headPacked)return null;let m=u.find(k=>k.id==="embed_tokens");if(!m)return null;let h=c(m.name),g=s?m:u.find(k=>k.id==="lm_head"),w=g?c(g.name):null,b=f.headPacked&&w!==null,v=l.info(h).type,y=f.embedFormat;if(y&&WS(y)){let k=r(await Th(n,m,y,o));if(!k)throw new Error(`${h} resolved to the ${y} gather but produced no pack`);e.embedBits=k.bits,e.embedScales=k.scales,e.embedFormat=y,t.add("embed_tokens")}else if(f.embedFormat==="q4_k"){let k=Ai(r(await o.packKq4GGUFWeights({...n,entries:[m]},{kq4Ids:new Set([m.id])})),`the kq4 gather table for ${h}`);if(!k)throw new Error(`${h} resolved to the kq4 gather but produced no pack`);if(Gr(k.offsets,m.id)!==0)throw new Error(`${h} must be packed at block 0 of its gather buffer`);e.embedBits=k.bits,e.embedScales=k.scales,e.embedFormat="q4_k",t.add("embed_tokens")}else if(f.embedFormat==="q8_rows"){let k=b&&w===h,P=k&&f.headFormat!=="q8_rows"?"q4_0":!1,_=r(await Ch(i,l,h,v,p,d,{embed:!0,head:P}));if(!_.embedQ8||!_.embedQ8Scales)throw new Error(`${h} produced no q8_rows gather pack`);let x=P==="scan2"?_.lmHeadQ2:_.lmHeadQ4,R=P==="scan2"?_.lmHeadQ2Scales:_.lmHeadQ4Scales;e.embedBits=_.embedQ8,e.embedScales=_.embedQ8Scales,e.embedFormat="q8_rows",t.add("embed_tokens"),k&&f.headFormat==="q8_rows"?(e.lmHeadQ8=_.embedQ8,e.lmHeadQ8Scales=_.embedQ8Scales,t.add("lm_head")):P&&x&&R&&(e.lmHeadQ4=x,e.lmHeadQ4Scales=R,P==="scan2"&&(e.lmHeadScanBits=2),t.add("lm_head"))}let S=b&&!e.lmHeadQ4&&!e.lmHeadQ8?f.headFormat:null;if(S==="q8_rows")throw new Error(`${w} resolved to the q8_rows head, which only the tied gather pass can bind`);let C=S;if(C&&f.headAliasesEmbed&&e.embedBits&&e.embedScales)e.lmHeadQ4=e.embedBits,e.lmHeadQ4Scales=e.embedScales,e.lmHeadFormat=C,t.add("lm_head");else if(C&&g){let k=r(await Th(n,g,C,o));if(!k)throw new Error(`${w} resolved to the ${C} head grid but produced no pack`);e.lmHeadQ4=k.bits,e.lmHeadQ4Scales=k.scales,e.lmHeadFormat=C,t.add("lm_head")}else if(b&&!e.lmHeadQ4&&!e.lmHeadQ8){let k=r(await Ch(i,l,w,l.info(w).type,p,d,{embed:!1,head:"q4_0"}));if(!k.lmHeadQ4||!k.lmHeadQ4Scales)throw new Error(`${w} produced no Q4_0 head pack`);e.lmHeadQ4=k.lmHeadQ4,e.lmHeadQ4Scales=k.lmHeadQ4Scales,t.add("lm_head")}return t.size>0?e:null}function QS(e,t){let n={[I.Q4_0]:{spec:qr.Q4_0,blockElements:32},[I.Q4_1]:{spec:qr.Q4_1,blockElements:32},[I.Q5_0]:{spec:qr.Q5_0,blockElements:32},[I.Q5_1]:{spec:qr.Q5_1,blockElements:32},[I.Q4_K]:{spec:qr.Q4_K,blockElements:256},[I.Q5_K]:{spec:qr.Q5_K,blockElements:256},[I.Q6_K]:{spec:qr.Q6_K,blockElements:256}}[e];return n?.spec&&t%n.blockElements===0?n.spec:void 0}async function Ch(e,t,r,n,a,s,o){let i=mh(t,r,Rn(n,s)),u=QS(n,s);if(!u)return yi(e,async(p,f)=>Zr(n,await i(p,f)),a,s,!0,void 0,o);let l=e.empty("float32",[1],"embedDequant.scales"),c=new Dn,d=new Map;try{return await yi(e,async(p,f)=>{c.destroyAll();let m=e.empty("float32",[f*s],"embedDequant.chunk");try{await vi(e,u,await i(p,f),m,l,0,f*s/32,!1,d,c)}catch(h){throw m.destroy(),h}return m},a,s,!0,void 0,o)}finally{c.destroyAll(),l.destroy()}}var jc={q6k:{wordsPerBlock:6.5,scalesPerBlock:.125,metadataU32:!1},q5k:{wordsPerBlock:5.5,scalesPerBlock:.25,metadataU32:!1},q3k:{wordsPerBlock:3.5,scalesPerBlock:.125,metadataU32:!1},q2k:{wordsPerBlock:2.5,scalesPerBlock:.25,metadataU32:!1},q4knc:{wordsPerBlock:4,scalesPerBlock:.625,metadataU32:!0},iq4xs:{wordsPerBlock:4,scalesPerBlock:.25,metadataU32:!0}};function Bh(e,t,r){let n=Number(e.device.limits.maxBufferSize),a=Number(e.device.limits.maxStorageBufferBindingSize);if(t%4!==0||t<=0||Number.isFinite(n)&&t>n||Number.isFinite(a)&&t>a)return null;let s=e.beginMappedTensorUpload("uint32",[t/4],`${r}.direct-source`);if(!s||s.destination.byteLength!==t){try{s?.abort()}catch{}return null}return{bytes:s.destination,commit:()=>s.commit(),rollback:o=>s.rollback(o),abort:()=>s.abort()}}function Eh(e,t,r,n){if(e.byteLength%mr!==0)throw new Error(`Q8_0 byte length ${e.byteLength} is not a multiple of ${mr}`);let a=e.byteLength/mr,s=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let o=0;o<a;++o){let i=o*mr;r[n+o]=fr(s.getUint16(i,!0));let u=i+2;for(let l=0;l<8;++l){let c=0;for(let d=0;d<4;++d)c|=(e[u+l*4+d]&255)<<d*8;t[(n+o)*8+l]=c>>>0}}}function NS(e){let t=e.length/En,r=new Uint32Array(t*8),n=new Float32Array(t);for(let a=0;a<t;++a){let s=a*En,o=0;for(let l=0;l<En;++l){let c=Math.abs(e[s+l]);c>o&&(o=c)}let i=o/127,u=i>0?1/i:0;n[a]=i;for(let l=0;l<8;++l){let c=0;for(let d=0;d<4;++d){let p=Math.round(e[s+l*4+d]*u);p>127?p=127:p<-127&&(p=-127),c|=(p&255)<<d*8}r[a*8+l]=c>>>0}}return{bits:r,scales:n}}function zS(e){switch(e){case I.Q4_K:return er.Q4_K;case I.Q6_K:return er.Q6_K;case I.Q5_K:return er.Q5_K;case I.Q3_K:return er.Q3_K;case I.Q2_K:return er.Q2_K;case I.IQ4_XS:return er.IQ4_XS;case I.IQ4_NL:return er.IQ4_NL;case I.Q5_0:return er.Q5_0;case I.Q5_1:return er.Q5_1;default:return null}}function KS(e,t){return e===4?ft.Q4_0:t===I.Q8_0?ft.Q8_0:t===I.Q8_1?ft.Q8_1:t===I.Q8_K?ft.Q8_KQ8:zS(t)}var US=8;async function xi({runtime:e,entries:t,gguf:r,nameToGGUF:n,signal:a,streamByteBudget:s,shardUnitOf:o},{routeName:i,include:u,guardElements:l=ga,bitsLength:c,scalesLength:d,scalesF16:p,metadataStorage:f="float",specForSlot:m,cpuFallback:h}){let g=[];for(let M of t){if(!u(M))continue;let j=n(M.name),A=r.info(j);if(A.elementCount%l!==0)throw new Error(`${i} tensor ${j} element count ${A.elementCount} not divisible by ${l}`);g.push({id:M.id,gname:j,blocks:A.elementCount/ga,type:A.type})}if(g.length===0)return[];let w=0,b=g.map(M=>{let j={gname:M.gname,baseBlock:w,blocks:M.blocks,type:M.type,shardIndex:0};return w+=M.blocks,j}),v=f==="float"&&p(b),y=f!=="float"?"uint32":v?"float16":"float32",S=wr(e.device.limits),C=Ae(y),k=M=>Math.max(c(M)*Uint32Array.BYTES_PER_ELEMENT,d(M)*C),B=On(g.map(M=>{let j=o?.(M.id);return{id:M.id,blockCount:M.blocks,...j===void 0?{}:{unitKey:j}}}),qn({byteLimit:S,maxBlocks:g.reduce((M,j)=>M+j.blocks,0),bytesForBlocks:k}),i),P=B.shardBlockCounts;for(let M=0;M<P.length;++M){let j=c(P[M])*Uint32Array.BYTES_PER_ELEMENT,A=d(P[M])*C;if(!(j<=S&&A<=S))throw new Error(`${i} pack ${M} needs ${j} bytes of bits and ${A} of scales, over this device's ${S}-byte buffer limit. One indivisible co-bound weight group is larger than a single buffer, or the support probe planned a different split than the loader.`)}let _=P.map(()=>Jt()),x=P.map(()=>Jt()),R=P.map(()=>new Set),O=g.map(M=>{let j=B.placements.get(M.id);return _t(_[j.shardIndex],M.id,j.blockOffset),_t(x[j.shardIndex],M.id,M.blocks),R[j.shardIndex].add(M.id),{gname:M.gname,baseBlock:j.blockOffset,blocks:M.blocks,type:M.type,shardIndex:j.shardIndex}}),N=M=>M.map((j,A)=>({bits:j.bits,scales:j.scales,offsets:_[A],blockCounts:x[A],packedIds:R[A],layout:O.filter(T=>T.shardIndex===A)})),Q=new Map(O.map(M=>[M.gname,M])),V=Pc();if(O.every(M=>m(M)!==null)){let M=P.map(Z=>({bits:e.empty("uint32",[c(Z)]),scales:e.empty(y,[d(Z)])})),j=new Map,A=new Set,T=new Map,E=[],X=[],z=0,G=Z=>{let U=Z?new Set(Z):null,te=E.filter(ne=>!U||U.has(ne.pending));E=E.filter(ne=>U&&!U.has(ne.pending));let ae=new Set;for(let[ne,xe]of T)U&&!U.has(ne)||(ae.add(xe),T.delete(ne),A.delete(xe));if(!U){for(let ne of A)ae.add(ne);A.clear()}return{direct:te,pools:ae}},L=Z=>{let{direct:U,pools:te}=G(Z);for(let ae of te)ae.destroyAll();for(let{staging:ae}of U)ae.destroy()},K=async Z=>{try{await e.queueIdle()}catch{}let{direct:U,pools:te}=G(Z);for(let{destination:ae,staging:ne}of U)try{await ae.rollback(ne)}catch{}for(let ae of te)ae.destroyAll()},D=async()=>{if(X.length===0)return;let Z=X;X=[],z=0;try{await Lc(e,Z),await e.queueIdle()}catch(U){throw await K(Z),U}L(Z)},F=async(Z,U,te,ae,ne)=>{let xe=m(Z),{bits:De,scales:Me}=M[Z.shardIndex];if(!te){let Ze=new Dn;A.add(Ze);let W=_i(e,xe,U,De,Me,ae,ne,v,j,Ze);return T.set(W,Ze),W}let ye=null;try{if(U.buffer!==te.bytes.buffer||U.byteOffset!==te.bytes.byteOffset||U.byteLength!==te.bytes.byteLength)throw new Error(`Direct GGUF source bytes for ${Z.gname} do not alias the destination`);let Ze=bi(e,xe,j);ye=await te.commit();let W=wi(e,xe,ye,De,Me,ae,ne,v,Ze);return E.push({destination:te,staging:ye,pending:W}),W}catch(Ze){if(ye)try{await te.rollback(ye)}catch{}else try{await te.abort()}catch{}throw Ze}},H=r.streamTensorRange?.bind(r),Y=H?O.filter(Z=>r.info(Z.gname).byteLength>_a):[],fe=new Set(Y.map(Z=>Z.gname)),ce=Y.length===0?O:O.filter(Z=>!fe.has(Z.gname)),re=r.supportsDirectDestinations===!0?new Map(ce.flatMap(Z=>{let U=r.info(Z.gname).byteLength;return U%4!==0?[]:[[Z.gname,()=>Bh(e,U,Z.gname)]]})):void 0;try{ce.length>0&&(await r.streamTensors(ce.map(Z=>Z.gname),async(Z,U,te)=>{let ae=Q.get(Z),ne=U.byteLength;X.length>0&&z+ne>_a&&await D();let xe=await F(ae,U,te,ae.baseBlock,ae.blocks);X.push(xe),z+=ne,(X.length>=US||z>=_a)&&await D(),await V(ne)},{signal:a,directDestinations:re,byteBudget:s}),await D(),await e.queueIdle(),L());for(let Z of Y){let U=r.info(Z.gname);for(let te of yh(Z.type,U.elementCount,U.byteLength)){let ae=!1,ne=r.supportsDirectDestinations===!0&&te.directMappable?()=>Bh(e,te.relEnd-te.relBegin,`${Z.gname}[${te.relBegin}:${te.relEnd}]`):void 0;if(await H(Z.gname,te.relBegin,te.relEnd,async(xe,De)=>{if(ae)throw new Error(`GGUF range stream delivered ${Z.gname} chunk more than once`);ae=!0;let Me=xe.byteLength,ye=te.relEnd-te.relBegin;if(Me!==ye)throw new Error(`GGUF range stream delivered ${Me} bytes for ${Z.gname}; expected ${ye}`);try{let Ze=await F(Z,xe,De,Z.baseBlock+te.outputBlockBegin,te.outputBlockCount);X.push(Ze),await D(),await e.queueIdle()}catch(Ze){for(let W of X)W.release();throw X=[],z=0,await K(),Ze}L()},{signal:a,directDestination:ne}),!ae)throw new Error(`GGUF range stream did not deliver ${Z.gname} chunk`);await V(te.relEnd-te.relBegin)}}}catch(Z){for(let U of X)U.release();X=[],z=0,await K();for(let U of M)U.bits.destroy(),U.scales.destroy();throw Z}return L(),N(M)}if(!h){let M=O.find(j=>m(j)===null);throw new Error(`${i}: no transcode kernel for ${M.gname} (type ${M.type})`)}if(f!=="float")throw new Error(`${i}: uint32 metadata routes require a GPU transcode kernel for every slot`);let $=P.map(M=>({bits:new Uint32Array(c(M)),scales:new Float32Array(d(M))}));return await r.streamTensors([...Q.keys()],async(M,j)=>{let A=Q.get(M),T=$[A.shardIndex];h(A,j,T.bits,T.scales),await V(j.byteLength)},{signal:a}),N($.map(M=>({bits:e.tensorFromTypedArray("uint32",[M.bits.length],M.bits),scales:v?e.tensorFromTypedArray("float16",[M.scales.length],Bn(M.scales)):e.tensorFromTypedArray("float32",[M.scales.length],M.scales)})))}async function Nc(e,{packSuffixes:t,excludeKq4Ids:r,forceDenseIds:n}){let{runtime:a,entries:s,gguf:o,nameToGGUF:i}=e,u=Fc(s,o,i,t,r,n);if(u===null)return[];let l=u;return(await xi(e,{routeName:"packed",include:d=>Mn(d,o,i,t,r,n),bitsLength:d=>d*l,scalesLength:d=>d,scalesF16:()=>a.device.features.has("shader-f16"),specForSlot:d=>KS(u,d.type),cpuFallback:(d,p,f,m)=>{if(d.type===I.Q8_0)Eh(p,f,m,d.baseBlock);else{let h=ha(d.type,p);if(!h)throw new Error(`packQ4GGUFWeights: ${d.gname} type ${d.type} is not dequantizable`);let g=NS(h);f.set(g.bits,d.baseBlock*8),m.set(g.scales,d.baseBlock)}}})).map(d=>({...d,packedQuantBits:u}))}function $S(e,t){return xi({...e,entries:[t]},{routeName:"q8_0 table",include:r=>r.id===t.id,bitsLength:r=>r*8,scalesLength:r=>r,scalesF16:()=>!1,specForSlot:()=>ft.Q8_0,cpuFallback:(r,n,a,s)=>Eh(n,a,s,r.baseBlock)})}async function zc(e,{kq4Ids:t,scalesF16:r}){return await xi(e,{routeName:"kq4",include:a=>t.has(a.id),bitsLength:a=>a*4,scalesLength:a=>a*2,scalesF16:()=>(r??!0)&&e.runtime.device.features.has("shader-f16"),specForSlot:a=>a.type===I.Q4_1?ft.Q4_1:ft.Q4_K})}var Kc=new Set(["gate_proj","up_proj","down_proj","o_proj","q_proj","k_proj","v_proj","linear_out_proj","conv_out_proj","conv_in_proj","linear_in_proj_qkv","linear_in_proj_z","attn_gate"]),Si=new Set([...Kc,"gate_exps","up_exps","down_exps"]),HS=new Set(["conv_in_proj","conv_out_proj"]),VS=new Set([...Si].filter(e=>!HS.has(e))),ZS=new Set(["gate_exps","up_exps","down_exps"]),Uc={q6k:{format:"q6_k",ggmlType:I.Q6_K,env:"Q6K_PACK",u32PerSuper:52,scalesPerSuper:1,spec:ft.Q6_KN,metadataU32:!1},q5k:{format:"q5_k",ggmlType:I.Q5_K,env:"Q5K_PACK",suffixes:Si,u32PerSuper:44,scalesPerSuper:2,spec:ft.Q5_KN,metadataU32:!1},q3k:{format:"q3_k",ggmlType:I.Q3_K,env:"Q3K_PACK",suffixes:Kc,u32PerSuper:28,scalesPerSuper:1,spec:ft.Q3_KN,metadataU32:!1},q2k:{format:"q2_k",ggmlType:I.Q2_K,env:"Q2K_PACK",suffixes:Si,u32PerSuper:20,scalesPerSuper:2,spec:ft.Q2_KN,metadataU32:!1},q4knc:{format:"q4_k_native_compact",ggmlType:I.Q4_K,env:"Q4KNC_PACK",suffixes:VS,u32PerSuper:32,scalesPerSuper:5,spec:ft.Q4_KNC,metadataU32:!0},iq4xs:{format:"iq4_xs",ggmlType:I.IQ4_XS,env:"IQ4XS_PACK",suffixes:ZS,u32PerSuper:32,scalesPerSuper:2,spec:ft.IQ4_XSN,metadataU32:!0}};function XS(e,t,r,n,a,s,o,i,u,l,c=!1){if(!t.has(e.format))return new Set;let d=e.format==="q3_k"||e.format==="q2_k"?i??o:o,p=d?new Set([...s].filter(h=>d.has(h))):s,f=e.suffixes?u??(c&&e.format==="q4_k_native_compact"?Si:e.suffixes):void 0,m=f?new Set([...p].filter(h=>f.has(h))):p;return m.size===0?new Set:JS(r,n,a,m,e.ggmlType,l)}function YS(e,t,r,n,a,s,o,i,u,l){return Object.fromEntries(Object.entries(Uc).map(([c,d])=>[c,XS(d,e,t,r,n,a,s,o,i,u,l)]))}function JS(e,t,r,n,a,s){return Mc(e,n,Wc(e,t,r,o=>o===a),s)}async function Rh(e,{ids:t,spec:r,u32PerSuper:n,scalesPerSuper:a,scalesF16:s=!1}){let o=r.metadataStorage??"float";if(o==="uint32"&&s)throw new Error(`k-super transcode ${r.id} cannot combine uint32 metadata with f16 scale storage`);return(await xi(e,{routeName:"k-super",include:u=>t.has(u.id),guardElements:256,bitsLength:u=>u/8*n,scalesLength:u=>u/8*a,scalesF16:()=>s&&e.runtime.device.features.has("shader-f16"),metadataStorage:o,specForSlot:()=>r})).map(u=>({...u,u32PerSuper:n,scalesPerSuper:a,metadataStorage:o}))}async function ya(e,t,r,n){if(r.size===0)return[];let{spec:a,u32PerSuper:s,scalesPerSuper:o}=Uc[t];return Rh(e,{ids:r,spec:a,u32PerSuper:s,scalesPerSuper:o,scalesF16:n})}var e6=new Map([[I.IQ2_XXS,yt.IQ2_XXS],[I.IQ2_XS,yt.IQ2_XS],[I.IQ2_S,yt.IQ2_S],[I.IQ3_XXS,yt.IQ3_XXS],[I.IQ3_S,yt.IQ3_S],[I.IQ4_NL,yt.IQ4_NL],[I.IQ4_XS,yt.IQ4_XS],[I.Q3_K,yt.Q3_K],[I.MXFP4,yt.MXFP4],[I.NVFP4,yt.NVFP4],[I.IQ1_S,yt.IQ1_S],[I.IQ1_M,yt.IQ1_M]].filter(e=>e[1]!==void 0)),t6=new Map([[I.Q2_0,rn.Q2_0],[I.PRISM_PQ2_0,rn.PRISM_PQ2_0],[I.PRISM_PTQ1_0,rn.PRISM_PTQ1_0],[I.IQ1_S,rn.IQ1_S],[I.IQ1_M,rn.IQ1_M]].filter(e=>e[1]!==void 0)),e9=new Map([[I.IQ2_XXS,Ct.IQ2_XXS],[I.IQ2_XS,Ct.IQ2_XS],[I.IQ2_S,Ct.IQ2_S],[I.IQ3_XXS,Ct.IQ3_XXS],[I.IQ3_S,Ct.IQ3_S],[I.IQ4_XS,Ct.IQ4_XS],[I.Q3_K,Ct.Q3_K],[I.MXFP4,Ct.MXFP4],[I.NVFP4,Ct.NVFP4],[I.IQ1_S,Ct.IQ1_S],[I.IQ1_M,Ct.IQ1_M]].filter(e=>e[1]!==void 0));var r6=new Set(["conv_in_proj"]),n6=new Set([...Kc,"linear_in_proj_qkv","linear_in_proj_z"].filter(e=>!r6.has(e)));function a6(){return n6}function Gh(e){let t=e6.get(e)??t6.get(e);return t?{lutId:t.lutId,per16:t.per16}:null}var Ot=[];function $c(e,t,r,n,a={}){let{nativeFormats:s=new Set,nativeSuffixes:o,routeSuffixes:i,fusedGroups:u=!0,decodeGroups:l,iqRawTrunkSuffixes:c}=a,d=u?l??tr:{},p=i??a6(),f=a.lutcSuffixes??o,m=s.has("lut32_compact")||s.has("lut16_compact")?f?new Set([...p].filter(L=>f.has(L))):p:void 0,h=new Map(e.map(L=>[L.id,L])),g=YS(s,e,t,r,n,o,a.subFourKSuperSuffixes,i,d,a.sourcePrecision??!1),{q6k:w,q5k:b,q3k:v,q2k:y,q4knc:S,iq4xs:C}=g,k=kh(e,t,r,n,d);for(let L of S)k.delete(L);let B=new Map(Ot.map(L=>[L.key,new Set])),P=new Set([...B.values()].flatMap(L=>[...L])),_=a.lut2Suffixes,x=m&&_?new Set([...m].filter(L=>_.has(L))):void 0,{lut32Ids:R,lut16Ids:O,lut32cIds:N,lut16cIds:Q,lut2_32Ids:V,lut2_16Ids:$}={lut32Ids:new Set,lut16Ids:new Set,lut32cIds:new Set,lut16cIds:new Set,lut2_32Ids:new Set,lut2_16Ids:new Set},M=[R,O,N,Q,V,$];for(let L of P)for(let K of M)K.delete(L);for(let L of Object.values(g))for(let K of L)for(let D of M)D.delete(K);let j=new Set(M.flatMap(L=>[...L])),A=xh(e,t,r,n,d),T=new Set,E=new Set([...k,...Object.values(g).flatMap(L=>[...L]),...j,...P]),X=[k,...M,T,...Object.values(g),...B.values()],z=L=>X.some(K=>K.has(L))||Mn(h.get(L)??{id:L,name:L,length:0},t,r,n,E,A);for(let L of e){let K=br(L.id),D=a.denseGroups?.[K]??Sh[K];if(!D)continue;let F=L.id.slice(0,L.id.length-K.length),H=D.map(Y=>F+Y).filter(Y=>h.has(Y));if(!(H.length===0||H.every(Y=>z(Y))))for(let Y of H){for(let fe of X)fe.delete(Y);j.delete(Y),P.delete(Y),E.delete(Y),A.add(Y)}}let G=Fc(e,t,r,n,E,A)??4;return s6({kq4:k,...g,lut32:R,lut16:O,lut32c:N,lut16c:Q,lut2_32:V,lut2_16:$,iq:T,...Object.fromEntries([...B].map(([L,K])=>[`iqRaw:${L}`,K]))}),{kq4Ids:k,kSuperIds:g,q6kIds:w,q5kIds:b,q3kIds:v,q2kIds:y,q4kncIds:S,iq4xsIds:C,lut32Ids:R,lut16Ids:O,lut32cIds:N,lut16cIds:Q,lut2_32Ids:V,lut2_16Ids:$,iqIds:T,iqRawIds:B,allIqRawIds:P,lutRoutedIds:j,forceDenseIds:A,excludeKq4Ids:E,quantBits:G}}function s6(e){let t=new Map;for(let[r,n]of Object.entries(e))for(let a of n){let s=t.get(a);if(s!==void 0)throw new Error(`resolveGGUFPackRoutes: ${a} is claimed by both the ${s} and ${r} packs, so it would be packed twice and cost its bytes twice. Subtract one route from the other.`);t.set(a,r)}}function o6(e,t){let r=e.filter(a=>a.transform&&t.has(a.id));if(r.length===0)return;let n=r.map(a=>a.id).join(", ");throw new Error(`GGUF packing would silently drop WeightEntry.transform for ${r.length} tensor(s): ${n}. Either exclude these ids from packSuffixes so they load dense, or apply the transform inside the packer.`)}async function Lh(e,{packSuffixes:t,...r}){let{entries:n,gguf:a,nameToGGUF:s}=e,{kq4Ids:o,q6kIds:i,q5kIds:u,q3kIds:l,q2kIds:c,q4kncIds:d,iq4xsIds:p,lut32Ids:f,lut16Ids:m,lut32cIds:h,lut16cIds:g,lut2_32Ids:w,lut2_16Ids:b,iqIds:v,iqRawIds:y,excludeKq4Ids:S,forceDenseIds:C}=$c(n,a,s,t,r),k=ys(n,r.decodeGroups);e={...e,shardUnitOf:x=>k.get(x)};let B=x=>{let R=r.scaleStorage?.[x];return R===void 0?void 0:R==="f16"},P=[],_=x=>(P.push(x),x);try{let x={q4:_(await Nc(e,{packSuffixes:t,excludeKq4Ids:S,forceDenseIds:C})),kq4:_(await zc(e,{kq4Ids:o,scalesF16:r.sourcePrecision?!1:B("kq4")})),q6k:_(await ya(e,"q6k",i,B("q6k"))),q5k:_(await ya(e,"q5k",u,B("q5k"))),q3k:_(await ya(e,"q3k",l,B("q3k"))),q2k:_(await ya(e,"q2k",c,B("q2k"))),q4knc:_(await ya(e,"q4knc",d,B("q4knc"))),iq4xs:_(await ya(e,"iq4xs",p,B("iq4xs"))),lut32:[],lut16:[],lut32c:[],lut16c:[],lut2_32:[],lut2_16:[],iq:[]};for(let O of Ot)x[O.key]=[];let R=new Set(Object.values(x).flatMap(O=>(Array.isArray(O)?O:[]).flatMap(N=>[...N.packedIds])));return o6(n,R),{packs:x,skipDenseIds:R}}catch(x){throw await cr(e.runtime,P),x}}function qh(e,t,r){return Ih(e,t,r,{kSuperRoutes:Uc,packKSuperGGUFWeights:Rh,packKq4GGUFWeights:zc,packQ8_0GGUFTable:$S})}function i6(e){let t=new Uint32Array(Math.ceil(e.byteLength/4));return new Uint8Array(t.buffer).set(e),t}async function Hc(e,t,r,n,a){if(r.length===0)return{tensors:[],locations:new Map};let s=await Nc({runtime:e,entries:[...r],gguf:t,nameToGGUF:i=>i,...a?{signal:a}:{}},{packSuffixes:n}),o=s.flatMap(i=>[i.bits,i.scales]);try{let i=new Map;for(let u of s){if(u.packedQuantBits!==8)throw new Error(`Q8_0 linears packed at ${u.packedQuantBits} bits`);for(let l of u.packedIds){let c=Gr(u.offsets,l);if(c===void 0)throw new Error(`Q8_0 pack lost the offset of ${l}`);i.set(l,{bits:u.bits,scales:u.scales,blockOffset:c})}}for(let u of r)if(!i.has(u.id))throw new Error(`Q8_0 linear ${u.name} was not packed`);return{tensors:o,locations:i}}catch(i){for(let u of o)u.destroy();throw i}}async function Wn(e,t,r,n=()=>!1){let a=new Map;try{await t.streamTensors(r,(s,o)=>{let i=t.info(s);a.set(s,pt(e,"float32",n(s)?i.shape:[i.elementCount],()=>Zr(i.type,o)))})}catch(s){for(let o of a.values())o.destroy();throw s}return a}async function Aa(e,t,r){let n=Object.keys(r),a=u=>t.info(`${r[u]}.weight`),s=n.filter(u=>a(u).type===I.Q8_0),o=await Hc(e,t,s.map(u=>({id:u,name:`${r[u]}.weight`,length:a(u).elementCount,shape:a(u).shape})),new Set(n)),i=[...o.tensors];try{let u=new Set(n.filter(p=>a(p).type===I.F16||a(p).type===I.BF16).map(p=>`${r[p]}.weight`)),l=n.flatMap(p=>[`${r[p]}.bias`,...s.includes(p)?[]:[`${r[p]}.weight`]]),c=new Map;await t.streamTensors(l,(p,f)=>{let m=t.info(p),h=u.has(p)?pt(e,"uint32",[Math.ceil(m.elementCount/2)],()=>i6(f)):pt(e,"float32",p.endsWith(".weight")?m.shape:[m.elementCount],()=>Zr(m.type,f));i.push(h),c.set(p,h)});let d={};for(let p of n){let[f,m]=a(p).shape,h={name:r[p],inFeatures:m,outFeatures:f,bias:c.get(`${r[p]}.bias`)},g=o.locations.get(p),w=a(p).type;d[p]=g?{...h,format:"q8_0",weight:g.bits,scales:g.scales,blockOffset:g.blockOffset}:{...h,format:w===I.BF16?"bf16":w===I.F16?"f16":"f32",weight:c.get(`${r[p]}.weight`)}}return{linears:d,tensors:i}}catch(u){for(let l of i)l.destroy();throw u}}var Oh=(e,t,r)=>e.op("ai.onnx.Gelu",{x:t},{attrs:{approximate:r==="tanh"?"tanh":"none"}}).y;function Dh(e,t,r,n,a,s,o,i){let u=r.shape[0],l={...s?{bT:s}:{},...o?{rT:o}:{}},c=e.scratch(`${n.name}.out`,"float32",[u,n.outFeatures]);if(n.format==="bf16"||n.format==="f16")return e.op("com.xenova.LlamaPrefillMatmul",{aT:r,bitsT:t(`${n.name}.weight`,n.weight),yT:c,...l},{args:{M:u,inFeatures:n.inFeatures,outFeatures:n.outFeatures,format:n.format,activation:i}}).yT;if(n.format!=="q8_0"&&n.format!=="q4_k"||!n.scales||n.blockOffset===void 0)throw new Error(`mmproj linear ${n.name}: ${n.format} is not a packed matmul format`);return e.op("com.xenova.LlamaPrefillMatmul",{aT:r,bitsT:t(`${n.name}.bits`,n.weight),scalesT:t(`${n.name}.scales`,n.scales),yT:c,...l},{args:{M:u,inFeatures:n.inFeatures,outFeatures:n.outFeatures,blockOffset:n.blockOffset,activation:i,format:n.format,exact:a?1:0,outStride:n.outFeatures,dstColStart:0}}).yT}function vr(e,t,r,n,{gelu:a,residual:s,exact:o=!0}={}){if(r.shape.length!==2||r.shape[1]!==n.inFeatures)throw new Error(`mmproj linear ${n.name} expects ${n.inFeatures} features, got [${r.shape}]`);let i=r.shape[0],u=n.bias?t(`${n.name}.bias`,n.bias):void 0;if(n.format==="f32"){if(!u)throw new Error(`mmproj linear ${n.name}: an f32 weight takes SupertonicDense, which needs a bias`);let p=e.op("com.xenova.SupertonicDense",{xT:r,wT:t(`${n.name}.weight`,n.weight),bT:u,...s?{rT:s}:{}},{args:{M:i,inFeatures:n.inFeatures,outFeatures:n.outFeatures,gelu:a==="erf"?1:0}}).yT;return a==="tanh"?Oh(e,p,"tanh"):p}let l=n.format==="bf16"||n.format==="f16"||n.format==="q8_0"&&o;if(i%64===0&&l)return Dh(e,t,r,n,o,u,s,a);let c=Dh(e,t,r,n,o);if(s){if(!u)return e.op("ai.onnx.Add",{a:c,b:s}).c;let p=[1,i,n.outFeatures],f=(m,h)=>e.view(m,0,m.dtype,h);return f(e.op("com.microsoft.BiasAdd",{X:f(c,p),bias:u,skip:f(s,p)}).Y,[i,n.outFeatures])}let d=u?e.op("ai.onnx.Add",{a:c,b:u}).c:c;return a?Oh(e,d,a):d}function rt(e,t,r="Config"){let n=e[t];if(typeof n!="number"||!Number.isInteger(n)||n<=0)throw new Error(`${r}.${t} must be a positive integer`);return n}function Vc(e,t,r="Config"){let n=e[t];if(n==null)throw new Error(`${r}.${t} is required`);return n}function Ti(e,t,r="Config"){for(let n of t)rt(e,n,r)}function u6(e){if(!(e.rope_theta===void 0&&e.rope_scaling==null))return{rope_type:e.rope_scaling?.rope_type??e.rope_scaling?.type??"default",rope_theta:e.rope_theta}}var nn=class{vocab_size;hidden_size;intermediate_size;num_hidden_layers;num_attention_heads;num_key_value_heads;hidden_act;max_position_embeddings;initializer_range;rms_norm_eps;use_cache;pad_token_id;bos_token_id;eos_token_id;pretraining_tp;tie_word_embeddings;rope_parameters;attention_bias;attention_dropout;mlp_bias;head_dim;model_type;constructor(t={}){this.vocab_size=rt(t,"vocab_size","LlamaConfig"),this.hidden_size=rt(t,"hidden_size","LlamaConfig"),this.intermediate_size=rt(t,"intermediate_size","LlamaConfig"),this.num_hidden_layers=rt(t,"num_hidden_layers","LlamaConfig"),this.num_attention_heads=rt(t,"num_attention_heads","LlamaConfig"),this.num_key_value_heads=t.num_key_value_heads??this.num_attention_heads,this.hidden_act=t.hidden_act,this.max_position_embeddings=rt(t,"max_position_embeddings","LlamaConfig"),this.initializer_range=t.initializer_range,this.rms_norm_eps=t.rms_norm_eps,this.use_cache=t.use_cache,this.pad_token_id=t.pad_token_id,this.bos_token_id=t.bos_token_id,this.eos_token_id=t.eos_token_id,this.pretraining_tp=t.pretraining_tp,this.tie_word_embeddings=t.tie_word_embeddings,this.rope_parameters=t.rope_parameters??u6(t),this.attention_bias=t.attention_bias,this.attention_dropout=t.attention_dropout??0,this.mlp_bias=t.mlp_bias,this.head_dim=t.head_dim??Math.floor(this.hidden_size/this.num_attention_heads),this.model_type="llama",this.validate()}validate(){if(l6(this,"LlamaConfig",["vocab_size","hidden_size","intermediate_size","num_hidden_layers","num_attention_heads","num_key_value_heads","max_position_embeddings","head_dim"]),this.hidden_act!=="silu")throw new Error(`Unsupported Llama hidden_act for V1: ${this.hidden_act}`);if(this.attention_bias||this.mlp_bias)throw new Error("Llama megakernel V1 does not support attention_bias or mlp_bias");if(this.attention_dropout!==0)throw new Error("Llama megakernel V1 only supports attention_dropout=0");if(this.rope_parameters?.rope_type!=="default")throw new Error("Llama megakernel V1 only supports default RoPE")}toJSON(){return{...this,rope_parameters:{...this.rope_parameters}}}};function Fn(e){if(!e||typeof e!="object")throw new Error("Llama-like config must be an object");if(e instanceof nn)return e;let t=e;if(t.model_type==="llama")return new nn(t);if(t.model_type){if(typeof t.validate!="function")throw new Error("Llama-like configs with model_type must be constructed by their Config class");return e}return new nn(t)}function _r(e){return Array.isArray(e.layer_types)&&e.layer_types.some(t=>t==="conv")}function rr(e){return Array.isArray(e.layer_types)&&e.layer_types.some(t=>t==="linear_attention")}function an(e){let t=e.layer_types,r=[];for(let n=0;n<e.num_hidden_layers;++n){let a=t?.[n];a!=="conv"&&a!=="linear_attention"&&r.push(n)}return r}function Mh(e){if(!Array.isArray(e.layer_types))throw new Error("linear attention config requires layer_types");return e.layer_types}function $e(e,t){if(typeof e!="number")throw new Error(`${t} must be a number`);return e}function Qn(e){let t=e.num_experts;return typeof t!="number"||t<=0?null:{numExperts:t,topK:$e(e.num_experts_per_tok,"config.num_experts_per_tok"),moeIntermediateSize:$e(e.moe_intermediate_size,"config.moe_intermediate_size"),numDenseLayers:$e(e.num_dense_layers,"config.num_dense_layers"),normTopkProb:e.norm_topk_prob!==!1,routedScale:typeof e.routed_scaling_factor=="number"?e.routed_scaling_factor:1,useExpertBias:!!e.use_expert_bias,sharedIntermediateSize:e.shared_expert_intermediate_size??0}}function Ci(e){return!e||e.sharedIntermediateSize<=0?null:{gate:"shared.gate_proj",up:"shared.up_proj",down:"shared.down_proj",width:e.sharedIntermediateSize,label:"shared_"}}function ka(e,t){return e!==null&&t>=e.numDenseLayers}function l6(e,t,r){if(Ti(e,r,t),e.hidden_size!==e.num_attention_heads*e.head_dim)throw new Error(`${t} hidden_size must equal num_attention_heads * head_dim for V1`);if(e.num_attention_heads%e.num_key_value_heads!==0)throw new Error(`${t} num_attention_heads must be divisible by num_key_value_heads`);if(e.head_dim%2!==0)throw new Error(`${t} head_dim must be even for RoPE`)}function Wh(e,t){let r=Number(e[`${t}.block_count`]),n=e[`${t}.attention.head_count_kv`],a=Array.isArray(n)?n.map(Number):Array.from({length:r},()=>Number(n));if(a.length!==r)throw new Error(`${t}.attention.head_count_kv length ${a.length} != block_count ${r}`);let s=a.map(i=>i>0?"full_attention":"conv"),o=Math.max(...a);if(!(o>0))throw new Error(`${t} GGUF has no attention layers (all head_count_kv are 0)`);return{blockCount:r,layer_types:s,num_key_value_heads:o}}function Fh(e,t){let r=e.hidden_size,n=e.num_attention_heads*e.head_dim,a=e.num_key_value_heads*e.head_dim,s=`L${t}.attn.qkv`;return[{id:`layers.${t}.q_proj`,name:`model.layers.${t}.self_attn.q_proj.weight`,length:n*r,packedGroup:s},{id:`layers.${t}.k_proj`,name:`model.layers.${t}.self_attn.k_proj.weight`,length:a*r,packedGroup:s},{id:`layers.${t}.v_proj`,name:`model.layers.${t}.self_attn.v_proj.weight`,length:a*r,packedGroup:s},{id:`layers.${t}.o_proj`,name:`model.layers.${t}.self_attn.out_proj.weight`,length:r*n},{id:`layers.${t}.q_layernorm`,name:`model.layers.${t}.self_attn.q_layernorm.weight`,length:e.head_dim},{id:`layers.${t}.k_layernorm`,name:`model.layers.${t}.self_attn.k_layernorm.weight`,length:e.head_dim}]}function Qh(e,t){let r=e.hidden_size;return[{id:`layers.${t}.conv_in_proj`,name:`model.layers.${t}.conv.in_proj.weight`,length:3*r*r},{id:`layers.${t}.conv_weight`,name:`model.layers.${t}.conv.conv.weight`,length:r*e.conv_L_cache},{id:`layers.${t}.conv_out_proj`,name:`model.layers.${t}.conv.out_proj.weight`,length:r*r}]}var sn=[64,128,256,512],Zc=512,Xc=64;function jn(e,t=sn){for(let r of t)if(e<=r)return r;return t[t.length-1]}function Nh(e){return e>0?[{kind:"replay",tokenCount:e}]:[]}var c6=48,d6=16;function zh(e,t,r){return r&&e===Xc&&t<=c6?Math.ceil(t/16)*16:e}var jh=new Map;function p6(e,t){let r=jh.get(t);if(r===void 0){let n=[0],a=[0],s=[0];for(let o=1;o<=Zc;++o){let i=jn(o),u={cost:zh(i,o,t),chunks:1,block:i};for(let l of[...sn].reverse()){if(l>=o)continue;let c=l+d6+n[o-l],d=1+a[o-l];(c<u.cost||c===u.cost&&d<u.chunks)&&(u={cost:c,chunks:d,block:l})}n.push(u.cost),a.push(u.chunks),s.push(u.block)}r=s,jh.set(t,r)}return r[e]}function Kh({tokenCount:e,position:t,maxLength:r,minGraphTokens:n,shortTail:a=!1,fewestRows:s=!1}){if(e<=0)return[];if(e<n||t+e>r)return Nh(e);let o=[],i=e,u=t;for(;i>0;){let l=s&&i<=Zc?p6(i,a):jn(i);if(u+l>r){o.push({kind:"replay",tokenCount:i});break}let c=Math.min(i,l),d=i<=l?zh(l,c,a):l;o.push({kind:"graph",blockLen:d,tokenCount:c}),i-=c,u+=c}return o}function Uh({tokenCount:e,position:t,maxLength:r,shortBlock:n,normalMinTokens:a,bucketMinTokens:s,bucketMaxTokens:o,bucketBlock:i,fitCache:u=!1}){if(e<=0)return[];if(n<=0||a<=n||s<=n||o<s||o>a||i<o)throw new Error(`invalid bucketed short-prefill geometry: short=${n}, normalMin=${a}, bucket=${s}..${o}->${i}`);if(e<n||t+e>r)return Nh(e);let l=[],c=e,d=t;for(;c>0;){if(c<n){l.push({kind:"replay",tokenCount:c});break}if(c>=s&&c<=o&&d+i<=r){l.push({kind:"graph",blockLen:i,tokenCount:c});break}let p=c<a?n:jn(c);if(u&&d+p>r&&(p=Math.max(0,...[n,i,...sn].filter(m=>m<=c&&d+m<=r))),p===0||d+p>r){l.push({kind:"replay",tokenCount:c});break}let f=Math.min(c,p);l.push({kind:"graph",blockLen:p,tokenCount:f}),c-=f,d+=f}return l}function Yc(e){let t=new Set;for(let r of e)r.kind==="graph"&&t.add(r.blockLen);return[...t].sort((r,n)=>r-n)}function Jc(e,t){let r=new Set,n=Math.min(t,2*Zc);for(let a=1;a<=n;++a)for(let s of Yc(e(a,0,t)))r.add(s);return[...r].sort((a,s)=>a-s)}function $h(e,t,r){if(!Number.isInteger(t)||t<0)throw new Error(`cache advance expects a non-negative integer, got ${t}`);let n=e+t;if(n>r)throw new Error(`cache advance overflows maxLength ${r}: ${e} + ${t}`);return n}function Hh(e,t,r){if(!Number.isInteger(t)||t<0||t>r)throw new Error(`cache truncate expects an integer in [0, ${r}], got ${t}`);if(t>e)throw new Error(`cache truncate cannot extend length from ${e} to ${t}; use advance()`);return t}var Vh=e=>e.gridHeight*e.gridWidth,Pi=e=>e.start+Vh(e),f6=e=>Vh(e)-Math.max(e.gridHeight,e.gridWidth),Ii=class{#e=[];get spans(){return this.#e}get isEmpty(){return this.#e.length===0}addImageSpan(t){for(let[n,a]of[["start",t.start],["gridHeight",t.gridHeight],["gridWidth",t.gridWidth]])if(!Number.isSafeInteger(a)||a<(n==="start"?0:1))throw new Error(`rope image span ${n} must be a ${n==="start"?"non-negative":"positive"} integer, got ${a}`);let r=this.#e.at(-1);if(r&&t.start<Pi(r))throw new Error(`rope image span at row ${t.start} overlaps the span ending at row ${Pi(r)}`);this.#e.push({start:t.start,gridHeight:t.gridHeight,gridWidth:t.gridWidth})}spanAt(t){for(let r of this.#e){if(t<r.start)return null;if(t<Pi(r))return r}return null}delta(t){let r=0;for(let n of this.#e){if(Pi(n)>t)break;r-=f6(n)}return r}fill(t,r,n){if(n.length<3*r)throw new Error(`rope positions need ${3*r} slots, got ${n.length}`);for(let a=0;a<r;++a){let s=t+a,o=this.spanAt(s);if(o===null){let l=s+this.delta(s);n[a]=l,n[r+a]=l,n[2*r+a]=l;continue}let i=o.start+this.delta(o.start),u=s-o.start;n[a]=i,n[r+a]=i+Math.floor(u/o.gridWidth),n[2*r+a]=i+u%o.gridWidth}return n}splits(t){let r=this.spanAt(t);return r!==null&&r.start!==t}boundaryAtMost(t){let r=this.spanAt(t);return r===null?t:r.start}truncate(t){if(this.splits(t))throw new Error(`cannot truncate rope positions to ${t}: it splits the image span starting at row ${this.spanAt(t).start}`);this.#e=this.#e.filter(r=>r.start<t)}clear(){this.#e=[]}};var Bi=256,ed=16*1024*1024,Zh=4;function m6(e,t){for(;t;)[e,t]=[t,e%t];return e}function nd(e,t,r,n){let a=t*r*n;if(a<=0)return e;let s=Bi/m6(a,Bi);return at(e,s)}function Ri(e,t){return at(e*t,Bi/4)}var g6=Math.max(sn[sn.length-1],2*Ir);function ad(e){if(!e.layer_types?.includes("sliding_attention"))return null;let t=$e(e.sliding_window,"config.sliding_window");return 2**Math.ceil(Math.log2(t+g6))}function sd(e,t){return e.num_hidden_layers*t*Ri(e.hidden_size,$e(e.conv_L_cache,"config.conv_L_cache"))}var Dt=class e{runtime;config;batchSize;maxLength;seqLength;kvLayers;keyStates;valueStates;ringLength;ringLayers;ringKeyStates;ringValueStates;convStates;linearConvStates;linearRecurrentStates;mutableStateCheckpoint;alternateMutableStateCheckpoint;linearConvStatesBacking=null;linearRecurrentStatesBacking=null;kvSlotByLayer;ringSlotByLayer;standaloneStateBuffersPadded=!1;mutableStateCheckpointStorage=null;mutableStateCheckpointPromise=null;alternateCheckpointStorage=null;alternateCheckpointPromise=null;stepCheckpointSlots=[];rewindPoints=[];rewindPointLimit=1;rewindPointClock=0;ropePositions=new Ii;constructor({runtime:t,config:r,batchSize:n,maxLength:a,keyStates:s,valueStates:o,ringLength:i=0,ringKeyStates:u=null,ringValueStates:l=null,convStates:c=null,linearConvStates:d=null,linearRecurrentStates:p=null,seqLength:f=0}){this.runtime=t,this.config=Fn(r),this.batchSize=n,this.maxLength=a,this.seqLength=f;let m=i>0?s1(this.config):[],h=an(this.config).filter(g=>!m.includes(g));if(i>0){if(m.length===0)throw new Error("a KV ring needs sliding_attention layers");if(!u||!l||u.shape[0]!==m.length||u.shape[2]!==i)throw new Error(`KV ring backing does not hold ${m.length} layer(s) of ${i} rows`)}if(this.ringLength=i,this.ringLayers=Object.freeze(m),this.ringKeyStates=u,this.ringValueStates=l,this.ringSlotByLayer=new Int32Array(this.config.num_hidden_layers).fill(-1),m.forEach((g,w)=>{this.ringSlotByLayer[g]=w}),s.shape[0]!==h.length)throw new Error(`KV cache backing has ${s.shape[0]} packed layer(s) but the config derives ${h.length} KV layer(s)`);this.kvSlotByLayer=new Int32Array(this.config.num_hidden_layers),this.kvSlotByLayer.fill(-1);for(let g=0;g<h.length;++g){let w=h[g];if(!Number.isInteger(w)||w<0||w>=this.config.num_hidden_layers)throw new Error(`KV cache layer ${w} is outside [0, ${this.config.num_hidden_layers})`);if(this.kvSlotByLayer[w]!==-1)throw new Error(`KV cache layer ${w} is duplicated`);this.kvSlotByLayer[w]=g}this.kvLayers=Object.freeze(h),this.keyStates=s,this.valueStates=o,this.convStates=c,this.linearConvStates=d,this.linearRecurrentStates=p,ks(this)&&(this.mutableStateCheckpoint=()=>this.mutableStateCheckpointPromise??=this.buildMutableStateCheckpoint(),this.alternateMutableStateCheckpoint=()=>this.alternateCheckpointPromise??=this.buildAlternateMutableStateCheckpoint())}static allocate(t,r,n,a,s="float32"){let o=Fn(r),i=nd(a??o.max_position_embeddings,o.num_key_value_heads,o.head_dim,Ae(Le(s))),u=ad(o),l=u!==null&&u<i?u:0,c=l>0?s1(o):[],p=[an(o).filter(g=>!c.includes(g)).length,n,i,o.num_key_value_heads,o.head_dim],f=[c.length,n,l,o.num_key_value_heads,o.head_dim],m=h6(t,o,n),h=new e({runtime:t,config:o,batchSize:n,maxLength:i,keyStates:t.empty(Le(s),p,"llama-cache-keys"),valueStates:t.empty(Le(s),p,"llama-cache-values"),...l>0?{ringLength:l,ringKeyStates:t.empty(Le(s),f,"llama-cache-ring-keys"),ringValueStates:t.empty(Le(s),f,"llama-cache-ring-values")}:{},convStates:_r(o)?t.empty("float32",[sd(o,n)],"llama-cache-conv"):null,linearConvStates:m.linearConvStates,linearRecurrentStates:m.linearRecurrentStates});return h.linearConvStatesBacking=m.linearConvStatesBacking,h.linearRecurrentStatesBacking=m.linearRecurrentStatesBacking,h.standaloneStateBuffersPadded=!0,h.clearMutableState(),h}kvSlotForLayer(t){if(!Number.isInteger(t)||t<0||t>=this.config.num_hidden_layers)throw new Error(`KV cache layer ${t} is outside [0, ${this.config.num_hidden_layers})`);let r=this.kvSlotByLayer[t];if(r<0)throw new Error(`logical layer ${t} does not own KV cache state`);return r}keyStateForLayer(t){return this.isRingLayer(t)?this.ringStateForLayer(this.ringKeyStates,t):this.kvStateForLayer(this.keyStates,t)}valueStateForLayer(t){return this.isRingLayer(t)?this.ringStateForLayer(this.ringValueStates,t):this.kvStateForLayer(this.valueStates,t)}isRingLayer(t){return this.ringLength>0&&this.ringSlotByLayer[t]>=0}kvRowsForLayer(t){return this.isRingLayer(t)?this.ringLength:this.maxLength}ringStateForLayer(t,r){let n=this.batchSize*this.ringLength*this.config.num_key_value_heads*this.config.head_dim;return Sa(t,[n],this.ringSlotByLayer[r]*n*Ae(t.dtype))}convStateForLayer(t){if(!this.convStates)throw new Error("cache has no conv state");let r=this.config.hidden_size,n=$e(this.config.conv_L_cache,"config.conv_L_cache"),a=Ri(r,n)*Ae(this.convStates.dtype);return Sa(this.convStates,[r*n],t*a)}kvStateForLayer(t,r){let n=this.kvSlotForLayer(r),a=this.batchSize*this.maxLength*this.config.num_key_value_heads*this.config.head_dim;return Sa(t,[a],n*a*Ae(t.dtype))}get_seq_length(){return this.seqLength}advance(t){this.seqLength=$h(this.seqLength,t,this.maxLength)}async stepCheckpoints(t){if(!ks(this))return[];for(;this.stepCheckpointSlots.length<t;)this.stepCheckpointSlots.push(await this.allocateCheckpointSlot());return this.stepCheckpointSlots.map(r=>r.checkpoint)}async buildMutableStateCheckpoint(){let t=await this.allocateCheckpointSlot();return this.mutableStateCheckpointStorage=t.storage,t.checkpoint}async buildAlternateMutableStateCheckpoint(){let t=await this.allocateCheckpointSlot();return this.alternateCheckpointStorage=t.storage,t.checkpoint}async allocateCheckpointSlot(){let t=this.mutableStateSource(),r=v6(this.runtime,t);try{let n=await this.runtime.prepareProgramSequence(r1(t,r,"capture")),a=await this.runtime.prepareProgramSequence(r1(r,t,"restore"));return{storage:r,checkpoint:{capture:()=>this.runtime.enqueuePreparedProgramSequence(n),restore:()=>this.runtime.enqueuePreparedProgramSequence(a)}}}catch(n){throw Ss(r),n}}async captureRewindPoint(){if(!this.rewindsNeedPoints())return;let t=this.rewindPointAt(this.seqLength);if(t){t.lastUsed=++this.rewindPointClock;return}let r=this.rewindPoints.find(n=>n.length<0);!r&&this.rewindPoints.length<this.rewindPointLimit&&(r={...ks(this)?await this.allocateCheckpointSlot():{storage:null,checkpoint:null},ringWindow:this.allocateRingWindow(),length:-1,lastUsed:0},this.rewindPoints.push(r)),r??=this.rewindPointToEvict(),r.checkpoint?.capture(),this.copyRingWindow(r.ringWindow,this.seqLength,"capture"),r.length=this.seqLength,r.lastUsed=++this.rewindPointClock}rewindsNeedPoints(){return ks(this)||this.ringLength>0}allocateRingWindow(){if(this.ringLength===0)return null;let t=$e(this.config.sliding_window,"config.sliding_window"),r=[this.ringLayers.length,this.batchSize,t,this.config.num_key_value_heads,this.config.head_dim],n=Le(this.ringKeyStates.dtype);return{keys:this.runtime.empty(n,r,"llama-cache-ring-window-keys"),values:this.runtime.empty(n,r,"llama-cache-ring-window-values")}}copyRingWindow(t,r,n){if(!t||r<=0)return;let a=t.keys.shape[2],s=Math.min(a,r),o=(r-s)%this.ringLength,i=this.batchSize*this.config.num_key_value_heads*this.config.head_dim*Ae(t.keys.dtype),u=[[o,0,Math.min(s,this.ringLength-o)]];u[0][2]<s&&u.push([0,u[0][2],s-u[0][2]]);for(let[l,c]of[[this.ringKeyStates,t.keys],[this.ringValueStates,t.values]])for(let d=0;d<this.ringLayers.length;++d){let p=(l.byteOffset??0)+d*this.ringLength*i,f=(c.byteOffset??0)+d*a*i;for(let[m,h,g]of u){let w={tensor:l,offset:p+m*i},b={tensor:c,offset:f+h*i},[v,y]=n==="capture"?[w,b]:[b,w];this.runtime.copyBufferToBuffer({src:v.tensor,dst:y.tensor,srcOffset:v.offset,dstOffset:y.offset,byteLength:g*i})}}}get rewindPointCapacity(){return this.rewindPointLimit}setRewindPointCapacity(t){if(!Number.isInteger(t)||t<1)throw new Error(`rewind point capacity must be a positive integer, got ${t}`);for(this.rewindPointLimit=t;this.rewindPoints.length>t;){let r=this.rewindPoints.find(n=>n.length<0)??this.rewindPointToEvict();this.rewindPoints.splice(this.rewindPoints.indexOf(r),1),a1(r)}}rewindPointAt(t){return this.rewindPoints.find(r=>r.length===t)}rewindPointToEvict(){let t=this.rewindPoints.filter(a=>a.length>=0);if(t.length===0)return;let r=t.reduce((a,s)=>s.length<a.length?s:a);return(t.length>1?t.filter(a=>a!==r):t).reduce((a,s)=>s.lastUsed<a.lastUsed?s:a)}dropRewindPoints(t){for(let r of this.rewindPoints)r.length>t&&(r.length=-1)}prefixSnapshotPlan(t){if(this.batchSize!==1||!Number.isInteger(t)||t<=0||t>this.maxLength)return null;let r=this.config.num_key_value_heads*this.config.head_dim,n=Ae(Le(this.keyStates.dtype)),a=t*r*n,s=this.maxLength*r*n;if(a%4!==0)return null;let o=[];for(let u of[this.keyStates,this.valueStates]){if((u.byteOffset??0)!==0)return null;for(let l=0;l<this.kvLayers.length;++l)o.push({base:u,byteOffset:l*s,byteLength:a})}let i=Math.min(t,this.ringLength)*r*n;for(let u of[this.ringKeyStates,this.ringValueStates])if(!(!u||i===0)){if((u.byteOffset??0)!==0)return null;for(let l=0;l<this.ringLayers.length;++l)o.push({base:u,byteOffset:l*this.ringLength*r*n,byteLength:i})}for(let u of Yh(this.mutableStateSource())){let l=u.dst,d=(u.dstOffset??0)-(l.byteOffset??0);if(d<0||d+u.byteLength>l.byteLength)return null;o.push({base:l,byteOffset:d,byteLength:u.byteLength})}return o}prefixSnapshotChunks(t){let r=this.prefixSnapshotPlan(t);if(!r)return null;let n=[];for(let a of r)for(let s=0;s<a.byteLength;s+=ed)n.push({base:a.base,byteOffset:a.byteOffset+s,byteLength:Math.min(ed,a.byteLength-s)});return n}prefixSnapshotShapes(){let t=[];for(let a of[this.keyStates,this.valueStates]){let[s,o,,i,u]=a.shape;t.push(`${[s,o,i,u].join("x")}@${a.dtype}`)}for(let a of[this.ringKeyStates,this.ringValueStates])a&&t.push(`ring${a.shape.join("x")}@${a.dtype}`);let r=[this.convStates,...this.linearConvStates??[],...this.linearRecurrentStates??[]];for(let a of r)a&&t.push(`${a.shape.join("x")}@${a.dtype}`);let n=[];for(let a of t){let s=n[n.length-1];s&&s.shape===a?s.count+=1:n.push({shape:a,count:1})}return n.map(({shape:a,count:s})=>s>1?`${s}x[${a}]`:`[${a}]`).join(",")}prefixSnapshotLayout(t){let r=this.prefixSnapshotChunks(t);if(!r)return null;let n=r.map(a=>`${a.byteLength}@${a.base.dtype}`).join(",");return`v1|len=${t}|chunk=${ed}|shapes=${this.prefixSnapshotShapes()}|bytes=${n}`}async exportPrefixSnapshot(){let t=this.seqLength,r=this.prefixSnapshotLayout(t),n=this.prefixSnapshotChunks(t);if(!r||!n||this.rewindsNeedPoints()&&!this.rewindPointAt(t)||!this.ropePositions.isEmpty)return null;let a=[];for(let s=0;s<n.length;s+=Zh){let o=n.slice(s,s+Zh).map(i=>this.runtime.readTensor(b6(i)));for(let i of o)a.push(w6(_n(await i)))}return{length:t,layout:r,chunks:a}}async importPrefixSnapshot(t){let r=this.prefixSnapshotChunks(t.length),n=this.prefixSnapshotLayout(t.length);if(!r||!n||n!==t.layout||r.length!==t.chunks.length)return!1;for(let a=0;a<r.length;++a)if(r[a].byteLength!==t.chunks[a].byteLength)return!1;for(let a=0;a<r.length;++a)this.runtime.writeWeightsRange(r[a].base,r[a].byteOffset,t.chunks[a]);return this.dropRewindPoints(-1),this.seqLength=t.length,this.ropePositions.clear(),await this.captureRewindPoint(),!0}rewindableLengthAtMost(t){if(t<=0)return 0;if(!this.rewindsNeedPoints()||t>=this.seqLength)return this.ropePositions.boundaryAtMost(Math.min(t,this.seqLength));let r=0;for(let n of this.rewindPoints)n.length>r&&n.length<=t&&!this.ropePositions.splits(n.length)&&(r=n.length);return r}mutableStateSource(){return{convStates:this.convStates,linearConvStates:this.linearConvStates,linearRecurrentStates:this.linearRecurrentStates,linearConvStatesBacking:this.linearConvStatesBacking,linearRecurrentStatesBacking:this.linearRecurrentStatesBacking,standaloneStateBuffersPadded:this.standaloneStateBuffersPadded}}clearMutableState(){let t=Yh(this.mutableStateSource());t.length>0&&this.runtime.clearBuffers(t)}canTruncateTo(t){return t!==this.seqLength&&this.ropePositions.splits(t)?!1:t===0||t===this.seqLength||!this.rewindsNeedPoints()||this.rewindPointAt(t)!==void 0}truncate(t){if(Hh(this.seqLength,t,this.maxLength),!this.canTruncateTo(t)){if(t!==this.seqLength&&this.ropePositions.splits(t))throw new Error(`cache cannot rewind from ${this.seqLength} to ${t}: it would split a cached image span`);let r=ks(this)?"mutable conv/recurrent state":"KV rings";throw new Error(`cache with ${r} cannot rewind from ${this.seqLength} to ${t}; reset to 0 instead`)}if(this.ropePositions.truncate(t),t===0)this.clearMutableState(),this.dropRewindPoints(-1);else if(t!==this.seqLength){let r=this.rewindPointAt(t);r&&(r.checkpoint?.restore(),this.copyRingWindow(r.ringWindow,t,"restore"),r.lastUsed=++this.rewindPointClock),this.dropRewindPoints(t)}this.seqLength=t}dispose(){Ss(this.mutableStateCheckpointStorage),Ss(this.alternateCheckpointStorage);for(let t of this.stepCheckpointSlots.splice(0))Ss(t.storage);for(let t of this.rewindPoints.splice(0))a1(t);this.ropePositions.clear(),ve(this.keyStates),ve(this.valueStates),ve(this.ringKeyStates),ve(this.ringValueStates),this.ringKeyStates=null,this.ringValueStates=null,ve(this.convStates),Ei(this.linearConvStates,this.linearConvStatesBacking),Ei(this.linearRecurrentStates,this.linearRecurrentStatesBacking),this.convStates=null,this.linearConvStates=null,this.linearRecurrentStates=null,this.linearConvStatesBacking=null,this.linearRecurrentStatesBacking=null,this.standaloneStateBuffersPadded=!1,this.mutableStateCheckpointStorage=null,this.mutableStateCheckpointPromise=null,this.alternateCheckpointStorage=null,this.alternateCheckpointPromise=null}};function h6(e,t,r){return rr(t)?Pn("linear-attention layers"):{linearConvStates:null,linearRecurrentStates:null,linearConvStatesBacking:null,linearRecurrentStatesBacking:null}}function Xh(e,t,r,n,a){let s=t.reduce((h,g)=>h+(g==="linear_attention"?1:0),0);if(s===0)return{states:Array(t.length).fill(null),backing:null};let o=n.reduce((h,g)=>h*g,1),i=Ae(Le(r)),u=o*i,l=at(u,Bi),c=s*l,d=e.device.limits.maxBufferSize;if(Number.isFinite(d)&&d>0&&c>d)return{states:t.map((g,w)=>g==="linear_attention"?e.empty(Le(r),n,`${a}-${w}`):null),backing:null};let p=e.empty(Le(r),[c/i],a),f=0;return{states:t.map(h=>{if(h!=="linear_attention")return null;let g=Sa(p,n,f*l);return f+=1,g}),backing:p}}function Sa(e,t,r){let n=t.reduce((s,o)=>s*o,1),a=n*Ae(e.dtype);if(r<0||r+a>e.byteLength)throw new Error("mutable-state tensor view exceeds its backing buffer");return{dtype:e.dtype,shape:[...t],byteOffset:(e.byteOffset??0)+r,size:n,byteLength:a,buffer:e.buffer,runtime:e.runtime,destroy:vn}}function w6(e){let t=e;if(!t.buffer||t.byteLength===void 0)throw new Error("a prefix snapshot needs the bytes behind readTensor's result, but it returned no buffer");return new Uint8Array(t.buffer,t.byteOffset??0,t.byteLength)}function b6(e){let t=Ae(Le(e.base.dtype));return Sa(e.base,[e.byteLength/t],e.byteOffset)}function Yh(e){let t=[];return td(t,e.convStates,e.standaloneStateBuffersPadded),Jh(t,e.linearConvStates,e.linearConvStatesBacking,e.standaloneStateBuffersPadded),Jh(t,e.linearRecurrentStates,e.linearRecurrentStatesBacking,e.standaloneStateBuffersPadded),t}function Jh(e,t,r,n){if(r){td(e,r,!0);return}if(Array.isArray(t))for(let a of t)td(e,a,n)}function td(e,t,r){t&&e.push({dst:t,dstOffset:t.byteOffset??0,byteLength:o1(t,r,"clear")})}function o1(e,t,r){if(e.byteLength%4===0)return e.byteLength;if(!t)throw new Error(`cannot ${r} ${e.byteLength}-byte mutable-state tensor safely: its backing allocation is not known to include 4-byte padding`);return at(e.byteLength,4)}function ks(e){return e.convStates!==null||e1(e.linearConvStates)||e1(e.linearRecurrentStates)}function e1(e){return Array.isArray(e)&&e.some(t=>t!==null)}function v6(e,t){let r=t1(e,t.linearConvStates,t.linearConvStatesBacking,"llama-cache-checkpoint-linear-conv"),n=t1(e,t.linearRecurrentStates,t.linearRecurrentStatesBacking,"llama-cache-checkpoint-linear-recurrent");return{convStates:i1(e,t.convStates,"llama-cache-checkpoint-conv"),linearConvStates:r.states,linearRecurrentStates:n.states,linearConvStatesBacking:r.backing,linearRecurrentStatesBacking:n.backing,standaloneStateBuffersPadded:!0}}function i1(e,t,r){return t?e.empty(t.dtype,t.shape,r):null}function t1(e,t,r,n){if(!Array.isArray(t)){if(r!==null)throw new Error("mutable-state backing has no tensor views");return{states:null,backing:null}}if(!r)return{states:t.map((i,u)=>i1(e,i,`${n}-${u}`)),backing:null};let a=e.empty(r.dtype,r.shape,n),s=r.byteOffset??0;return{states:t.map(i=>{if(!i)return null;if(i.buffer!==r.buffer||i.dtype!==r.dtype)throw new Error("mutable-state tensor views do not match their backing buffer");return Sa(a,i.shape,(i.byteOffset??0)-s)}),backing:a}}function r1(e,t,r){let n=[],a=e.standaloneStateBuffersPadded&&t.standaloneStateBuffersPadded;return rd(n,e.convStates,t.convStates,`llama-cache-${r}-conv`,a),n1(n,e.linearConvStates,t.linearConvStates,e.linearConvStatesBacking,t.linearConvStatesBacking,`llama-cache-${r}-linear-conv`,a),n1(n,e.linearRecurrentStates,t.linearRecurrentStates,e.linearRecurrentStatesBacking,t.linearRecurrentStatesBacking,`llama-cache-${r}-linear-recurrent`,a),n}function n1(e,t,r,n,a,s,o){if(n||a){if(!n||!a)throw new Error("mutable-state checkpoint packed backings do not match");rd(e,n,a,s,!0);return}if(!Array.isArray(t)||!Array.isArray(r)||t.length!==r.length){if(t===null&&r===null)return;throw new Error("mutable-state checkpoint tensor arrays do not match")}for(let i=0;i<t.length;++i)rd(e,t[i],r[i],`${s}-${i}`,o)}function rd(e,t,r,n,a){if(!t||!r){if(t===null&&r===null)return;throw new Error("mutable-state checkpoint tensors do not match")}if(t.dtype!==r.dtype||t.byteLength!==r.byteLength)throw new Error("mutable-state checkpoint tensor shape or dtype changed");let s=o1(t,a,"copy");e.push({kind:"copy",name:n,src:t,dst:r,srcOffset:t.byteOffset??0,dstOffset:r.byteOffset??0,byteLength:s})}function a1(e){Ss(e.storage),e.ringWindow&&ve([e.ringWindow.keys,e.ringWindow.values])}function s1(e){return(e.layer_types??[]).flatMap((r,n)=>r==="sliding_attention"?[n]:[])}function Ss(e){e&&(ve(e.convStates),Ei(e.linearConvStates,e.linearConvStatesBacking),Ei(e.linearRecurrentStates,e.linearRecurrentStatesBacking))}function Ei(e,t){if(t){ve(t);return}ve(e)}var u1=32,_6=["q6k","q5k","q3k","q2k","q4knc","iq4xs"],y6=["lut32","lut16","lut32c","lut16c","lut2_32","lut2_16","lut2_128"],A6=["kq4","iq"],k6=Ot.map(e=>e.key),od=[...A6,..._6,...y6,...k6],xs=2048,S6=[16,32,...sn];function x6(){return{}}function T6(e,t){return{device:e,recordSupportOp:t,empty:(r,n)=>he(Le(r),n),tensorFromTypedArray:(r,n)=>he(Le(r),n),createUniformU32:()=>x6(),host:{writeBuffer:()=>{}},clearBuffers:()=>{}}}function C6(e,t,r,n,a,s,o,i,u,l=()=>null){let c=new Set,d=[];for(let k of e)if(k.id==="lm_head"&&n)continue;let p={locations:new Map,shardBlockCounts:[]},f=new Map(p.locations),m=Jt(),h=Jt(),g=new Set,w=Object.fromEntries(od.map(k=>[k,{offsets:Jt(),blockCounts:Jt(),packedIds:new Set,totalBlocks:0}])),b=0;for(let k of e){if(c.has(k.id)||!t(k))continue;let B=k.length/u1,P=l(k);if(P){let _=w[P];_t(_.offsets,k.id,_.totalBlocks),_t(_.blockCounts,k.id,B),_.packedIds.add(k.id),_.totalBlocks+=B;continue}_t(m,k.id,b),_t(h,k.id,B),g.add(k.id),b+=B}let v=Jt(),y=[],S=null;for(let k of e)if(!c.has(k.id)&&!g.has(k.id)&&!od.some(B=>w[B].packedIds.has(k.id))&&!(a&&k.id==="embed_tokens")&&!(s&&!n&&k.id==="lm_head")){if(k.id==="lm_head"&&n){if(c.has("embed_tokens")){let B=f.get("embed_tokens");if(!B)throw new Error("Q1 support layout is missing tied embed_tokens");f.set(k.id,B)}else S=k.id;continue}y.push({id:k.id,length:k.length})}let C=pi({entries:y,elementBytes:i,limits:u});for(let k of y)_t(v,k.id,C.placements.get(k.id).elementOffset);return S&&_t(v,S,v.top.embed_tokens??0),{offsets:v,q4Offsets:m,q4BlockCounts:h,totalBlocks:b,densePlan:C,q1Locations:f,q1PackedIds:c,kSuper:w,q1ShardBlockCounts:p.shardBlockCounts,maxQ1SourceBlocks:d.reduce((k,B)=>Math.max(k,B.blockCount),0)}}function P6({config:e,entries:t,isPacked:r=()=>!1,isQ1:n=()=>!1,packedQuantBits:a=4,packSlots:s,decodeGroups:o,tieWordEmbeddings:i,canUseFloat16:u=Lr,keepBf16:l=!1,canUseQ8LmHead:c=Ec,canUseQ4LmHead:d=vs,embedHeadRoutes:p,emitGraph:f,emitPrefillGraph:m,prefillBlockLengths:h=S6,device:g}){let w=Ue(g),b=w.features.has("shader-f16"),v=!l&&b&&u(e),y=l?"uint32":v?"float16":"float32",S=l?2:Ae(y),C=v?"float16":"float32",k=b?"float16":"float32",B=!1,P=t.some(oe=>oe.id==="lm_head"),_=P&&!B&&v&&d(e),x=P&&!B&&v&&!_&&c(e),R=B?null:p?.embedFormat??null,O=!B&&(p?.headPacked??!1),N=O?p?.headFormat??null:null,Q=O?p?.headLut2??null:null,V=R==="lut2_32"?p?.embedLut2??null:null,$=(N!==null||Q!==null)&&(p?.headAliasesEmbed??!1),M=oe=>{let ue=s?.(oe)?.buffer;return od.find(We=>We===ue)??null},{offsets:j,q4Offsets:A,q4BlockCounts:T,totalBlocks:E,densePlan:X,q1Locations:z,q1PackedIds:G,kSuper:L,q1ShardBlockCounts:K,maxQ1SourceBlocks:D}=C6(t,r,n,i,R!==null,O,Rc(w.limits),S,w.limits,M),F=ys(t,o),H=[],Y=T6(w,(oe,ue)=>H.push({op:oe,request:ue})),fe=X.shardElementCounts.map(oe=>he(y,[Math.max(1,l?oe/2:oe)])),ce=Object.fromEntries(X.shardIndexByUnit),re=fe[0],Z=E>0?{bits:he("uint32",[E*a]),scales:he(k,[E]),offsets:A,blockCounts:T,packedIds:new Set(t.filter(oe=>r(oe)&&!G.has(oe.id)&&!M(oe)).map(oe=>oe.id)),packedQuantBits:a}:null,U=oe=>{let ue=L[oe],We=t.find(Fe=>ue.packedIds.has(Fe.id));if(!We)return null;let ge=s(We);return{bits:he("uint32",[ue.totalBlocks*ge.wordsPerBlock]),scales:he(ge.metadataU32?"uint32":"float32",[ue.totalBlocks*ge.scalesPerBlock]),offsets:ue.offsets,blockCounts:ue.blockCounts,packedIds:ue.packedIds,u32PerSuper:ge.wordsPerBlock*8,scalesPerSuper:ge.scalesPerBlock*8}},te=oe=>{let ue=L[oe],We=t.find(Fe=>ue.packedIds.has(Fe.id));if(!We)return null;let ge=s(We);return{bits:he("uint32",[ue.totalBlocks*ge.wordsPerBlock]),scales:he(ge.metadataU32?"uint32":ge.scalesF32?"float32":k,[ue.totalBlocks*ge.scalesPerBlock]),offsets:ue.offsets,blockCounts:ue.blockCounts,packedIds:ue.packedIds}},ae=oe=>{let ue=L[oe];if(!ue)return null;let We=t.find(ir=>ue.packedIds.has(ir.id));if(!We)return null;let ge=s(We),Fe=Math.ceil(ue.totalBlocks*ge.wordsPerBlock),kr=Ot.find(ir=>ir.key===oe)?.gridInBits===!0;return{bits:he("uint32",[kr?Fe+4096:Fe]),scales:he(k,[Math.ceil(ue.totalBlocks*ge.scalesPerBlock)]),grid:he("uint32",[512]),offsets:ue.offsets,blockCounts:ue.blockCounts,packedIds:ue.packedIds,...kr?{gridWordBase:Fe}:{}}},ne=oe=>{let ue=L[oe],We=t.find(ur=>ue.packedIds.has(ur.id));if(!We)return null;let ge=s(We),Fe=new Map;for(let ur of t)ue.packedIds.has(ur.id)&&Fe.set(ur.id,s(ur)?.lut??0);let kr=oe==="lut16"||oe==="lut16c"||oe==="lut2_16",ir=oe==="lut2_32"||oe==="lut2_16"||oe==="lut2_128",jt=oe==="lut32c"||oe==="lut16c"||ir;return{bits:he("uint32",[Math.ceil(ue.totalBlocks*ge.wordsPerBlock)]),scales:he(ge.metadataU32?"uint32":"float32",[Math.ceil(ue.totalBlocks*ge.scalesPerBlock)]),offsets:ue.offsets,blockCounts:ue.blockCounts,packedIds:ue.packedIds,lutIds:Fe,lut16:kr?1:0,...jt?{compact:1}:{},...ir?{lut2:1}:{}}},xe=ne("lut32"),De=ne("lut16"),Me=ne("lut32c"),ye=ne("lut16c"),Ze=ne("lut2_32"),W=ne("lut2_128"),J=ne("lut2_16"),ie=te("kq4"),pe=te("iq"),me=U("q6k"),se=U("q5k"),Te=U("q3k"),Ce=U("q2k"),He=U("q4knc"),_e=U("iq4xs"),Ge=null,ze=N==="q8_rows",Ft=O&&!ze&&(p?.headRescoreQ8??!1),yr=ze?null:N,mt=N?As(N,e.vocab_size,e.hidden_size,b):null,sr=Ft&&(p?.headScan2??!1),gt=!ze&&mt!==null&&(p?.headKSuperScan??!1),Xe=e.vocab_size*(e.hidden_size/32),ht=Q===null?0:Q===9?1:3,dn=ze?null:Q!==null?he("uint32",[Xe*2]):sr||gt?he("uint32",[Xe*2]):mt?he("uint32",[mt.bits]):_||O?he("uint32",[Xe*4]):null,nt=ze?null:Q!==null?he("uint32",[Math.ceil(Xe/8)*ht]):sr||gt?he("uint32",[Xe]):mt?he(mt.scalesDtype,[mt.scales]):_||O?he(k,[Xe]):null,Ar=ze||Ft?As("q8_rows",e.vocab_size,e.hidden_size,b):null,Mr=gt?he("uint32",[mt.bits]):Ar?he("uint32",[Ar.bits]):x&&!O?he("uint32",[e.vocab_size*(e.hidden_size/4)]):null,Wr=gt?he("float32",[mt.scales]):Ar?he("float32",[Ar.scales]):x&&!O?he("float32",[e.vocab_size]):null,Pt=R?As(R,e.vocab_size,e.hidden_size,b,V):null,pn=Pt?he("uint32",[Pt.bits]):null,kt=Pt?he(Pt.scalesDtype,[Pt.scales]):null,or={runtime:Y,config:e,weights:re,offsets:j,denseShards:fe,denseShardIndex:ce,packs:{q1:Ge,q4:Z?[Z]:[],kq4:ie?[ie]:[],iq:pe?[pe]:[],q6k:me?[me]:[],q5k:se?[se]:[],q3k:Te?[Te]:[],q2k:Ce?[Ce]:[],q4knc:He?[He]:[],iq4xs:_e?[_e]:[],lut32:xe?[xe]:[],lut16:De?[De]:[],lut32c:Me?[Me]:[],lut16c:ye?[ye]:[],lut2_32:[...Ze?[Ze]:[],...W?[W]:[]],lut2_16:J?[J]:[],...Object.fromEntries(Ot.map(({key:oe})=>{let ue=ae(oe);return[oe,ue?[ue]:[]]}))},lmHeadQ4:dn,lmHeadQ4Scales:nt,lmHeadQ8:Mr,lmHeadQ8Scales:Wr,embedBits:pn,embedScales:kt,embedFormat:R,embedLut:V,lmHeadFormat:gt?null:yr,lmHeadLut:Q,lmHeadScanBits:sr||gt?2:null,lmHeadRescoreFormat:gt?yr:null,cacheDtype:C,programCache:new Map,defaultPositionIdsCache:new Map,ropeCache:{cos:he("float32",[xs,e.head_dim>>1]),sin:he("float32",[xs,e.head_dim>>1])},partialRopeCache:{cos:he("float32",[xs,(e.rotary_dim??e.head_dim)>>1]),sin:he("float32",[xs,(e.rotary_dim??e.head_dim)>>1])}},Zn=Dt.allocate(Y,e,1,xs,C),fn=f(or,Zn);if(en(fn.graph,Y,{weights:fn.weights,states:fn.states}),m)for(let oe of h){let ue=m(or,Zn,oe);if(!ue)break;en(ue.graph,Y,{weights:ue.weights,states:ue.states,...ue.inputs?{inputs:ue.inputs}:{}})}let U3=H.map(({op:oe,request:ue})=>({op:oe,request:ue,id:oe.manifest.id})),Qt=fe.map((oe,ue)=>({name:fe.length===1?"weights":`weights.${ue}`,dtype:y,shape:oe.shape,boundWhole:!0}));if(s){let oe=new Map;for(let We of t){let ge=s(We);if(!ge)continue;let Fe=oe.get(ge.buffer)??{wordsPerBlock:ge.wordsPerBlock,scalesPerBlock:ge.scalesPerBlock,scalesF32:!!ge.scalesF32,metadataU32:!!ge.metadataU32,entries:[]};if(Fe.wordsPerBlock!==ge.wordsPerBlock||Fe.scalesPerBlock!==ge.scalesPerBlock)throw new Error(`pack buffer ${ge.buffer} is priced two ways: ${Fe.wordsPerBlock}/${Fe.scalesPerBlock} per block and ${ge.wordsPerBlock}/${ge.scalesPerBlock} at ${We.id}`);let kr=F.get(We.id);Fe.entries.push({id:We.id,blockCount:We.length/u1,...kr===void 0?{}:{unitKey:kr}}),Fe.scalesF32=Fe.scalesF32||!!ge.scalesF32,Fe.metadataU32=Fe.metadataU32||!!ge.metadataU32,oe.set(ge.buffer,Fe)}let ue=wr(w.limits);for(let[We,ge]of oe){let Fe=ge.metadataU32?"uint32":ge.scalesF32?"float32":k,kr=Ae(Fe),ir=On(ge.entries,qn({byteLimit:ue,maxBlocks:ge.entries.reduce((jt,ur)=>jt+ur.blockCount,0),bytesForBlocks:jt=>Math.max(jt*ge.wordsPerBlock*Uint32Array.BYTES_PER_ELEMENT,Math.ceil(jt*ge.scalesPerBlock)*kr)}),We);for(let jt=0;jt<ir.shardBlockCounts.length;++jt){let ur=ir.shardBlockCounts.length===1?"":`.${jt}`,Zd=ir.shardBlockCounts[jt];Qt.push({name:`${We}Bits${ur}`,dtype:"uint32",shape:[Zd*ge.wordsPerBlock],boundWhole:!0}),Qt.push({name:`${We}Scales${ur}`,dtype:Fe,shape:[Math.ceil(Zd*ge.scalesPerBlock)],boundWhole:!0})}}}else Z&&(Qt.push({name:"q4Bits",dtype:Z.bits.dtype,shape:Z.bits.shape}),Qt.push({name:"q4Scales",dtype:Z.scales.dtype,shape:Z.scales.shape}));let Hd=$&&!gt;dn&&!Hd&&Qt.push({name:"lmHeadQ4",dtype:dn.dtype,shape:dn.shape,boundWhole:!0}),nt&&!Hd&&Qt.push({name:"lmHeadQ4Scales",dtype:nt.dtype,shape:nt.shape,boundWhole:!0});let Vd=Ft||$;return Mr&&!Vd&&Qt.push({name:"lmHeadQ8",dtype:Mr.dtype,shape:Mr.shape,boundWhole:!0}),Wr&&!Vd&&Qt.push({name:"lmHeadQ8Scales",dtype:Wr.dtype,shape:Wr.shape,boundWhole:!0}),pn&&Qt.push({name:"embedBits",dtype:pn.dtype,shape:pn.shape,boundWhole:!0}),kt&&Qt.push({name:"embedScales",dtype:kt.dtype,shape:kt.shape,boundWhole:!0}),{opRequests:U3,tensors:Qt.map(oe=>oe.name==="q1TranscodeSource"?oe:{...oe,allocation:!0})}}function id({config:e,device:t,...r}){return hs(P6({config:e,device:t,...r}),t)}function Li(e,t){return{cacheKeysT:e.keyStateForLayer(t),cacheValuesT:e.valueStateForLayer(t)}}function I6(e,t,r){return E6(e,r.map(n=>`layers.${t}.${n}`))}function qi(e,t){let r=B6(e,t);return Object.assign((a,s,o)=>{let i=r(a,s,o);if(i.format==="iq4_xs")throw new Error(`layer ${a} ${s.join(", ")}: iq4_xs is an expert-only pack`);return{...i,format:i.format}},{experts:r})}function B6(e,t){return(r,n,a)=>{let s=I6(t,r,n);return{bitsT:e(`L${r}.q4bits.${a}`,s.q4BitsT),scalesT:e(`L${r}.q4scales.${a}`,s.q4ScalesT),offset:o=>s.offset(o),format:s.format,quantBits:s.quantBits,hasMin:s.hasMin,iq:s.iq,lut:s.lut,lut16:s.lut16,iqRaw:s.iqRaw,gridT:s.gridT?e(`iqgrid${s.iqRaw}`,s.gridT):null,...s.gridWordBase===void 0?{}:{gridWordBase:s.gridWordBase}}}}function Oi(e,t){let r=t.denseShards;if(!r||r.length<=1){let n=e("weights",t.weights);return()=>n}return n=>{let a=dh(t,n);return e(`weights.${a}`,r[a])}}function Di(e,t,r){let n=new Map;return a=>{let s=n.get(a);if(s)return s;let o=t(a),i=r.offsets.layers[a],u=Math.min(...Object.values(i)),l=o.dtype==="uint32"?2:Ae(o.dtype),c=o.dtype==="uint32"?.5:1,d={w:o,off:i};if(Number.isFinite(u)&&u>0&&u*l%Ts===0){let p=u*c,f=o.shape.reduce((m,h)=>m*h,1);d={w:e.view(o,p,o.dtype,[f-p],`${o.name}.L${a}`),off:Object.fromEntries(Object.entries(i).map(([m,h])=>[m,h-u]))}}return n.set(a,d),d}}var ud=[{pack:e=>e.packs.q6k,format:"q6_k",align:512},{pack:e=>e.packs.q5k,format:"q5_k",align:256},{pack:e=>e.packs.q3k,format:"q3_k",align:512},{pack:e=>e.packs.q2k,format:"q2_k",align:256},{pack:e=>e.packs.q4knc,format:"q4_k_native_compact",align:512},{pack:e=>e.packs.iq4xs,format:"iq4_xs",align:256,lut:1}],Ts=256;function E6(e,t){let r=R6(e,t);for(let[n,a]of[["bits",r.q4BitsT],["scales",r.q4ScalesT]]){let s=a.byteOffset??0;if(s%Ts!==0)throw new Error(`packed ${n} view for ${t.join(", ")} (${r.format}) starts at byte ${s}, which is not a multiple of ${Ts}: this route's block alignment is too coarse for its per-block scale stride.`)}return r}function R6(e,t){let r=Ot.map(p=>({route:e.packs[p.key],iqRaw:p.iqRaw,wordsPerBlock:p.wordsPerBlock}));for(let p of r){let f=nr(p.route,t[0]);if(!f)continue;let{baseBlock:m,blockCount:h}=Gi(t,v=>({offset:Mt(f.offsets,v,"offset"),count:Mt(f.blockCounts,v,"block count")}),1024),g=m*p.wordsPerBlock,w=f.gridWordBase,b=w===void 0?h*p.wordsPerBlock:f.bits.byteLength/Uint32Array.BYTES_PER_ELEMENT-g;return{q4BitsT:Or(f.bits,g,b),q4ScalesT:Or(f.scales,m/8,Math.ceil(h/8)),offset:v=>Mt(f.offsets,v,"offset")-m,format:"q4_0",hasMin:0,iq:0,lut:0,lut16:0,quantBits:4,iqRaw:p.iqRaw,gridT:f.grid,...w===void 0?{}:{gridWordBase:w-g}}}let n=d1(e,t[0]);if(n){let p=n.lutIds.get(t[0])??0;for(let b of t)if(n.lutIds.get(b)!==p)throw new Error(`co-bound LUT weights disagree on lutId: ${t.join(", ")}`);let f=n.compact===1,m=n.lut2===1?2:4,h=n.lut16?2:1,g=p===9?1:n.lut16?5:3,w=f?Gi(t,b=>({offset:Mt(n.offsets,b,"offset"),count:Mt(n.blockCounts,b,"block count")}),512):l1(n,t);return{q4BitsT:Or(n.bits,w.baseBlock*m,w.blockCount*m),q4ScalesT:f?Or(n.scales,w.baseBlock/8*g,Math.ceil(w.blockCount/8)*g):Or(n.scales,w.baseBlock*h,w.blockCount*h),offset:b=>Mt(n.offsets,b,"offset")-w.baseBlock,format:n.lut2===1?n.lut16?"lut2_16":"lut2_32":f?n.lut16?"lut16_compact":"lut32_compact":n.lut16?"lut16":"lut32",hasMin:0,iq:0,lut:p,lut16:n.lut16,quantBits:4,iqRaw:0,gridT:null}}for(let{pack:p,format:f,align:m,lut:h=0}of ud){let g=nr(p(e),t[0]);if(!g)continue;let{baseBlock:w,blockCount:b}=Gi(t,v=>({offset:Mt(g.offsets,v,"offset"),count:Mt(g.blockCounts,v,"block count")}),m);return{q4BitsT:Or(g.bits,w/8*g.u32PerSuper,b/8*g.u32PerSuper),q4ScalesT:Or(g.scales,w/8*g.scalesPerSuper,b/8*g.scalesPerSuper),offset:v=>Mt(g.offsets,v,"offset")-w,format:f,hasMin:0,iq:0,lut:h,lut16:0,quantBits:4,iqRaw:0,gridT:null}}let a=nr(e.packs.iq,t[0]),s=nr(e.packs.kq4,t[0]),o=nr(e.packs.q4,t[0]),i=a?{pack:a,wordsPerBlock:8,scalesPerBlock:2,hasMin:0,iqFlag:1,format:"iq"}:s?{pack:s,wordsPerBlock:4,scalesPerBlock:2,hasMin:1,iqFlag:0,format:"q4_k"}:o?{pack:o,wordsPerBlock:o.packedQuantBits,scalesPerBlock:1,hasMin:0,iqFlag:0,format:o.packedQuantBits===8?"q8_0":"q4_0"}:null;if(!i)throw new Error(`No packed-quant route holds ${t[0]}`);let{pack:u,wordsPerBlock:l,scalesPerBlock:c}=i,d=l1(u,t);return{q4BitsT:Or(u.bits,d.baseBlock*l,d.blockCount*l),q4ScalesT:Or(u.scales,d.baseBlock*c,d.blockCount*c),offset:p=>Mt(u.offsets,p,"offset")-d.baseBlock,format:i.format,hasMin:i.hasMin,iq:i.iqFlag,lut:0,lut16:0,quantBits:l,iqRaw:0,gridT:null}}function Gi(e,t,r){let n=Number.POSITIVE_INFINITY,a=0;for(let o of e){let{offset:i,count:u}=t(o);n=Math.min(n,i),a=Math.max(a,i+u)}let s=Math.floor(n/r)*r;return{baseBlock:s,blockCount:a-s}}function l1(e,t){return Gi(t,r=>({offset:Mt(e.offsets,r,"offset"),count:Mt(e.blockCounts,r,"block count")}),Ae(e.scales.dtype)===2?128:64)}function c1(e,t){if(e==="lut32_compact"||e==="lut16_compact")throw new Error(`${t} is outside lutcRouteSuffixes(), so it has no compact-LUT kernel`);return e}function Cs(e,t){if(e==="lut2_32"||e==="lut2_16")throw new Error(`${t} is outside lut2Suffixes(), so it has no ternary 2-bit LUT kernel`);return e}function Nn(e,t){let r=a=>{let{q4:s,kq4:o,iq:i,lut32:u,lut16:l,lut32c:c,lut16c:d,lut2_32:p,lut2_16:f}=e.packs,m=h=>(h??[]).findIndex(g=>Gr(g.offsets,a)!==void 0);for(let{key:h}of Ot){let g=m(e.packs[h]);if(g>=0)return`${h}#${g}`}for(let[h,g]of[["lut32",u],["lut16",l],["lut32c",c],["lut16c",d],["lut2_32",p],["lut2_16",f]]){let w=m(g);if(w>=0)return`${h}#${w}:${g[w].lutIds.get(a)??0}`}for(let{pack:h,format:g}of ud){let w=m(h(e));if(w>=0)return`${g}#${w}`}for(let[h,g]of[["iq",i],["kq4",o],["q4",s]]){let w=m(g);if(w>=0)return`${h}#${w}`}return null},n=r(t[0]);return n!==null&&t.every(a=>r(a)===n)}function Je(e,t){return nr(e.packs.q4,t)!==null||G6(e,t)||L6(e,t)||d1(e,t)!==null||ud.some(({pack:r})=>nr(r(e),t)!==null)||Ot.some(({key:r})=>nr(e.packs[r],t)!==null)}function G6(e,t){return nr(e.packs.kq4,t)!==null}function L6(e,t){return nr(e.packs.iq,t)!==null}function d1(e,t){for(let r of[e.packs.lut32,e.packs.lut16,e.packs.lut32c,e.packs.lut16c,e.packs.lut2_32,e.packs.lut2_16]){let n=nr(r,t);if(n)return n}return null}function nr(e,t){for(let r of e??[])if(Gr(r.offsets,t)!==void 0)return r;return null}function Mt(e,t,r){let n=Gr(e,t);if(n===void 0||!Number.isInteger(n)||n<0)throw new Error(`Missing packed weight ${r} for ${t}`);return n}function Or(e,t,r){if(!Number.isInteger(t)||t<0)throw new Error(`tensor view elementOffset must be a non-negative integer, got ${t}`);if(!Number.isInteger(r)||r<=0)throw new Error(`tensor view elementCount must be a positive integer, got ${r}`);let n=Ae(e.dtype),a=e.byteOffset??0,s=a+t*n,o=r*n;if(s+o>a+e.byteLength)throw new Error(`tensor view range exceeds tensor byteLength for dtype ${e.dtype}`);return{...e,shape:[r],size:r,byteOffset:s,byteLength:o}}function q6(e,t,r){let n=new Float64Array(t/2);for(let a=0;a<n.length;++a)n[a]=1/Math.pow(r,a*2/t);return p1(e,n)}function p1(e,t){let r=t.length,n=new Float32Array(e*r),a=new Float32Array(e*r);for(let s=0;s<e;++s){let o=s*r;for(let i=0;i<r;++i){let u=s*t[i];n[o+i]=Math.cos(u),a[o+i]=Math.sin(u)}}return{cos:n,sin:a}}function ld(e){let t=e.rotary_dim;return typeof t=="number"&&t>0?t:e.head_dim}function O6(e,t){let r=ld(e.config);return r===e.config.head_dim?D6(e,t):(e.partialRopeCache=f1(e,r,t,e.partialRopeCache),e.partialRopeCache)}function D6(e,t){return e.ropeCache=f1(e,e.config.head_dim,t,e.ropeCache),e.ropeCache}function f1(e,t,r,n){let a=m1(r);if(n&&n.cos.shape[0]>=r)return n;let s=e.config.rope_parameters?.rope_theta;if(typeof s!="number")throw new Error("RoPE cache requires config.rope_parameters.rope_theta");let o=t/2;return g1(e,a,o,q6(a,t,s),n)}function m1(e){if(!Number.isSafeInteger(e)||e<1)throw new Error(`RoPE cache needs a positive cache length, got ${e}`);return 2**Math.ceil(Math.log2(e))}function g1(e,t,r,{cos:n,sin:a},s){return{cos:e.runtime.tensorFromTypedArray("float32",[t,r],n),sin:e.runtime.tensorFromTypedArray("float32",[t,r],a),...s?{superseded:s}:{}}}function Mi(e,t){let r=e.config,n=r.layer_rope;if(!n){let o={name:"rope",table:O6(e,t),rotaryDim:ld(r)};return()=>o}let a=r.head_dim/2,s=new Map;return o=>{let i=r.layer_types?.[o]??"full_attention",u=s.get(i);if(u)return u;let l=n[i];if(!l)throw new Error(`config.layer_rope has no entry for layer ${o} (${i})`);let c=`${l.rope_theta}/${l.rotated_pairs}/${a}`,d=e.layerRopeCache??=new Map,p=d.get(c),f=m1(t),m=p;if(!m||m.cos.shape[0]<t){let h=new Float64Array(a);for(let g=0;g<l.rotated_pairs;++g)h[g]=1/Math.pow(l.rope_theta,g*2/r.head_dim);m=g1(e,f,a,p1(f,h),p),d.set(c,m)}return u={name:`rope.${i}`,table:m,rotaryDim:r.head_dim},s.set(i,u),u}}var h1=32,w1=8,M6=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj","input_layernorm","post_attention_layernorm","q_layernorm","k_layernorm"],aL=M6.length;var W6=[16,32,64,128,256],F6=256;function cd(e){let t=e.head_dim;if(t===128)return 64;let r=Ps(e);return t<=r||t===r*2?r:W6.find(n=>t<=n)??0}function Ta(e){return e.head_dim%4===0&&e.head_dim<=F6&&cd(e)>0}function Ps(e){return e.head_dim<=64?64:128}function zn(e,t){return e.layer_types?e.layer_types[t]:"full_attention"}function Kn(e){return!!e.attention_qk_norm}function b1(e){return e.packs.q1!=null}function Is(e){return e.features.has("shader-f16")&&e.features.has("subgroups")&&e.features.has("chromium-experimental-subgroup-matrix")}function xa(e){if(!e)return!1;let t=e.features;return t.has("shader-f16")&&t.has("subgroups")&&t.has("chromium-experimental-subgroup-matrix")}function Bs(e,t){return e&&t!==1&&t!==7&&t!==8}function dd(e,t=!1,r=!0){if(!e)return new Set;if(!e.features.has("shader-f16"))return new Set;if(!r&&!(e.features.has("subgroups")&&e.features.has("chromium-experimental-subgroup-matrix")))return new Set;let n=xa(e)?["linear_in_proj_qkv","linear_in_proj_z"]:[],a=xa(e)&&t?["q_proj","k_proj","v_proj"]:[],s=["down_proj"],o=["o_proj","linear_out_proj"];return new Set(["gate_proj","up_proj",...a,...n,...s,...o])}function v1(e,t){let r=e;for(let n of t)r=Q6(r,n);return r}function Q6(e,t){let r=new Map(e.tensors.map(f=>[f.id,f.id])),n=f=>{let m=f;for(;r.has(m)&&r.get(m)!==m;)m=r.get(m);for(;r.has(f)&&r.get(f)!==m;){let h=r.get(f);r.set(f,m),f=h}return m},a=(f,m)=>{r.set(n(f),n(m))};for(let f of e.tensors)f.view&&a(f.id,f.view.parent);for(let f of e.aliases??[])a(f.from,f.to);let s=new Set(e.tensors.filter(f=>f.kind!=="intermediate").map(f=>n(f.id))),o=new Map;e.nodes.forEach((f,m)=>{for(let h of Object.values(f.outputs)){let g=n(h),w=o.get(g)??new Set;w.add(m),o.set(g,w)}});let i=new Map,u=f=>{let m=n(f);i.set(m,(i.get(m)??0)+1)};for(let f of e.nodes){let m=new Set(Object.values(f.outputs));for(let h of Object.values(f.inputs))m.has(h)||u(h)}for(let f of e.tensors)f.kind==="output"&&u(f.id);let l=new Map;e.nodes.forEach((f,m)=>{if(f.op!==t.consumer)return;let h=f.inputs[t.consumerInput];h!==void 0&&l.set(h,l.has(h)?-1:m)});let c=new Map,d=new Set;if(e.nodes.forEach((f,m)=>{if(f.op!==t.producer)return;let h=f.outputs[t.producerOutput];if(h===void 0||(i.get(n(h))??0)!==1||s.has(n(h))||o.get(n(h))?.size!==1)return;let g=l.get(h);if(g===void 0||g<=m||Object.values(f.outputs).some(S=>S!==h&&(s.has(n(S))||(i.get(n(S))??0)>0)))return;let w=new Set(Object.values(f.inputs).map(n));for(let S=m+1;S<g;++S)if(Object.values(e.nodes[S].outputs).some(C=>w.has(n(C))))return;let b=e.nodes[g];if(Object.values(b.outputs).some(S=>w.has(n(S))))return;let v=t.fuse(f,b,e);if(v===null)return;let y=new Set(Object.values(v.outputs));Object.values(b.outputs).some(S=>!y.has(S))||(c.set(g,v),d.add(m))}),c.size===0)return e;let p=[];return e.nodes.forEach((f,m)=>{d.has(m)||p.push(c.get(m)??f)}),Object.freeze({...e,nodes:Object.freeze(p)})}var pd=class{steps=[];prefillSteps=null;chainedSteps=null;get stepCount(){return this.steps.length}get prefillStepCount(){return(this.prefillSteps??this.steps).length}get chainedStepCount(){return(this.chainedSteps??this.steps).length}async step(t,r){return this.writeStepInputs(t,r),this.collector.enqueue(this.steps),(await this.runtime.readTensor(this.idsTensor))[0]}submitStep(t,r){return this.writeStepInputs(t,r),this.collector.enqueue(t===null?this.chainedSteps??this.steps:this.steps),this.runtime.readTensor(this.idsTensor).then(n=>n[0])}enqueueStep(t,r,{full:n=!1}={}){this.writeStepInputs(t,r),this.collector.enqueue(n?this.steps:this.prefillSteps??this.steps)}async replayKnownInputs(t,r){for(let n=0;n<t.length;++n)this.enqueueStep(t[n],r+n);await this.runtime.queueIdle()}finish(){}measureGpuMs(){return this.runtime.measurePreparedSequenceGpuMs?this.runtime.measurePreparedSequenceGpuMs(this.chainedSteps??this.steps):Promise.resolve(null)}},fd=class extends pd{#e=null;get compiled(){if(!this.#e)throw new Error("Graph decode session is not built");return this.#e}set compiled(t){this.#e=t}detachCompiled(t){this.#e===t&&(this.#e=null)}get collector(){return this.compiled.collector}get idsTensor(){return this.compiled.tensor("input_ids")}dispose(){this.#e?.dispose(),this.#e=null}};function _1(e,t){if(t===void 0)return null;let r=e.nodes.length-t;if(!Number.isInteger(t)||t<=0||r<=0)throw new Error(`graph "${e.name}": next-token tail must be a positive proper suffix (got ${t} of ${e.nodes.length} nodes)`);return r}var Es="sampling";var j6=Object.freeze({top_k:0,temperature:0,top_p:1,min_p:0,seed:0,position:0}),Wi=class{#e=null;#t=!1;set(t){this.#e=t}write(t,r,n){let a=this.#e;a===null&&!this.#t||(n(r()(a===null?j6:{...a,position:t})),this.#t=a!==null)}};function y1(e){return e.tensors.some(t=>t.kind==="uniform"&&t.name===Es)}function A1(e){return new Error(`graph "${e.name}" picks its next token inside a fused lm_head; sampling needs a head that writes the logits, so this model decodes greedily only`)}function k1(e,t){try{t.dispose()}catch(r){throw new AggregateError([e,r],"Graph session build and cleanup both failed")}throw e}var Fi=class extends fd{model;cache;emission;#e;#t=new Map;#n=new Wi;#r=!1;constructor(t,r){super(),this.model=t,this.cache=r}stepUniformPack(t){let r=this.#t.get(t);return r||(r=ws(this.#e,t,this.compiled.nodeVariants),this.#t.set(t,r)),r}get runtime(){return this.model.runtime}async prepareBuild(){}onCompiled(){}async build(t,{fuse:r=!1}={}){await this.prepareBuild();let{emission:n,graph:a}=this.buildGraph(t,r),s=n.nextTokenTailNodeCount,o=n.gpuChainedPrefixNodeCount,i=_1(a,s);if(o!==void 0&&(!Number.isInteger(o)||o<=0||o>=(i??a.nodes.length)))throw new Error(`graph "${a.name}": GPU-chained prefix must be a positive proper prefix of the reusable body (got ${o} of ${i??a.nodes.length} body nodes)`);let u=en(a,this.runtime,{weights:n.weights,states:n.states,...n.inputs?{inputs:n.inputs}:{}});this.emission=n,this.#e=a,this.#t.clear(),this.#r=y1(a),this.compiled=u;try{if(i===null)this.prefillSteps=null,this.steps=await u.buildSteps(),this.chainedSteps=o===void 0?null:await u.buildSteps({startNode:o});else{let l=o===void 0?[]:await u.buildSteps({endNode:o}),c=await u.buildSteps({...o!==void 0?{startNode:o}:{},endNode:i}),d=await u.buildSteps({startNode:i});if(c.length===0||d.length===0||o!==void 0&&l.length===0)throw new Error(`graph "${a.name}": next-token tail split produced an empty prepared sequence`);this.prefillSteps=[...l,...c],this.steps=[...this.prefillSteps,...d],this.chainedSteps=o===void 0?null:[...c,...d]}this.onCompiled()}catch(l){this.steps=[],this.prefillSteps=null,this.chainedSteps=null,this.#t.clear(),this.detachCompiled(u),k1(l,u)}}setSampling(t){if(t!==null&&!this.#r)throw A1(this.#e);this.#n.set(t)}writeStepInputs(t,r){t!==null&&this.runtime.host.writeBuffer(this.compiled.tensor("input_ids").buffer,0,new Uint32Array([t])),this.writeStepUniforms(r),this.#r&&this.#n.write(r,()=>this.stepUniformPack(Es),n=>this.runtime.host.writeBuffer(this.compiled.uniformBuffer(Es),0,n))}},Qi=class{model;cache;blockLen;emission;#e=null;steps=[];bodySteps=null;#t=new Map;#n=new Wi;#r=!1;constructor(t,r,n){this.model=t,this.cache=r,this.blockLen=n}get compiled(){if(!this.#e)throw new Error("Graph prefill session is not built");return this.#e}stepUniformPack(t){let r=this.#t.get(t);return r||(r=ws(this.emission.graph,t,this.compiled.nodeVariants),this.#t.set(t,r)),r}get runtime(){return this.model.runtime}async prepareBuild(){}onCompiled(){}async build(){await this.prepareBuild();let t=this.buildEmission(),r=_1(t.graph,t.nextTokenTailNodeCount),n=en(t.graph,this.runtime,{weights:t.weights,states:t.states,...t.inputs?{inputs:t.inputs}:{}});this.emission=t,this.#t.clear(),this.#r=y1(t.graph),this.#e=n;try{if(r===null)this.bodySteps=null,this.steps=await n.buildSteps();else{let a=await n.buildSteps({endNode:r}),s=await n.buildSteps({startNode:r});if(a.length===0||s.length===0)throw new Error(`graph "${t.graph.name}": next-token tail split produced an empty prepared sequence`);this.bodySteps=a,this.steps=[...a,...s]}this.onCompiled()}catch(a){this.steps=[],this.bodySteps=null,this.#t.clear(),this.#e===n&&(this.#e=null),k1(a,n)}}writeInputs(t,r){if(t.length===0||t.length>this.blockLen)throw new Error(`prefill block ${this.blockLen} cannot run ${t.length} tokens`);let n=new Uint32Array(this.blockLen);n.set(t),this.runtime.host.writeBuffer(this.compiled.tensor("input_ids").buffer,0,n),this.writeRunInputs(t.length,r),this.#r&&this.#n.write(r+t.length-1,()=>this.stepUniformPack(Es),a=>this.runtime.host.writeBuffer(this.compiled.uniformBuffer(Es),0,a))}setSampling(t){if(t!==null&&!this.#r)throw A1(this.emission.graph);this.#n.set(t)}enqueueBody(t,r=0){if(!this.bodySteps)throw new Error(`prefill graph "${this.emission.graph.name}" has no removable next-token tail`);this.writeInputs(t,r),this.compiled.collector.enqueue(this.bodySteps)}async run(t,r=0){return this.writeInputs(t,r),this.compiled.collector.enqueue(this.steps),(await this.runtime.readTensor(this.compiled.tensor("next_token")))[0]}dispose(){this.#e?.dispose(),this.#e=null}};var Ni="image_rows",zi="image_mask";function Ki(e,t,r,n,a){return e.op("ai.onnx.Where",{condition:e.stepInput(zi,"bool",[r,1]),x:e.stepInput(Ni,a,[r,n]),y:t,output:e.scratch("embed.multimodal",a,[r,n])}).output}var ji=class{#e;#t;constructor(t,r){this.#e=t,this.#t=new Uint32Array(r)}write(t,r,n,a,s){let o=this.#t.fill(0),i=r.tensor(Ni),u=i.shape[1],l=u*4,c=this.#e.pendingRows.get(n),d=s+a;for(let f of n.ropePositions.spans){let m=f.gridHeight*f.gridWidth,h=Math.max(f.start,s),g=Math.min(f.start+m,d);if(h>=g)continue;let w=c?.get(f.start);if(!w)throw new Error(`prefill reached media rows ${h}..${g-1} with no features pending`);let{dtype:b,shape:v}=w;if(b!=="float32"||v.length!==2||v[0]!==m||v[1]!==u)throw new Error(`media rows at ${f.start} are ${b} [${v.join(", ")}]; expected float32 [${m}, ${u}]`);o.fill(1,h-s,g-s),t.copyBufferToBuffer({src:w,dst:i,srcOffset:(w.byteOffset??0)+(h-f.start)*l,dstOffset:(i.byteOffset??0)+(h-s)*l,byteLength:(g-h)*l})}let p=r.tensor(zi);t.host.writeBuffer(p.buffer,p.byteOffset??0,o)}};var N6=256;function z6(e){let r=e.config;if(rr(r))throw new Error("emitLlamaDecode: linear-attention layers decode through the Qwen3.5 decode graph");if(r.attention_output_gate==="headwise"&&!Kn(r))throw new Error("emitLlamaDecode: a headwise attention output gate needs the qk-norm path without Q1 weights");if(r.attention_sinks&&!Ta(r))throw new Error(`emitLlamaDecode: attention sinks need split-K decode attention (head_dim ${r.head_dim} is not served by it)`)}function gd(e,t){return zn(e,t)!=="sliding_attention"?0:$e(e.sliding_window,"config.sliding_window")}var hd=e=>Number(Math.pow(e,-.5).toPrecision(9));function S1(e){return!1}function $i(e,t){let n=e.nodeCount,a=!!(t.model.lmHeadQ4&&t.model.lmHeadQ4Scales),s=!!(t.model.lmHeadQ8&&t.model.lmHeadQ8Scales);if(null===null&&t.model.lmHeadScanBits===2)throw new Error(`lm_head is packed as a 2-bit scan grid, but the two-stage head cannot serve a vocabulary of ${t.vocabSize} on this device`);let i=a?512:N6,u=t.nameSuffix??"",l=e.scratch(`${u}bestValues`,"float32",[i]),c=e.scratch(`${u}bestIds`,"uint32",[i]);return a?e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,bitsT:t.w("lmhead.q4bits",t.model.lmHeadQ4),scalesT:t.w("lmhead.q4scales",t.model.lmHeadQ4Scales),weightsT:t.weights,bestValuesT:l,bestIdsT:c},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,lmHeadOffset:0,finalNormOffset:t.finalNormOffset,numBlocks:i,...t.model.lmHeadFormat?{format:t.model.lmHeadFormat}:{}}}):s?e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,weightsT:t.weights,bitsT:t.w("lmhead.q8bits",t.model.lmHeadQ8),scalesT:t.w("lmhead.q8scales",t.model.lmHeadQ8Scales),bestValuesT:l,bestIdsT:c},{args:{vocabSize:t.vocabSize,hiddenSize:t.hiddenSize,numBlocks:i,rmsEps:t.rmsEps,finalNormOffset:t.finalNormOffset,format:"q8_rows"}}):e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,weightsT:t.weights,bestValuesT:l,bestIdsT:c},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,rmsEps:t.rmsEps,finalNormOffset:t.finalNormOffset,lmHeadOffset:t.lmHeadOffset,numBlocks:i}}),e.op("com.xenova.CandidateArgMax",{valuesT:l,indicesT:c,outT:t.ids},{args:{outputOffset:t.outputOffset??0,count:i}}),e.nodeCount-n}var x1="hidden_states";function Rs(e,t,r,{mediaRows:n=!1}={}){z6(e);let s=e.config;s.prismHadamard&&Pn("Prism Hadamard rotations");let o=Kn(s),i=s.hidden_size,u=s.num_hidden_layers,l=s.num_attention_heads,c=s.num_key_value_heads,d=s.head_dim,p=c*d,f=_r(s)?$e(s.conv_L_cache,"config.conv_L_cache"):0,m=s.intermediate_size,h=s.vocab_size,g=Number(s.rms_norm_eps),w=t.maxLength,b=hd(d),v=cd(s),y=Mi(e,w),S=s.attention_output_gate==="headwise",C=!!s.use_sandwich_norm,k=e.offsets.top,P=e.runtime.device.features.has("shader-f16")&&e.weights.dtype==="float16"?"float16":"float32",_="float32",x=new lt,{w:R,state:O,boundWeights:N,states:Q}=vt(x),V=Oi(R,e),$=Di(x,V,e),M=V("top"),j=W=>{let J=y(W);return{cosT:R(`${J.name}.cos`,J.table.cos),sinT:R(`${J.name}.sin`,J.table.sin),rotaryDim:J.rotaryDim}};s.layer_rope||j(0);let A=x.uniform("params",16),T=xa(e.runtime.device),E=qi(R,e),X=null,z=W=>W.shape.reduce((J,ie)=>J*ie,1),G=W=>W<u&&(zn(s,W)==="conv"?C||!Je(e,`layers.${W}.conv_in_proj`):o&&(C||S||!Je(e,`layers.${W}.q_proj`))),L=new Map,K=(W,J)=>{let ie=L.get(J);if(ie)return ie;let pe=x.scratch(`L${J}.normed`,"float32",[i]);return x.op("co.huggingface.RMSNorm",{xT:W,wT:$(J).w,yT:pe},{args:{rows:1,dim:i,eps:g,weightBaseOffset:$(J).off.input_layernorm}}).yT},D=new Map,F=W=>C&&!0&&!ka(re,W)&&Je(e,`layers.${W}.gate_proj`)&&Nn(e,[`layers.${W}.gate_proj`,`layers.${W}.up_proj`]),H=W=>G(W)?{layer:W,offset:$(W).off.input_layernorm,name:`L${W}.normed`,into:L}:void 0,Y=W=>F(W)?{layer:W,offset:$(W).off.post_attention_layernorm,name:`L${W}.ffnNormed`,into:D}:void 0,fe=(W,J,ie,pe,me)=>{let se=me?x.scratch(me.name,"float32",[i]):null,Te=x.op("com.xenova.RmsNormResidualAdd",{xT:x.view(J,0,"float32",[1,i],`${J.name}.row`),wT:$(ie).w,yT:x.view(W,0,"float32",[1,i],`L${ie}.residual.row`),...se?{w2T:$(me.layer).w,normed2T:x.view(se,0,"float32",[1,i],`${se.name}.row`)}:{}},{args:{rows:1,dim:i,eps:g,weightBaseOffset:pe,...se?{eps2:g,weight2BaseOffset:me.offset}:{}}});return se&&me.into.set(me.layer,x.view(Te.normed2T,0,"float32",[i],se.name)),x.view(Te.yT,0,"float32",[i],`L${ie}.residual`)},ce=(W,J,ie,pe)=>{let me=`layers.${J}.${ie}`,se=Je(e,me)?E(J,[ie],`o.${ie}`):null,Te=C?x.scratch(`L${J}.operatorOut`,"float32",[i]):W,Ce={...z(pe)!==i?{inFeatures:z(pe)}:{},...C?{accumulate:0}:{}},He=se?x.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:Te,bitsT:se.bitsT,scalesT:se.scalesT,attnT:pe,...se.gridT&&!Bs(T,se.iqRaw)?{gridT:se.gridT}:{}},{args:{hiddenSize:i,oOffset:se.offset(me),format:Cs(se.format,"output projection"),lut:se.lut,iqRaw:se.iqRaw,...Ce}}).hiddenT:x.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:Te,weightsT:$(J).w,attnT:pe},{args:{hiddenSize:i,oOffset:$(J).off[ie],...Ce}}).hiddenT;return C?fe(W,He,J,$(J).off.post_operator_norm,Y(J)):He},re=Qn(s);re&&Pn("MoE FFN layers");let Z=Ci(re),U=void 0,te=(W,J,ie)=>{let pe=$(J).off,me=ie.width,se=x.scratch(`L${J}.${ie.label}intermediate`,P,[me]);if(Je(e,`layers.${J}.${ie.gate}`)){let Te=`layers.${J}.${ie.gate}`,Ce=`layers.${J}.${ie.up}`,He=Nn(e,[Te,Ce]),_e=E(J,He?[ie.gate,ie.up]:[ie.gate],`${ie.label}gateup`),Ge=He?null:E(J,[ie.up],`${ie.label}gateupU`),ze=_e.gridT??Ge?.gridT??null,Ft=Ge&&_e.iqRaw&&Ge.iqRaw&&Ge.iqRaw!==_e.iqRaw?Ge.gridT:null,yr=_e.gridT?_e.iqRaw:Ge?.iqRaw??0,mt=Ge?.iqRaw??0,sr={...ze&&!Bs(T,yr)?{gridT:ze}:{},...Ft&&!Bs(T,mt)?{upGridT:Ft}:{}};if(Ge){let gt=x.scratch(`L${J}.${ie.label}postNormed`,"float32",[i]);x.op("co.huggingface.RMSNorm",{xT:W,wT:$(J).w,yT:gt},{args:{rows:1,dim:i,eps:g,weightBaseOffset:pe.post_attention_layernorm}}),x.op("com.xenova.LlamaDecodeGateUp",{normedT:gt,bitsT:_e.bitsT,scalesT:_e.scalesT,upBitsT:Ge.bitsT,upScalesT:Ge.scalesT,intermediateT:se,...sr},{args:{hiddenSize:i,intermediateSize:me,gateOffset:_e.offset(Te),upOffset:Ge.offset(Ce),format:_e.format,lut:_e.lut,iqRaw:_e.iqRaw,upFormat:Ge.format,upLut:Ge.lut,upIqRaw:Ge.iqRaw}})}else ie===ne&&D.has(J)?x.op("com.xenova.LlamaDecodeGateUp",{normedT:D.get(J),bitsT:_e.bitsT,scalesT:_e.scalesT,intermediateT:se,...sr},{args:{hiddenSize:i,intermediateSize:me,gateOffset:_e.offset(Te),upOffset:_e.offset(Ce),format:_e.format,lut:_e.lut,iqRaw:_e.iqRaw}}):x.op("com.xenova.LlamaDecodeGateUp",{hiddenT:W,bitsT:_e.bitsT,scalesT:_e.scalesT,intermediateT:se,weightsT:$(J).w,...sr},{args:{hiddenSize:i,intermediateSize:me,rmsEps:g,normOffset:pe.post_attention_layernorm,gateOffset:_e.offset(Te),upOffset:_e.offset(Ce),format:Cs(_e.format,"fused gate|up"),lut:_e.lut,iqRaw:_e.iqRaw}})}else x.op("com.xenova.LlamaDecodeGateUp",{hiddenT:W,weightsT:$(J).w,intermediateT:se},{args:{hiddenSize:i,intermediateSize:me,rmsEps:g,normOffset:pe.post_attention_layernorm,gateOffset:pe[ie.gate],upOffset:pe[ie.up]}});return se},ae=(W,J,ie,pe)=>{let me=$(J).off,se=ie.width,Te=`layers.${J}.${ie.down}`,Ce=Je(e,Te)?E(J,[ie.down],`${ie.label}down`):null,He=C&&ie===ne,_e=He?x.scratch(`L${J}.ffnOut`,"float32",[i]):W,Ge=Ce?x.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:_e,bitsT:Ce.bitsT,scalesT:Ce.scalesT,intermediateT:pe,...Ce.gridT&&!Bs(T,Ce.iqRaw)?{gridT:Ce.gridT}:{}},{args:{hiddenSize:i,intermediateSize:se,downOffset:Ce.offset(Te),format:Cs(Ce.format,"down projection"),lut:Ce.lut,iqRaw:Ce.iqRaw,...He?{accumulate:0}:{}}}).hiddenT:x.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:_e,weightsT:$(J).w,intermediateT:pe},{args:{hiddenSize:i,intermediateSize:se,downOffset:me[ie.down],...He?{accumulate:0}:{}}}).hiddenT;return He?fe(W,Ge,J,me.post_ffn_norm,H(J+1)):Ge},ne={gate:"gate_proj",up:"up_proj",down:"down_proj",width:m,label:""},xe=(W,J)=>U&&ka(re,J)?U(W,J):ae(W,J,ne,te(W,J,ne)),De=x.stepInput("input_ids","uint32",[1]),Me=x.scratch("embed.hidden","float32",[i]),ye;e.embedBits&&e.embedScales?ye=x.op("com.xenova.LlamaEmbed",{inputT:De,bitsT:R("embed.bits",e.embedBits),scalesT:R("embed.scales",e.embedScales),hiddenT:Me,paramsT:A},{args:{hiddenSize:i,vocabSize:h,tokenOffset:0,format:e.embedFormat??"q8_rows",...e.embedLut!==null&&e.embedLut!==void 0?{lut:e.embedLut}:{}}}).hiddenT:ye=x.op("com.xenova.LlamaEmbed",{inputT:De,weightsT:M,hiddenT:Me,paramsT:A},{args:{hiddenSize:i,vocabSize:h,embedOffset:k.embed_tokens,tokenOffset:0}}).hiddenT,n&&(ye=x.view(Ki(x,x.view(ye,0,"float32",[1,i]),1,i,"float32"),0,"float32",[i]));for(let W=0;W<u;++W){let J=$(W).off;if(zn(s,W)==="conv"){if(!t.convStates)throw new Error("emitLlamaDecode: conv layer requires cache.convStates");let ie=O(`conv.state.${W}`,t.convStateForLayer(W)),pe=x.scratch(`L${W}.convY`,"float32",[i]);if(Je(e,`layers.${W}.conv_in_proj`)){let me=x.scratch(`L${W}.convProj`,"float32",[3*i]),se=E(W,["conv_in_proj"],"convin"),Te=G(W);x.op("com.xenova.LlamaDecodeConvInProj",{hiddenT:Te?K(ye,W):ye,bitsT:se.bitsT,scalesT:se.scalesT,...Te?{}:{weightsT:$(W).w},convProjT:me},{args:{hiddenSize:i,...Te?{preNormed:1}:{rmsEps:g,normOffset:J.input_layernorm},convInOffset:se.offset(`layers.${W}.conv_in_proj`),format:Cs(c1(se.format,"conv in-proj"),"conv in-proj"),lut:se.lut}}),x.op("com.xenova.LlamaDecodeShortConv",{bcxT:me,weightsT:$(W).w,convStatesT:ie,convYT:pe},{args:{layer:0,hiddenSize:i,convLCache:f,convWeightOffset:J.conv_weight}})}else x.op("com.xenova.LlamaDecodeShortConv",{normedT:K(ye,W),weightsT:$(W).w,convStatesT:ie,convYT:pe},{args:{layer:0,hiddenSize:i,convLCache:f,convInOffset:J.conv_in_proj,convWeightOffset:J.conv_weight}});ye=ce(ye,W,"conv_out_proj",pe)}else{let ie=Li(t,W),pe=O(`cache.key.${W}`,ie.cacheKeysT),me=O(`cache.value.${W}`,ie.cacheValuesT),{cosT:se,sinT:Te,rotaryDim:Ce}=j(W),He=gd(s,W),_e=t.kvRowsForLayer(W),Ge=t.isRingLayer(W)?{cacheRing:1}:{};if(t.isRingLayer(W)&&!o)throw new Error("emitLlamaDecode: KV rings are wired for the separate qk-norm cache write");let ze=S?x.scratch(`L${W}.gate`,"float32",[l]):null,Ft,yr=null;if(o){let Xe=l*d,ht=Pt=>Pt.dtype==="float32"||Pt.dtype==="float16",nt=Ta(s)&&Ce>=d&&d%8===0&&Xe*4%256===0&&p*4%256===0&&ht(pe)&&ht($(W).w)&&J.q_layernorm%4===0&&J.k_layernorm%4===0?x.scratch(`L${W}.qkv`,"float32",[Xe+2*p]):null,Ar=nt?x.view(nt,0,"float32",[Xe],`L${W}.q`):x.scratch(`L${W}.q`,"float32",[Xe]),Mr=nt?x.view(nt,Xe,"float32",[p],`L${W}.k`):x.scratch(`L${W}.k`,"float32",[p]),Wr=nt?x.view(nt,Xe+p,"float32",[p],`L${W}.v`):x.scratch(`L${W}.v`,"float32",[p]);if(Je(e,`layers.${W}.q_proj`)){let Pt=ze?["q_proj","k_proj","v_proj","attn_gate"]:["q_proj","k_proj","v_proj"],pn=ze!==null&&!Nn(e,Pt.map(fn=>`layers.${W}.${fn}`)),kt=E(W,pn?Pt.filter(fn=>fn!=="v_proj"):Pt,"qkv"),or=pn?E(W,["v_proj"],"qkvV"):null,Zn=G(W);x.op("com.xenova.LlamaDecodeQkvProj",{hiddenT:Zn?K(ye,W):ye,bitsT:kt.bitsT,scalesT:kt.scalesT,qT:Ar,kT:Mr,vT:Wr,...Zn?{}:{normWeightsT:$(W).w},...ze?{headGateT:ze}:{},...or?{vBitsT:or.bitsT,vScalesT:or.scalesT}:{}},{args:{hiddenSize:i,numKvHeads:c,headDim:d,qOffset:kt.offset(`layers.${W}.q_proj`),kOffset:kt.offset(`layers.${W}.k_proj`),vOffset:(or??kt).offset(`layers.${W}.v_proj`),format:kt.format,lut:kt.lut,...Zn?{preNormed:1}:{rmsEps:g,normOffset:J.input_layernorm},...l*d!==i?{qSize:l*d}:{},...ze?{numHeads:l,gateOffset:kt.offset(`layers.${W}.attn_gate`)}:{},...or?{vFormat:or.format,vLut:or.lut}:{}}})}else x.op("com.xenova.LlamaDecodeQkvProj",{normedT:K(ye,W),weightsT:$(W).w,qT:Ar,kT:Mr,vT:Wr,...ze?{headGateT:ze}:{}},{args:{hiddenSize:i,numKvHeads:c,headDim:d,qOffset:J.q_proj,kOffset:J.k_proj,vOffset:J.v_proj,...l*d!==i?{qSize:l*d}:{},...ze?{numHeads:l,gateOffset:J.attn_gate}:{}}});nt?yr=nt:x.op("com.xenova.LlamaDecodeRopeCacheKv",{stageQT:Ar,stageKT:Mr,stageVT:Wr,weightsT:$(W).w,cacheKeysT:pe,cacheValuesT:me,cosT:se,sinT:Te,paramsT:A},{args:{layer:0,cacheLen:_e,...Ge,pastLen:r,numHeads:l,numKvHeads:c,headDim:d,...Ce<d?{rotaryDim:Ce}:{},rmsEps:g,qNormOffset:J.q_layernorm,kNormOffset:J.k_layernorm}}),Ft=nt??Ar}else{let Xe=x.scratch(`L${W}.q`,"float32",[i]),ht=Je(e,`layers.${W}.q_proj`)?E(W,["q_proj"],"q"):null;x.op("com.xenova.LlamaDecodeQkvProj",{hiddenT:ye,weightsT:$(W).w,qT:Xe,...ht?{bitsT:ht.bitsT,scalesT:ht.scalesT}:{},cacheKeysT:pe,cacheValuesT:me,cosT:se,sinT:Te,paramsT:A},{args:{layer:0,cacheLen:w,pastLen:r,hiddenSize:i,rmsEps:g,normOffset:J.input_layernorm,numHeads:l,numKvHeads:c,headDim:d,...Ce<d?{rotaryDim:Ce}:{},qOffset:ht?ht.offset(`layers.${W}.q_proj`):J.q_proj,...ht?{format:ht.format}:{},kOffset:J.k_proj,vOffset:J.v_proj}}),Ft=Xe}let mt=Ta(s),sr=x.scratch(`L${W}.attn`,"float32",[l*d]),gt=x.op("com.xenova.CachedDecodeAttention",{qT:Ft,cacheKeysT:pe,cacheValuesT:me,attnT:sr,paramsT:A,...ze?{gateT:ze}:{},...s.attention_sinks?{sinkT:$(W).w}:{},...yr?{normWeightsT:$(W).w,cosT:se,sinT:Te}:{}},{args:{layer:0,cacheLen:_e,...Ge,pastLen:r,numHeads:l,numKvHeads:c,headDim:d,scale:b,...He>0?{windowSize:He}:{},...s.attention_sinks?{sinkOffset:$(W).off.attn_sinks}:{},...yr?{qOffset:0,kOffset:l*d,vOffset:l*d+c*d,qNormOffset:J.q_layernorm,kNormOffset:J.k_layernorm,rmsEps:g}:{},workgroupSize:mt?v:Ps(s),...mt?{tileSize:16,numKvChunks:h1,minKeysPerChunk:w1}:{}}}).attnT;ye=ce(ye,W,"o_proj",gt)}ye=xe(ye,W)}n&&x.output(x.op("co.huggingface.RMSNorm",{xT:x.view(ye,0,"float32",[1,i]),wT:M,yT:x.scratch("hidden.normed","float32",[1,i])},{args:{rows:1,dim:i,eps:g,weightBaseOffset:k.model_norm}}).yT,x1);let Ze=$i(x,{w:R,model:e,hidden:ye,weights:M,ids:De,hiddenSize:i,vocabSize:h,rmsEps:g,finalNormOffset:k.model_norm,lmHeadOffset:k.lm_head,q1:!1,normedDtype:_});return{graph:x.finish({name:"llama-decode",params:{positionOffset:r}}),weights:N,states:Q,paramsName:"params",nextTokenTailNodeCount:Ze}}var K6={id:"rms-qkv",producer:"co.huggingface.RMSNorm",producerOutput:"yT",consumer:"com.xenova.LlamaDecodeQkvProj",consumerInput:"normedT",fuse:(e,t,r)=>{let n=e.args??{},a=t.args??{};return t.inputs.qGateT!==void 0||t.outputs.qGateT!==void 0||r.tensors.find(s=>s.id===e.inputs.xT)?.dtype!=="float32"||r.tensors.find(s=>s.id===e.outputs.yT)?.dtype!=="float32"||Object.keys(t.inputs).some(s=>s!=="normedT"&&s!=="weightsT"&&t.outputs[s]===void 0)||n.rows!==1||typeof n.weightBaseOffset!="number"||typeof n.dim!="number"||typeof n.eps!="number"||(n.weightOffset??0)!==0||n.castBeforeScale||(n.exact??0)!==0||e.inputs.wT===void 0||t.inputs.weightsT!==e.inputs.wT||a.hiddenSize!==n.dim?null:{op:"com.xenova.LlamaDecodeQkvProj",inputs:{hiddenT:e.inputs.xT,weightsT:e.inputs.wT},outputs:{...t.outputs},args:{hiddenSize:n.dim,numKvHeads:a.numKvHeads,headDim:a.headDim,rmsEps:n.eps,normOffset:n.weightBaseOffset,qOffset:a.qOffset,kOffset:a.kOffset,vOffset:a.vOffset,...a.qSize!==void 0?{qSize:a.qSize}:{},...a.gateOffset!==void 0?{numHeads:a.numHeads,gateOffset:a.gateOffset}:{}}}}},U6=[K6],md=class extends Fi{writeStepUniforms(t){let r=this.stepUniformPack(this.emission.paramsName)({past_len:t,cache_len:this.cache.maxLength,token_offset:0});this.runtime.host.writeBuffer(this.compiled.uniformBuffer(this.emission.paramsName),0,r)}},Ui=class extends md{#e;constructor(t,r,{mediaRows:n=!1}={}){super(t,r),this.#e=n}buildGraph(t,r){let n=Rs(this.model,this.cache,t,{mediaRows:this.#e});return{emission:n,graph:r?v1(n.graph,U6):n.graph}}submitWithRow(t,r,n,a){let s=this.runtime.host,o=this.compiled.tensor(zi).buffer;if(r){let i=this.compiled.tensor(Ni);this.runtime.copyBufferToBuffer({src:r,dst:i,byteLength:i.byteLength}),s.writeBuffer(o,0,new Uint32Array([1]))}try{return a?this.submitStep(t,n):(this.enqueueStep(t,n),null)}finally{r&&s.writeBuffer(o,0,new Uint32Array([0]))}}get hiddenStates(){return this.compiled.tensor(x1)}};function Hn(e){let t=e.config;return!(b1(e)||rr(t))}var $6=480,H6=96,V6=8,Z6=8;function T1(e,t,r){if(e<=8||e>16)return 1;let n=Math.ceil(t/128),a=Math.min(V6,Math.max(1,Math.round($6/n)));return Math.min(a,Math.max(1,Math.floor(r/32/Z6)))}var X6=512;function C1(e,t,r){let n=e%64===0?64:e<64&&e%16===0?e:0;if(n===0)return 1;let a=Math.ceil(t/64)*(e/n),s=X6*(64/n),o=n===64?8:16,i=1;for(;i<o&&a*i<s&&r%(64*i)===0&&r/(2*i)>=256;)i*=2;return i}var P1=1024;function I1(e,t,r){let n=Math.ceil(e/16)*Math.ceil(t/64),a=1;for(;a<16&&n*a<P1&&r%(64*a)===0&&r/(2*a)>=256;)a*=2;return a}function Y6(e,t,r){let n=Math.ceil(e/64)*Math.ceil(t/64)>=128&&t>=1024;return e<=16||t<512||n?1:I1(e,t,r)}function J6(e,t,r,n){if(e>=64&&Math.ceil(e/64)*Math.ceil(t/64)>=(n==="q8_0"?24:48))return 1;let a=Math.ceil(e/32)*Math.ceil(t/32),s=1;for(;s<16&&a*s<P1&&r%(128*s)===0&&r/(2*s)>=256;)s*=2;return s}function B1(e,t,r,n){for(let a=1;a<t.length;++a)if(e.offset(t[a])!==e.offset(t[a-1])+r[a-1]*n/32)return!1;return!0}function Un(e,t,r,n,a,s,o,i,u,l){if(s.dtype==="uint32"){let p=o*2%Ts===0,f=p&&o>0?e.view(s,o/2,"uint32",[u*i/2],`${s.name}@${o}`):s,m=p||o%8===0?t?C1(n,u,i):I1(n,u,i):1,h=m>1?{partialsT:e.scratch(`${l.name}.c0.partials`,"float32",[m*n*u])}:{};return e.op("com.xenova.LlamaPrefillMatmul",{aT:a,bitsT:f,yT:l,...h},{args:{M:n,inFeatures:i,outFeatures:u,format:"bf16",...!p&&o?{weightOffset:o}:{},...m>1?{kSplits:m}:{}}}).yT}let c=t&&r==="float16"&&u%64===0&&i%32===0?T1(n,u,i):s.dtype==="float32"&&r==="float32"&&o%4===0?Y6(n,u,i):1,d=c>1?{partialsT:e.scratch(`${l.name}.c0.partials`,"float32",[c*n*u])}:{};return e.op("com.xenova.LlamaPrefillMatmul",{aT:a,weightsT:s,yT:l,...d},{args:{weightOffset:o,M:n,inFeatures:i,outFeatures:u,...c>1?{kSplits:c}:{}}}).yT}var ex=64,tx=new Set(["q4_0","q8_0","q4_k","q6_k","q3_k","q2_k","q5_0","q4_k_native_compact","q5_k","lut32","lut16","iq","lut32_compact","lut16_compact","lut2_32","lut2_16"]),rx=new Set(["q4_0","q8_0","q4_k","q6_k","q3_k","q2_k","q4_k_native_compact","q5_k","lut32","lut16","iq","lut32_compact","lut16_compact","lut2_32","lut2_16"]),nx=new Set(["q2_k","q3_k","q5_k","q6_k","q4_k_native_compact","lut32_compact","lut16_compact","lut2_32","lut2_16"]),ax=new Set(["q4_0","q8_0","q4_k","q6_k","q3_k","q2_k","q5_0","q4_k_native_compact","q5_k","lut32","lut16","iq","lut32_compact","lut16_compact"]),sx=new Set(["q4_0","q4_1","q4_k","q4_k_native","q5_0","q8_0","q8_0_pair","q2_k","q3_k","q6_k","lut16","lut32","iq","q5_k","q4_k_native_compact","lut16_compact","lut32_compact","lut2_16","lut2_32"]);function bd(e){return e.runtime.device.features.has("shader-f16")&&e.weights.dtype==="float16"?"float16":"float32"}function ox(e,t,r=ex){return t>=r?e.stepInput("real_rows","uint32",[1]):void 0}function ix({packedQ1:e,activationDtype:t,rows:r,inFeatures:n,intermediateSize:a}){return e&&t==="float16"&&n%128===0&&a%64===0&&a>=n&&(r>=64||r===16&&2*a>=3*n)}function ux({g:e,model:t,denseW:r,q4w:n,q1w:a,q1RealLenT:s,realRowsT:o,nativeLut2PolicyRows:i,T:u,H:l,inter:c,eps:d,actGemm:p,fusedQ4:f,q1Activation:m,transformInput:h,steerActivation:g}){let b=t.runtime.device.adapterInfo,v=b.subgroupMinSize===32&&b.subgroupMaxSize===32||t.runtime.device.features.has("subgroup-size-control")&&b.subgroupMinSize!==void 0&&b.subgroupMinSize<=32&&b.subgroupMaxSize!==void 0&&b.subgroupMaxSize>=32,y=u>=512&&v&&(b.subgroupMatrixConfigs===void 0||b.subgroupMatrixConfigs.some(G=>G.componentType==="f16"&&G.M===8&&G.N===8&&G.K===8)),S=xa(t.runtime.device),C=void 0,k=void 0,B=(G,L)=>G%4===0&&L%128===0&&u<=65535,P=G=>G.format==="lut2_32"&&(G.lut===8||G.lut===9),_=G=>p==="float32"&&(!!G.iqRaw||tx.has(G.format)),x=u>16&&!1,R=G=>!f&&t.runtime.device.features.has("shader-f16")&&!0&&!_(G)&&!G.iqRaw&&sx.has(G.format),O=G=>!f&&u>64&&!0&&!G.iqRaw&&(G.format==="lut2_32"||G.format==="lut2_16"),N=(G,L,K,D,F,H,Y,fe)=>{let ce=_(L),re=f&&(u>=64||u%16===0),Z=f||L.format==="q4_k_native_compact"&&Math.ceil(D/32)<=H6,U=ce?re&&(L.iqRaw||rx.has(L.format))?C1(u,D,F):!re&&!L.iqRaw&&ax.has(L.format)?J6(u,D,F,L.format):1:Z&&p==="float16"&&D%64===0&&F%32===0?T1(u,D,F):1,te=U>1?{partialsT:e.scratch(`${H.name}.c${fe}.partials`,"float32",[U*u*D])}:{},ae=x&&P(L);return e.op("com.xenova.LlamaPrefillMatmul",{aT:G,bitsT:L.bitsT,scalesT:L.scalesT,yT:H,...te,...o&&(ae||(f?!ce&&(L.format!=="q4_k_native_compact"||y):ce?O(L):R(L)))?{realLenT:o}:{},...L.gridT&&!S&&!(ce&&f)?{gridT:L.gridT}:{}},{args:{M:u,inFeatures:F,outFeatures:D,blockOffset:K,outStride:Y,dstColStart:fe,format:L.format,lut:L.lut,iqRaw:L.iqRaw,...ce&&!ae?{exact:1}:{},...ae?{activationQuant:"int8"}:{},...U>1?{kSplits:U}:{}},...f&&P(L)&&i!==void 0?{tunables:{LUT2_PREFILL_POLICY_ROWS:i}}:{}}).yT},Q=(G,L,K)=>(f||p==="float16"||_(G))&&(G.format!=="q5_k"||K%256===0)&&(!_(G)||K%256===0||!G.iqRaw&&!nx.has(G.format))&&L%64===0&&K%32===0,V=(G,L,K,D,F,H,Y)=>e.op("com.xenova.LlamaPrefillDequant",{bitsT:K.bitsT,scalesT:K.scalesT,denseT:G},{args:{denseElems:L,blockOffset:D,outRows:F,inFeatures:H,dstElemOffset:Y,format:K.format,lut:K.lut}}).denseT,$=(G,L,K,D,F,H,Y,fe)=>{if(L=h?.(L,`layers.${G}.${K}`)??L,Je(t,`layers.${G}.${K}`)){let ce=n(G,[K],K);if(Q(ce,F,H))return N(L,ce,ce.offset(`layers.${G}.${K}`),F,H,Y,F,0);let re=F*H,Z=V(e.scratch(`${fe}.w`,p,[re]),re,ce,ce.offset(`layers.${G}.${K}`),F,H,0);return Un(e,f,p,u,L,Z,0,H,F,Y)}return Un(e,f,p,u,L,r(G),D,H,F,Y)},M=(G,L,K,D,F,H,Y)=>$(G,L,K,D,F,H,e.scratch(Y,p,[u,F]),Y),j=(G,L,K,D,F={gate:"gate_proj",up:"up_proj",width:c,label:""})=>{let H=F.width;L=h?.(L,`layers.${G}.${F.gate}`)??L;let Y=e.scratch(D,p,[u,2*H]);if(Je(t,`layers.${G}.${F.gate}`)){let fe=`layers.${G}.${F.gate}`,ce=`layers.${G}.${F.up}`,re=Nn(t,[fe,ce]),Z=n(G,re?[F.gate,F.up]:[F.gate],`${F.label}gateup`),U=re?Z:n(G,[F.up],`${F.label}gateupU`);if(Q(Z,H,l)&&Q(U,H,l)){if(re&&B1(Z,[fe,ce],[H,H],l)&&Q(Z,2*H,l))return N(L,Z,Z.offset(fe),2*H,l,Y,2*H,0);let ne=N(L,Z,Z.offset(fe),H,l,Y,2*H,0);return N(L,U,U.offset(ce),H,l,ne,2*H,H)}let te=2*H*l,ae=e.scratch(`${D}.w`,p,[te]);return ae=V(ae,te,Z,Z.offset(fe),H,l,0),ae=V(ae,te,U,U.offset(ce),H,l,H*l),Un(e,f,p,u,L,ae,0,l,2*H,Y)}return Un(e,f,p,u,L,r(G),K,l,2*H,Y)},A=Qn(t.config);A&&Pn("MoE FFN layers");let T=Ci(A),E=void 0,X=(G,L,K,D={})=>{if(E&&ka(A,G)){if(D.ffNormed||D.next)throw new Error(`prefill layer ${G}: MoE FFNs take no chained norms`);return E(G,L,K),null}let F=D.ffNormed??e.op("co.huggingface.RMSNorm",{xT:K,wT:r(G),yT:e.scratch(`L${G}.ffnorm`,p,[u,l])},{args:{rows:u,dim:l,eps:d,weightBaseOffset:L.post_attention_layernorm}}).yT,H=e.scratch(`L${G}.mlpact`,p,[u,c]),Y;if(ix({packedQ1:!1,activationDtype:p,rows:u,inFeatures:l,intermediateSize:c})){let ce=a(G,["gate_proj","up_proj"],"gateup");Y=e.op("com.xenova.LlamaPrefillGateUp",{aT:F,bitsT:ce.bitsT,scalesT:ce.scalesT,yT:H,...s?{realLenT:s}:{}},{args:{M:u,inFeatures:l,intermediateSize:c,gateOffset:ce.offset(`layers.${G}.gate_proj`),upOffset:ce.offset(`layers.${G}.up_proj`),format:"q1_0",...m?{activation:m}:{}}}).yT}else{let ce=j(G,F,L.gate_proj,`L${G}.gateup`);Y=e.op("co.huggingface.GatedMlpActivation",{xT:ce,yT:H},{args:{rows:u,mlpInner:c,...m?{activation:m}:{}}}).yT}g&&(Y=g(G,Y));let fe=M(G,Y,"down_proj",L.down_proj,l,c,`L${G}.down`);return z(G,K,fe,L.post_ffn_norm,D.next)},z=(G,L,K,D,F)=>{if(!t.config.use_sandwich_norm)return e.op("com.xenova.Axpy",{xT:L,yT:K},{args:{count:u*l,alpha:1}}),null;if(D===void 0)throw new Error(`prefill layer ${G}: sandwich norm weight missing`);let H=F&&p==="float32"?e.scratch(F.name,"float32",[u,l]):null,Y=e.op("com.xenova.RmsNormResidualAdd",{xT:K,wT:r(G),yT:L,...H?{w2T:r(F.layer),normed2T:H}:{}},{args:{rows:u,dim:l,eps:d,weightBaseOffset:D,...H?{eps2:d,weight2BaseOffset:F.offset}:{}}});return H?Y.normed2T:null};return{dequant:V,fused:N,canFuse:Q,projInto:$,proj:M,gateUp:j,mlpInto:X,residualInto:z}}function lx(e){let{g:t,w:r,weights:n,model:a,ids:s,act:o,H:i,vocab:u,T:l,embedOffset:c}=e;return a.embedBits&&a.embedScales?t.op("com.xenova.LlamaEmbed",{inputT:s,bitsT:r("embed.bits",a.embedBits),scalesT:r("embed.scales",a.embedScales),hiddenT:t.scratch("embed.hidden",o,[l,i])},{args:{hiddenSize:i,vocabSize:u,seqLen:l,format:a.embedFormat??"q8_rows",...a.embedLut!==null&&a.embedLut!==void 0?{lut:a.embedLut}:{}}}).hiddenT:t.op("com.xenova.LlamaEmbed",{inputT:s,weightsT:n,hiddenT:t.scratch("embed.hidden",o,[l,i])},{args:{embedOffset:c,hiddenSize:i,vocabSize:u,seqLen:l}}).hiddenT}function cx(e){let{g:t,w:r,weights:n,model:a,hidden:s,act:o,H:i,vocab:u,T:l,eps:c,finalNormOffset:d,lmHeadOffset:p}=e,f=t.scratch("last_row",o,[i]),m=t.uniform("last_row_uni",32);t.op("com.xenova.StridedCopy",{srcT:s,dstT:f,p:m},{args:{rows:1,srcStride:i,srcStart:(l-1)*i,dstStride:i,dstStart:0,copyCols:i}});let h=t.scratch("next_token","uint32",[1]);if(e.nextTokenPick)return t.op("com.xenova.StridedCopy",{srcT:e.nextTokenPick.tokens,dstT:h,p:e.nextTokenPick.uni},{args:{rows:1,srcStride:1,srcStart:l-1,dstStride:1,dstStart:0,copyCols:1}}),t.output(h,"next_token"),m.name;let g=!1,w=a.runtime.device.features.has("shader-f16");return $i(t,{w:r,model:a,hidden:f,weights:n,ids:h,hiddenSize:i,vocabSize:u,rmsEps:c,finalNormOffset:d,lmHeadOffset:p,q1:g,normedDtype:g&&w?"float16":"float32"}),t.output(h,"next_token"),m.name}var E1=512;function dx(e,t,r,n,a){return!t&&!!(e.lmHeadQ4&&e.lmHeadQ4Scales)&&e.lmHeadScanBits!==2&&e.lmHeadFormat!=="q8_0"&&r>=2&&r<=16&&n%256===0&&a<=256*E1}function Gs(e,t,r){if(!Hn(e))throw new Error("emitLlamaPrefill: only full-causal attention models (plain or dense qk-norm; dense or Q8/Q4/Q4_K/IQ packed o_proj+MLP) are supported");let n=e.config,a=Kn(n),s=_r(n),o=s?$e(n.conv_L_cache,"config.conv_L_cache"):0,i=r,u=n.hidden_size,l=s?Ri(u,o):0,c=n.num_hidden_layers,d=n.num_attention_heads,p=n.num_key_value_heads,f=n.head_dim,m=d*f,h=p*f,g=m+2*h,w=n.intermediate_size,b=n.vocab_size,v=Number(n.rms_norm_eps),y=t.maxLength,S=hd(f),C=64,k="float32",B=e.runtime.device.features.has("shader-f16"),P=bd(e),_=Is(e.runtime.device),x=Mi(e,y),R=n.attention_output_gate==="headwise",O=e.offsets.top,N=new lt,{w:Q,state:V,boundWeights:$,states:M}=vt(N),j=Oi(Q,e),A=Di(N,j,e),T=j("top"),E=U=>{let te=x(U);return{cosT:Q(`${te.name}.cos`,te.table.cos),sinT:Q(`${te.name}.sin`,te.table.sin),rotaryDim:te.rotaryDim}},X=n.layer_rope?null:E(0),z=N.uniform("params",16),G=qi(Q,e),L=null,K=ox(N,i),D=U=>typeof U=="number"?A(U).w:j(U),{dequant:F,fused:H,canFuse:Y,proj:fe,mlpInto:ce,residualInto:re}=ux({g:N,model:e,denseW:D,q4w:G,q1w:L,realRowsT:K,T:i,H:u,inter:w,eps:v,actGemm:P,fusedQ4:_});return{cache:t,config:n,qkNorm:a,hasConv:s,convL:o,T:i,H:u,convStateStride:l,numLayers:c,qHeads:d,kvHeads:p,hd:f,qHidden:m,vocab:b,eps:v,scale:S,wgSize:C,act:k,hasF16:B,actGemm:P,headGate:R,top:O,g:N,w:Q,state:V,boundWeights:$,states:M,layerW:A,topWeights:T,ropeAt:E,sharedRope:X,params:z,realRowsT:K,proj:fe,mlpInto:ce,residualInto:re,qkvMatmul:(U,te,ae)=>{let ne=N.scratch(`L${U}.qkv`,P,[i,g]);if(Je(e,`layers.${U}.q_proj`)){if(!Je(e,`layers.${U}.k_proj`)&&!Je(e,`layers.${U}.v_proj`)){let pe=G(U,["q_proj"],"q");if(pe.format!=="q4_k_native_compact"||te.v_proj!==te.k_proj+h*u)throw new Error("Mixed QKV prefill requires native Q4_K query and contiguous dense K/V");if((i<=16||i>=32)&&P==="float16"&&_)return N.op("com.xenova.LlamaPrefillMatmul",{aT:ae,bitsT:pe.bitsT,scalesT:pe.scalesT,weightsT:A(U).w,yT:ne},{args:{M:i,inFeatures:u,outFeatures:g,outStride:g,blockOffset:pe.offset(`layers.${U}.q_proj`),weightOffset:te.k_proj,denseTailStart:m,dstColStart:0,format:pe.format}}).yT;let me=g*u,se=N.scratch(`L${U}.qkvDense`,P,[me]);return se=F(se,me,pe,pe.offset(`layers.${U}.q_proj`),m,u,0),se=N.op("com.xenova.StridedCopy",{srcT:A(U).w,dstT:se},{args:{rows:1,srcStride:2*h*u,srcStart:te.k_proj,dstStride:me,dstStart:m*u,copyCols:2*h*u}}).dstT,Un(N,_,P,i,ae,se,0,u,g,ne)}let xe=["q_proj","k_proj","v_proj"].map(pe=>`layers.${U}.${pe}`),Me=Nn(e,xe)?G(U,["q_proj","k_proj","v_proj"],"qkv"):null,ye=Me??G(U,["q_proj","k_proj"],"qkv"),Ze=Me??G(U,["v_proj"],"qkvV"),W=[[ye,xe[0],m,0],[ye,xe[1],h,m],[Ze,xe[2],h,m+h]];if(W.every(([pe,,me])=>Y(pe,me,u))&&h%64===0)return Me&&B1(Me,xe,[m,h,h],u)&&Y(Me,g,u)?H(ae,Me,Me.offset(xe[0]),g,u,ne,g,0):W.reduce((pe,[me,se,Te,Ce])=>H(ae,me,me.offset(se),Te,u,pe,g,Ce),ne);let J=g*u,ie=N.scratch(`L${U}.qkvDense`,P,[J]);for(let[pe,me,se,Te]of W)ie=F(ie,J,pe,pe.offset(me),se,u,Te*u);return Un(N,_,P,i,ae,ie,0,u,g,ne)}return Un(N,_,P,i,ae,A(U).w,te.q_proj,u,g,ne)}}}function Ls(e,t,{dspark:r,bidirectional:n=!1}={}){let{cache:a,config:s,qkNorm:o,hasConv:i,convL:u,T:l,H:c,convStateStride:d,numLayers:p,qHeads:f,kvHeads:m,hd:h,qHidden:g,eps:w,scale:b,wgSize:v,actGemm:y,headGate:S,g:C,w:k,state:B,layerW:P,ropeAt:_,sharedRope:x,params:R,proj:O,mlpInto:N,residualInto:Q,qkvMatmul:V}=e;if(r&&!x)throw new Error("emitLlamaPrefill: DSpark needs a single shared RoPE table");if(n&&(r||i))throw new Error("emitLlamaLayerStack: bidirectional blocks are attention-only and take no DSpark taps");let $=r?.tapLayers.length??0,M=y,j=r?C.scratch("dspark.features",M,[l,$*c]):null,A=r?.convSnapshots?B("dspark.conv_snapshots",r.convSnapshots):null,T=Qn(s),E=!!s.use_sandwich_norm&&y==="float32",X=D=>i&&zn(s,D)==="conv"?`L${D}.convnorm`:`L${D}.normed`,z=D=>E&&!ka(T,D)?{layer:D,offset:P(D).off.post_attention_layernorm,name:`L${D}.ffnorm`}:void 0,G=D=>E&&D<p?{layer:D,offset:P(D).off.input_layernorm,name:X(D)}:void 0,L=null,K=null;for(let D=0;D<p;++D){let F=P(D).off;if(j&&r){let H=r.tapLayers.indexOf(D);H>=0&&C.op("com.xenova.StridedCopy",{srcT:t,dstT:j},{args:{rows:l,srcStride:c,srcStart:0,dstStride:$*c,dstStart:H*c,copyCols:c}})}if(i&&zn(s,D)==="conv"){if(!a.convStates)throw new Error("emitLlamaPrefill: conv layer requires cache.convStates");let H=B(`conv.state.${D}`,a.convStateForLayer(D)),Y=L??C.op("co.huggingface.RMSNorm",{xT:t,wT:P(D).w,yT:C.scratch(X(D),y,[l,c])},{args:{rows:l,dim:c,eps:w,weightBaseOffset:F.input_layernorm}}).yT,fe=O(D,Y,"conv_in_proj",F.conv_in_proj,3*c,c,`L${D}.bcx`),ce=C.op("com.xenova.LlamaPrefillConv",{bcxT:fe,weightsT:P(D).w,convStatesT:H,convYT:C.scratch(`L${D}.convY`,y,[l,c]),params:R,...A?{snapshotsT:A}:{}},{args:{hiddenSize:c,convLCache:u,convWeightOffset:F.conv_weight,seqLen:l,...A?{snapshotOffset:D*d,snapshotStride:a.convStates.shape[0]}:{}}}).convYT,re=O(D,ce,"conv_out_proj",F.conv_out_proj,c,c,`L${D}.convout`);K=Q(D,t,re,F.post_operator_norm,z(D))}else{let H=Li(a,D),Y=B(`cache.key.${D}`,H.cacheKeysT),fe=B(`cache.value.${D}`,H.cacheValuesT),ce=L??C.op("co.huggingface.RMSNorm",{xT:t,wT:P(D).w,yT:C.scratch(X(D),y,[l,c])},{args:{rows:l,dim:c,eps:w,weightBaseOffset:F.input_layernorm}}).yT,re=V(D,F,ce),{cosT:Z,sinT:U,rotaryDim:te}=_(D),ae=gd(s,D),ne=a.kvRowsForLayer(D),xe=a.isRingLayer(D)?{cacheRing:1}:{};if(a.isRingLayer(D)&&(!o||r))throw new Error("emitLlamaPrefill: KV rings are wired for the qk-norm cache write without DSpark");let De=S?O(D,ce,"attn_gate",F.attn_gate,f,c,`L${D}.gate`):null;C.op("com.xenova.LlamaPrefillRopeCacheKv",{qkvT:re,cacheKeysT:Y,cacheValuesT:fe,cosT:Z,sinT:U,params:R,...o?{weightsT:P(D).w}:{}},{args:{layer:0,cacheLen:ne,...xe,pastLen:0,seqLen:l,numHeads:f,numKvHeads:m,headDim:h,...te<h?{rotaryDim:te}:{},...o?{qNormOffset:F.q_layernorm,kNormOffset:F.k_layernorm,rmsEps:w}:{}}});let Me=n?px(e,D,re,Y,fe,De):C.op("com.xenova.CachedDecodeAttention",{qkvT:re,cacheKeysT:Y,cacheValuesT:fe,attnT:C.scratch(`L${D}.attn`,y,[l,g]),params:R,...De?{gateT:De}:{},...s.attention_sinks?{sinkT:P(D).w}:{}},{args:{layer:0,cacheLen:ne,...xe,pastLen:0,seqLen:l,numHeads:f,numKvHeads:m,headDim:h,scale:b,workgroupSize:v,...ae>0?{windowSize:ae}:{},...s.attention_sinks?{sinkOffset:F.attn_sinks}:{}}}).attnT,ye=O(D,Me,"o_proj",F.o_proj,c,g,`L${D}.oproj`);K=Q(D,t,ye,F.post_operator_norm,z(D))}L=N(D,F,t,{ffNormed:K,next:G(D+1)})}r&&j&&r.inject(C,{featuresT:j,cosT:x.cosT,sinT:x.sinT,paramsT:R,rows:l,actGemm:y,w:k,state:B})}function px(e,t,r,n,a,s){let{g:o,T:i,qHeads:u,kvHeads:l,hd:c,qHidden:d,scale:p,actGemm:f}=e,m=n.dtype,h=o.op("com.xenova.StridedCopy",{srcT:r,dstT:o.scratch(`L${t}.q`,f,[i,d])},{args:{rows:i,srcStride:d+2*l*c,srcStart:0,dstStride:d,dstStart:0,copyCols:d}}).dstT;f!==m&&(h=o.op("ai.onnx.Cast",{x:h},{attrs:{to:io(m)}}).y);let g=[i,l,c],w=o.op("com.xenova.Attention",{qT:o.view(h,0,m,[i,u,c]),kT:o.view(n,0,m,g),vT:o.view(a,0,m,g),outT:o.scratch(`L${t}.attn`,f,[i,u,c])},{args:{seq:i,qHeads:u,kvHeads:l,headDim:c,scale:p,causal:!1}}).outT;if(s){let b=o.op("ai.onnx.Sigmoid",{x:s}).y;w=o.op("ai.onnx.Mul",{a:w,b:o.view(b,0,b.dtype,[i,u,1])}).c}return o.view(w,0,f,[i,d])}function qs(e,t,r,n,a=!1){let s=Gs(e,t,r),{T:o,H:i,vocab:u,eps:l,act:c,hasF16:d,top:p,g:f,w:m,boundWeights:h,states:g,topWeights:w,realRowsT:b}=s,v=f.stepInput("input_ids","uint32",[o]),y=lx({g:f,w:m,weights:w,model:e,ids:v,act:c,H:i,vocab:u,T:o,embedOffset:p.embed_tokens}),S=a?Ki(f,y,o,i,c):y;Ls(s,S,{dspark:n});let C=f.nodeCount,k;if(n?.allRowsHead){let _=f.scratch("verify_tokens","uint32",[o]),x=null;if(dx(e,!1,o,i,u)){let R=E1,O=f.scratch("V.head.bestValues","float32",[o*R]),N=f.scratch("V.head.bestIds","uint32",[o*R]);f.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:S,bitsT:m("lmhead.q4bits",e.lmHeadQ4),scalesT:m("lmhead.q4scales",e.lmHeadQ4Scales),weightsT:w,bestValuesT:O,bestIdsT:N},{args:{hiddenSize:i,vocabSize:u,lmHeadOffset:0,finalNormOffset:p.model_norm,numBlocks:R,mRows:o,...e.lmHeadFormat?{format:e.lmHeadFormat}:{}}});for(let Q=0;Q<o;++Q)f.op("com.xenova.CandidateArgMax",{valuesT:f.view(O,Q*R,"float32",[R],`V.r${Q}.bestValues`),indicesT:f.view(N,Q*R,"uint32",[R],`V.r${Q}.bestIds`),outT:_},{args:{outputOffset:Q,count:R}})}else for(let R=0;R<o;++R)$i(f,{w:m,model:e,hidden:f.view(S,R*i,c,[i],`V.row${R}`),weights:w,ids:_,hiddenSize:i,vocabSize:u,rmsEps:l,finalNormOffset:p.model_norm,lmHeadOffset:p.lm_head,q1:!1,normedDtype:"float32",nameSuffix:`V.r${R}.`,outputOffset:R});f.output(_,"verify_tokens"),k={tokens:_,uni:f.uniform("next_token_pick_uni",32)}}let B=cx({g:f,w:m,weights:w,model:e,hidden:S,act:c,H:i,vocab:u,T:o,eps:l,finalNormOffset:p.model_norm,lmHeadOffset:p.lm_head,...k?{nextTokenPick:k}:{}});return{graph:f.finish({name:"llama-prefill",params:{T:o}}),weights:h,states:g,paramsName:"params",lastRowUniform:B,...k?{nextTokenPickUniform:k.uni.name}:{},...b?{realRowsInput:b.name}:{},nextTokenTailNodeCount:f.nodeCount-C}}var $n=class extends Qi{writeRunInputs(t,r){let n=this.runtime.host,a=this.stepUniformPack(this.emission.paramsName);n.writeBuffer(this.compiled.uniformBuffer(this.emission.paramsName),0,a({past_len:r,cache_len:this.cache.maxLength,seq_len:this.blockLen,...a.fields.has("real_len")?{real_len:t}:{}})),this.emission.realRowsInput&&n.writeBuffer(this.compiled.tensor(this.emission.realRowsInput).buffer,0,new Uint32Array([t]))}},wd=class extends $n{writeRunInputs(t,r){super.writeRunInputs(t,r);let n=this.runtime.host,a=this.model.config.hidden_size;n.writeBuffer(this.compiled.uniformBuffer(this.emission.lastRowUniform),0,this.stepUniformPack(this.emission.lastRowUniform)({rows:1,copyCols:a,srcStride:a,srcStart:(t-1)*a,dstStride:a,dstStart:0})),this.emission.nextTokenPickUniform&&n.writeBuffer(this.compiled.uniformBuffer(this.emission.nextTokenPickUniform),0,this.stepUniformPack(this.emission.nextTokenPickUniform)({rows:1,copyCols:1,srcStride:1,srcStart:t-1,dstStride:1,dstStart:0}))}},Hi=class extends wd{#e;constructor(t,r,n,a=null){super(t,r,n),this.#e=a?new ji(a,n):null}buildEmission(){return qs(this.model,this.cache,this.blockLen,void 0,this.#e!==null)}writeRunInputs(t,r){super.writeRunInputs(t,r),this.#e?.write(this.runtime,this.compiled,this.cache,t,r)}};var fx=(e,t,r)=>Hn(e)?qs(e,t,r):null,vd=["q6_k","q5_k","q3_k","q2_k","q4_k_native_compact","lut32_compact","lut16_compact"];function Ca(e){let{entriesFor:t,nameToGGUF:r,packSuffixes:n}=e,a=new Set(e.nativeFormats??[]),s=e.nativeSuffixes?new Set(e.nativeSuffixes):void 0,o=e.lutcSuffixes?new Set(e.lutcSuffixes):void 0,i=f=>typeof e.decodeGroups=="function"?e.decodeGroups(f):e.decodeGroups,u=f=>{let m=typeof e.lut2Suffixes=="function"?e.lut2Suffixes(f):e.lut2Suffixes;return m?new Set(m):void 0},l=e.subFourKSuperSuffixes?new Set(e.subFourKSuperSuffixes):void 0,c=e.densePrecision==="source";function d(f,m,h){let g=t(f),w=va(g,A=>h.has(r(A))),b=A=>{let T=r(A.name);return A.id==="lm_head"&&!h.has(T)?h.has("token_embd.weight")&&h.info("token_embd.weight").type===I.Q1_0:h.has(T)&&h.info(T).type===I.Q1_0},{kq4Ids:v,kSuperIds:y,lut32Ids:S,lut16Ids:C,lut32cIds:k,lut16cIds:B,lut2_32Ids:P,lut2_16Ids:_,iqIds:x,iqRawIds:R,excludeKq4Ids:O,forceDenseIds:N,quantBits:Q}=$c(g,h,r,n,{nativeFormats:a,nativeSuffixes:s,lutcSuffixes:o,lut2Suffixes:u(f),subFourKSuperSuffixes:l,decodeGroups:i(f),denseGroups:e.denseGroups,sourcePrecision:c,iqRawTrunkSuffixes:dd(Ue(m),e.gatedQkv,Lr(f))}),V=A=>{let T=r(A.name);return h.has(T)?Gh(h.info(T).type)?.lutId??0:0},$=A=>{for(let T of Ot)if(R.get(T.key).has(A.id))return{buffer:T.key,wordsPerBlock:T.wordsPerBlock,scalesPerBlock:.125};if(v.has(A.id))return{buffer:"kq4",wordsPerBlock:4,scalesPerBlock:2,...c?{scalesF32:!0}:{}};for(let[T,E]of Object.entries(y)){if(!E.has(A.id))continue;let X=jc[T];return{buffer:T,wordsPerBlock:X.wordsPerBlock,scalesPerBlock:X.scalesPerBlock,scalesF32:!0,metadataU32:X.metadataU32}}if(S.has(A.id))return{buffer:"lut32",wordsPerBlock:4,scalesPerBlock:1,scalesF32:!0,lut:V(A)};if(C.has(A.id))return{buffer:"lut16",wordsPerBlock:4,scalesPerBlock:2,scalesF32:!0,lut:V(A)};if(k.has(A.id))return{buffer:"lut32c",wordsPerBlock:4,scalesPerBlock:.375,metadataU32:!0,lut:V(A)};if(B.has(A.id))return{buffer:"lut16c",wordsPerBlock:4,scalesPerBlock:.625,metadataU32:!0,lut:V(A)};if(P.has(A.id)){let T=V(A);return{buffer:T===9?"lut2_128":"lut2_32",wordsPerBlock:2,scalesPerBlock:T===9?.125:.375,metadataU32:!0,lut:T}}return _.has(A.id)?{buffer:"lut2_16",wordsPerBlock:2,scalesPerBlock:.625,metadataU32:!0,lut:V(A)}:x.has(A.id)?{buffer:"iq",wordsPerBlock:8,scalesPerBlock:2}:Mn(A,h,r,n,O,N)?{buffer:"q4",wordsPerBlock:Q,scalesPerBlock:1}:null},M=ki(h,r,f,g,w,Ue(m).features.has("shader-f16"),c),j=c?mi(g.filter(A=>$(A)===null&&!(A.id==="embed_tokens"&&M.embedFormat!==null)&&!(A.id==="lm_head"&&(w||M.headPacked))),A=>{let T=r(A);return h.has(T)?ds(h.info(T).type):"F32"}):{};return id({config:f,device:m,entries:g,...j,emitGraph:e.emitGraph,emitPrefillGraph:e.emitPrefillGraph??fx,prefillBlockLengths:e.prefillBlockLengths,isPacked:A=>$(A)!==null,isQ1:b,packedQuantBits:Q,packSlots:$,decodeGroups:i(f),tieWordEmbeddings:w,embedHeadRoutes:M})}async function p(f,m,h,{onProgress:g,signal:w}){let b=t(h);m.planReads?.(b.map(x=>r(x.name)));let v=b.map(x=>({...x,name:r(x.name)})),y=va(b,x=>m.has(r(x))),S=qc(m,r),C=Oc(m,r),{packs:k,skipDenseIds:B}=await Lh({runtime:f,entries:b,gguf:m,nameToGGUF:r,signal:w},{packSuffixes:n,nativeFormats:a,nativeSuffixes:s,lutcSuffixes:o,lut2Suffixes:u(h),subFourKSuperSuffixes:l,decodeGroups:i(h),denseGroups:e.denseGroups,sourcePrecision:c,iqRawTrunkSuffixes:dd(f.device,e.gatedQkv,Lr(h))}),P=null,_=null;try{let x=va(b,R=>m.has(r(R)));P=await qh({runtime:f,entries:b,gguf:m,nameToGGUF:r,signal:w,sourcePrecision:c},h,x);for(let R of P?.skipDenseIds??[])B.add(R);return _=await Ic({runtime:f,config:h,entries:b,reader:S,source:C,...c?mi(b.filter(R=>!B.has(R.id)&&!(R.id==="lm_head"&&x)),R=>S.describe(R).dtype):{},modelName:e.modelName,onProgress:g,signal:w,skipDenseIds:B,...P?.lmHeadQ4||P?.lmHeadQ8?{canUseQ4LmHead:()=>!1,canUseQ8LmHead:()=>!1}:{}}),e.construct({..._,runtime:f,config:h,packs:k,...P?.embedBits&&P.embedScales?{embedBits:P.embedBits,embedScales:P.embedScales,embedFormat:P.embedFormat,embedLut:P.embedLut??null}:{},...P?.lmHeadQ4&&P.lmHeadQ4Scales?{lmHeadQ4:P.lmHeadQ4,lmHeadQ4Scales:P.lmHeadQ4Scales,lmHeadFormat:P.lmHeadFormat??null,lmHeadLut:P.lmHeadLut??null,lmHeadScanBits:P.lmHeadScanBits??null,lmHeadRescoreFormat:P.lmHeadRescoreFormat??null}:{},...P?.lmHeadQ8&&P.lmHeadQ8Scales?{lmHeadQ8:P.lmHeadQ8,lmHeadQ8Scales:P.lmHeadQ8Scales}:{}})}catch(x){throw await cr(f,[k,P,_]),x}}return{...e.configTensors?{configTensors:e.configTensors}:{},configFromGGUF:e.configFromGGUF,checkSupport:d,loadWeights:p}}var on=class{#e=new Map;#t=!1;#n;constructor(t=r=>r.dispose()){this.#n=t}getOrBuild(t,r){if(this.#t)return Promise.reject(new Error("SessionPool is closed"));let n=this.#e.get(t);if(n?.state==="dispose-failed")return Promise.reject(new Error("SessionPool entry cannot be reused until its failed disposal is retried"));if(n)return n.promise;let a,s=Promise.resolve().then(()=>{if(a.invalidation)throw a.invalidation;return r()}).then(o=>{if(a.invalidation){let i=a.invalidation;try{this.#n(o)}catch(u){throw new AggregateError([i,u],"Invalidated session build and cleanup both failed")}throw i}return a.state="ready",a.session=o,o}).catch(o=>{throw this.#e.get(t)===a&&this.#e.delete(t),o});return a={state:"building",promise:s},this.#e.set(t,a),s}release(t){let r=this.#e.get(t);if(r){if(r.state==="building"){this.#e.delete(t),r.invalidation=new Error("SessionPool entry was released while its session was building");return}try{this.#n(r.session),this.#e.delete(t)}catch(n){throw r.state="dispose-failed",n}}}disposeAll(){this.#r("SessionPool was disposed while its session was building")}close(){this.#t=!0,this.#r("SessionPool was closed while its session was building")}#r(t){let r=[];for(let[n,a]of this.#e){if(a.state==="building"){this.#e.delete(n),a.invalidation=new Error(t);continue}try{this.#n(a.session),this.#e.delete(n)}catch(s){a.state="dispose-failed",r.push(s)}}xt(r,"Multiple pooled sessions failed to dispose")}};var G1="trunk_rows",L1="hidden_states",q1=64;function Pa(e,t,r=q1,{bidirectional:n=!1}={}){let a=Gs(e,t,r),{g:s,T:o,H:i,eps:u,top:l,topWeights:c,boundWeights:d,states:p,realRowsT:f}=a,m=s.stepInput(G1,"float32",[o,i]),h=s.op("com.xenova.StridedCopy",{srcT:m,dstT:s.scratch("trunk.hidden","float32",[o,i])},{args:{rows:o,srcStride:i,srcStart:0,dstStride:i,dstStart:0,copyCols:i}}).dstT;Ls(a,h,{bidirectional:n});let g=s.op("co.huggingface.RMSNorm",{xT:h,wT:c,yT:s.scratch("trunk.out","float32",[o,i])},{args:{rows:o,dim:i,eps:u,weightBaseOffset:l.model_norm}}).yT;return s.output(g,L1),{graph:s.finish({name:"llama-trunk",params:{T:o}}),weights:d,states:p,paramsName:"params",...f?{realRowsInput:f.name}:{}}}var _d=class extends $n{buildEmission(){return Pa(this.model,this.cache,this.blockLen)}enqueueRows(t,r,n,a,s){let o=this.model.config.hidden_size*4,i=this.compiled.tensor(G1),u=this.compiled.tensor(L1);this.runtime.copyBufferToBuffer({src:t,dst:i,byteLength:a*o,srcOffset:(t.byteOffset??0)+n*o,dstOffset:i.byteOffset??0}),this.writeRunInputs(a,s),this.compiled.collector.enqueue(this.steps),this.runtime.copyBufferToBuffer({src:u,dst:r,byteLength:a*o,srcOffset:u.byteOffset??0,dstOffset:(r.byteOffset??0)+n*o})}};function R1(e,t){let r=[];for(let n=e;n>0;n-=r[r.length-1])r.push(jn(n,t));return r}var Vi=class{runtime;config;weights;offsets;denseShards;denseShardIndex;packs;cacheDtype;programCache=new Map;defaultPositionIdsCache=new Map;ropeCache=null;partialRopeCache=null;layerRopeCache;blockLens;#e=null;#t=new on;constructor({runtime:t,config:r,weights:n,offsets:a,denseShards:s,denseShardIndex:o,cacheDtype:i="float32",packs:u={},blockLens:l=[q1]}){this.runtime=t,this.config=Fn(r),this.weights=n,this.offsets=a,this.denseShards=s??[n],this.denseShardIndex=o??{},this.packs=u,this.cacheDtype=i,this.blockLens=l}async encode(t,r,n,{append:a=!1}={}){let s=this.config.hidden_size,o=this.config.max_position_embeddings,i=a?this.#e?.get_seq_length()??0:0;if(!Number.isInteger(r)||r<=0||i+r>o)throw new Error(`a trunk holds ${o} rows: ${r} more after ${i} do not fit`);for(let[p,f]of[["input",t],["output",n]])if(f.dtype!=="float32"||f.shape.length!==2||f.shape[1]!==s||f.shape[0]<r)throw new Error(`trunk ${p} must be float32 [>= ${r}, ${s}], got ${f.dtype} [${f.shape.join(", ")}]`);let u=R1(r,this.blockLens);i+u.reduce((p,f)=>p+f,0)>o&&(u=R1(r,this.blockLens.slice(0,1)));let l=await Promise.all(u.map(p=>this.#n(p))),c=this.#e;a||c.truncate(0);let d=0;for(let p of l){let f=Math.min(p.blockLen,r-d);p.enqueueRows(t,n,d,f,c.get_seq_length()),c.advance(f),d+=f}}async prepare(){for(let t of this.blockLens)await this.#n(t)}#n(t){return this.#t.getOrBuild(t,async()=>{this.#e??=Dt.allocate(this.runtime,this.config,1,this.config.max_position_embeddings,this.cacheDtype);let r=new _d(this,this.#e,t);return await r.build(),r})}dispose(){this.#t.close(),this.#e?.dispose(),this.#e=null,ve([this.denseShards,this.packs,this.ropeCache,this.partialRopeCache,this.layerRopeCache,this.defaultPositionIdsCache]),this.ropeCache=null,this.partialRopeCache=null,this.layerRopeCache=void 0}};function O1(e){let t=e.hidden_size,r=e.intermediate_size,n=e.num_attention_heads*e.head_dim,a=e.num_key_value_heads*e.head_dim,s=[{id:"embed_tokens",name:"model.embed_tokens.weight",length:e.vocab_size*t}];for(let o=0;o<e.num_hidden_layers;++o){let i=`L${o}.attn.qkv`,u=`L${o}.mlp.gate_up`;s.push({id:`layers.${o}.q_proj`,name:`model.layers.${o}.self_attn.q_proj.weight`,length:n*t,packedGroup:i},{id:`layers.${o}.k_proj`,name:`model.layers.${o}.self_attn.k_proj.weight`,length:a*t,packedGroup:i},{id:`layers.${o}.v_proj`,name:`model.layers.${o}.self_attn.v_proj.weight`,length:a*t,packedGroup:i},{id:`layers.${o}.o_proj`,name:`model.layers.${o}.self_attn.o_proj.weight`,length:t*n},{id:`layers.${o}.gate_proj`,name:`model.layers.${o}.mlp.gate_proj.weight`,length:r*t,packedGroup:u},{id:`layers.${o}.up_proj`,name:`model.layers.${o}.mlp.up_proj.weight`,length:r*t,packedGroup:u},{id:`layers.${o}.down_proj`,name:`model.layers.${o}.mlp.down_proj.weight`,length:t*r},{id:`layers.${o}.input_layernorm`,name:`model.layers.${o}.input_layernorm.weight`,length:t},{id:`layers.${o}.post_attention_layernorm`,name:`model.layers.${o}.post_attention_layernorm.weight`,length:t})}return s.push({id:"model_norm",name:"model.norm.weight",length:t},{id:"lm_head",name:"lm_head.weight",length:e.vocab_size*t}),s}function Zi(e,t){return{cache:Dt.allocate(e.runtime,e.config,1,t,e.cacheDtype)}}function Ia(e){return rr(e)||Qn(e)!==null||!!e.attention_sinks||(e.layer_types?.includes("sliding_attention")??!1)||e.hidden_size>128||e.num_hidden_layers>4||e.vocab_size>4096}var gx=8192;function Xi(e){return Math.max(1,Math.min(gx,Ba(e)))}function Ba(e){let{config:t,runtime:r,cacheDtype:n}=e;if(!Ia(t))return t.max_position_embeddings;let a=r.device.limits;if(!a)return t.max_position_embeddings;let s=n==="float16"?2:4,o=t.num_key_value_heads*t.head_dim*s,i=an(t).length,u=$e(a.maxComputeWorkgroupStorageSize,"limits.maxComputeWorkgroupStorageSize"),l=Ps(t)*4,c=i===0||rr(t)||Ta(t)?1/0:Math.floor((u-l)/4),d=typeof a.maxStorageBufferBindingSize=="number"&&a.maxStorageBufferBindingSize>0?a.maxStorageBufferBindingSize:1/0,p=Number.isFinite(d)&&i>0?Math.max(1,Math.floor(d/o)):1/0,f=typeof a.maxBufferSize=="number"&&a.maxBufferSize>0?a.maxBufferSize:1/0,m=ad(t)===null?i:an(t).filter(b=>t.layer_types?.[b]!=="sliding_attention").length,h=Number.isFinite(f)&&m>0?Math.max(1,Math.floor(f/(m*o))):1/0,g=i>0?nd(1,t.num_key_value_heads,t.head_dim,s):1,w=Math.floor(Math.min(c,p,h)/g)*g;return Math.max(1,Math.min(t.max_position_embeddings,w))}function hx(e,t){return t instanceof e.ConfigClass?t:new e.ConfigClass(t)}function M1(e,t,r,n){let a=hx(e,r),{weights:s,offsets:o}=fh({runtime:t,entries:e.expectedWeightEntries(a),stateDict:n,tieWordEmbeddings:a.tie_word_embeddings,modelName:e.MODEL_NAME});return new e({runtime:t,config:a,weights:s,offsets:o})}function W1(e,...t){let r=[];for(let n of t)try{n.release(e)}catch(a){r.push(a)}try{e.dispose()}catch(n){r.push(n)}xt(r,"Generation cache and bound sessions failed to dispose")}var wx=()=>{throw new Error("This feature is not included in this bundle.")},F1=wx;function yd(e,t){if(typeof e!="number"||!Number.isInteger(e)||e<0)throw new Error(`${t} requires a non-negative integer max_new_tokens`)}function Q1(e,t){if(e)throw new Error(`${t} V1 only supports greedy decoding`)}function j1(e,t,r,n){if(!(e instanceof Dt))throw new Error("past_key_values must be a LlamaDynamicCache");if(e.runtime!==t)throw new Error("past_key_values does not belong to this runtime");if(e.batchSize!==n)throw new Error(`past_key_values batch size ${e.batchSize} does not match input batch size ${n}`);let a=an(r),s=[...e.kvLayers,...e.ringLayers].sort((i,u)=>i-u);if(s.length!==a.length||s.some((i,u)=>i!==a[u]))throw new Error(`past_key_values KV layers [${s}] do not match expected [${a}]`);let o=[e.kvLayers.length,n,e.maxLength,r.num_key_value_heads,r.head_dim];if(Os(e.keyStates,o,"past_key_values.keyStates"),Os(e.valueStates,o,"past_key_values.valueStates"),e.ringLength>0){let i=[e.ringLayers.length,n,e.ringLength,r.num_key_value_heads,r.head_dim];Os(e.ringKeyStates,i,"past_key_values.ringKeyStates"),Os(e.ringValueStates,i,"past_key_values.ringValueStates")}_r(r)&&Os(e.convStates,[sd(r,n)],"past_key_values.convStates")}function Os(e,t,r){if(!wt(e)||e.dtype!=="float32"&&e.dtype!=="float16")throw new Error(`${r} must be a float32 or float16 WebGpuTensor`);if(e.shape.length!==t.length||e.shape.some((n,a)=>n!==t[a]))throw new Error(`${r} shape [${e.shape}] does not match expected [${t}]`)}function Ad(e,t,r){let n=0,a=e[t];for(let s=1;s<r;++s){let o=e[t+s];o>a&&(a=o,n=s)}return n}function Sd(e,t){if(e instanceof Uint32Array)return{shape:[1,e.length],data:e};if(!Array.isArray(e))throw new Error(`${t} must be a uint32 tensor, Uint32Array, or array`);let r=kd(e,t);if(r.length===1&&r.unshift(1),r.length!==2)throw new Error(`${t} must be rank 1 or rank 2`);let n=new Uint32Array(ke(r)),a=0,s=o=>{if(Array.isArray(o)){for(let i of o)s(i);return}if(typeof o!="number"||!Number.isInteger(o)||o<0)throw new Error("token ids must be non-negative integers");n[a++]=o};return s(e),{shape:r,data:n}}function kd(e,t){if(!Array.isArray(e))return[];let r=e.length;if(r===0)throw new Error(`${t} arrays must not be empty`);let n=Array.isArray(e[0])?kd(e[0],t):[];for(let a=1;a<e.length;++a){let s=Array.isArray(e[a])?kd(e[a],t):[];if(s.length!==n.length||s.some((o,i)=>o!==n[i]))throw new Error(`${t} must be a rectangular array`)}return[r,...n]}var vx=4,Yi=class{runtime;config;weights;offsets;denseShards;denseShardIndex;lmHeadQ8;lmHeadQ8Scales;lmHeadQ4;lmHeadQ4Scales;lmHeadFormat;lmHeadLut;lmHeadScanBits;lmHeadRescoreFormat;embedBits;embedScales;embedFormat;embedLut;packs;cacheDtype;programCache;defaultPositionIdsCache;llamaForwardLayerOffsetsT;ropeCache;partialRopeCache;layerRopeCache;hadamardSigns;#e=new on;#t=new on(t=>t.disposeAll());#n;mediaRows=null;constructor({runtime:t,config:r,weights:n,offsets:a,denseShards:s,denseShardIndex:o,lmHeadQ8:i=null,lmHeadQ8Scales:u=null,lmHeadQ4:l=null,lmHeadQ4Scales:c=null,lmHeadFormat:d=null,lmHeadLut:p=null,lmHeadScanBits:f=null,lmHeadRescoreFormat:m=null,embedBits:h=null,embedScales:g=null,embedFormat:w=null,embedLut:b=null,cacheDtype:v="float32",packs:y={}}){this.runtime=t,this.config=Fn(r),this.weights=n,this.offsets=a,this.denseShards=s??[n],this.denseShardIndex=o??{},this.lmHeadQ8=i,this.lmHeadQ8Scales=u,this.lmHeadQ4=l,this.lmHeadQ4Scales=c,this.lmHeadFormat=d,this.lmHeadLut=p,this.lmHeadScanBits=f,this.lmHeadRescoreFormat=m,this.embedBits=h,this.embedScales=g,this.embedFormat=w,this.embedLut=b,this.packs=y,this.cacheDtype=v,this.programCache=new Map,this.defaultPositionIdsCache=new Map,this.llamaForwardLayerOffsetsT=null,this.ropeCache=null}static ConfigClass=nn;static MODEL_NAME="Llama";static expectedWeightEntries=O1;static DECODE_GRAPH_EMIT=(t,r)=>Rs(t,r,0);static loadStateDict(t,r,n){return M1(this,t,r,n)}canSample(){return Ia(this.config)&&S1(this)}dispose(){this.#u?.dispose(),this.#u=null,this.#e.disposeAll(),this.#t.disposeAll(),ve(this.denseShards),ve(this.lmHeadQ8),ve(this.lmHeadQ8Scales),ve(this.lmHeadQ4),ve(this.lmHeadQ4Scales),ve(this.embedBits),ve(this.embedScales),ve(this.packs),ve(this.ropeCache),ve(this.partialRopeCache),ve(this.layerRopeCache),this.layerRopeCache=void 0,ve(this.hadamardSigns),this.hadamardSigns=void 0,ve(this.defaultPositionIdsCache),ve(this.llamaForwardLayerOffsetsT),this.programCache.clear(),this.ropeCache=null,this.partialRopeCache=null,this.llamaForwardLayerOffsetsT=null,this.lmHeadQ8=null,this.lmHeadQ8Scales=null,this.lmHeadQ4=null,this.lmHeadFormat=null,this.lmHeadLut=null,this.lmHeadScanBits=null,this.lmHeadRescoreFormat=null,this.lmHeadQ4Scales=null,this.embedBits=null,this.embedScales=null,this.embedFormat=null,this.embedLut=null,this.packs={}}async forward(t={}){return F1(this,t)}async generate(t){let{input_ids:r,max_new_tokens:n,do_sample:a=!1,force_token_path:s=!1}=t;yd(n,"generate"),Q1(a,"LlamaForCausalLM.generate");let{shape:o,data:i}=Sd(r,"input_ids");if(o.length!==2)throw new Error("generate expects input_ids shape [batch, sequence]");let[u,l]=o,c=l+n;if(c>this.config.max_position_embeddings)throw new Error(`generated length ${c} exceeds max_position_embeddings ${this.config.max_position_embeddings}`);let d=new Uint32Array(u*c);for(let m=0;m<u;++m)for(let h=0;h<l;++h)d[m*c+h]=i[m*l+h];if(n===0)return this.runtime.tensorFromTypedArray("uint32",[u,c],d);if(s||Ia(this.config))return this.#a({data:i,batchSize:u,promptLength:l,totalLength:c,maxNewTokens:n});let p=null,f=this.runtime.tensorFromTypedArray("uint32",[u,l],i);try{for(let m=0;m<n;++m){let h=f;if(h===null)throw new Error("generate lost its next input tensor");let g=await this.forward({input_ids:h,past_key_values:p,use_cache:!0,logits_to_keep:1});p=g.past_key_values;let w=new Uint32Array(u);try{let b=await this.runtime.readTensor(g.logits),v=g.logits.shape[1];for(let y=0;y<u;++y){let S=(y*v+v-1)*this.config.vocab_size,C=Ad(_n(b),S,this.config.vocab_size);w[y]=C,d[y*c+l+m]=C}}finally{g.logits.destroy()}h.destroy(),f=null,m+1<n&&(f=this.runtime.tensorFromTypedArray("uint32",[u,1],w))}return this.runtime.tensorFromTypedArray("uint32",[u,c],d)}finally{f?.destroy(),p?.dispose()}}async*streamTokenIds(t){yield*this.#r(t,{fromCache:!1})}createGenerationState(t={}){let r=t.batchSize??t.batch_size??1,n=Ba(this),a=gp(t,Xi(this));if(r!==1)throw new Error("createGenerationState V1 supports batch size 1");if(!Number.isInteger(a)||a<1||a>n)throw new Error(`maxLength must be an integer in [1, ${n}]`);return Zi(this,a)}disposeGenerationState(t){let r=this.specDecodeRunner();W1(t.cache,this.#e,this.#t,...r?[r]:[])}async*streamTokenIdsFromCache(t){yield*this.#r(t,{fromCache:!0})}async*#r(t,{fromCache:r}){let n=r?"streamTokenIdsFromCache":"streamTokenIds",{input_ids:a,max_new_tokens:s,generation_state:o=null,past_key_values:i=null,force_token_path:u=!1}=t,l=r?o?.cache??i:null;yd(s,n);let{eosTokenId:c,stopOnEos:d}=Yn(t,{eosTokenId:this.config.eos_token_id}),p=Qr(t);if(p&&!this.canSample())throw new Error(`LlamaForCausalLM.${n}: this model decodes greedily only`);if(r&&l==null)throw new Error("streamTokenIdsFromCache requires generation_state, cache, or past_key_values");let{shape:f,data:m}=Sd(a,"input_ids");if(f.length!==2)throw new Error(`${n} expects input_ids shape [batch, sequence]`);let[h,g]=f;if(h!==1)throw new Error(`${n} V1 supports batch size 1`);if(r){if(g<1)throw new Error("streamTokenIdsFromCache requires at least one input token");if(j1(l,this.runtime,this.config,h),l.get_seq_length()+g+s>l.maxLength)throw new Error(`cache length ${l.get_seq_length()} + input length ${g} + max_new_tokens ${s} exceeds maxLength ${l.maxLength}`)}else{let b=g+s;if(b>this.config.max_position_embeddings)throw new Error(`generated length ${b} exceeds max_position_embeddings ${this.config.max_position_embeddings}`)}if(s===0)return;if(!u&&!Ia(this.config)){if(p)throw new Error(`LlamaForCausalLM.${n}: this model's forward path decodes greedily only`);yield*this.#s({tokenIds:m,inputLength:g,cache:l,maxNewTokens:s,eosTokenId:c,stopOnEos:d,signal:t.signal??null});return}let w=!r;if(r||(l=Zi(this,g+s).cache),!l)throw new Error(`${n} requires a generation cache`);try{yield*this.#l({tokenIds:m,inputLength:g,cache:l,eosTokenId:c,generationArgs:t,sampling:p})}finally{w&&this.disposeGenerationState({cache:l})}}async#a({data:t,batchSize:r,promptLength:n,totalLength:a,maxNewTokens:s}){if(r!==1)throw new Error("large-model generation V1 supports batch size 1");let o=new Uint32Array(a);if(o.set(t),s===0)return this.runtime.tensorFromTypedArray("uint32",[1,a],o);let i=Zi(this,a),u=0;try{for await(let l of this.#l({tokenIds:o,inputLength:n,cache:i.cache,generationArgs:{max_new_tokens:s,stop_on_eos:!1}}))o[n+u]=l,u+=1}finally{this.disposeGenerationState(i)}return this.runtime.tensorFromTypedArray("uint32",[1,a],o)}async*#s({tokenIds:t,inputLength:r,cache:n=null,maxNewTokens:a,eosTokenId:s,stopOnEos:o,signal:i=null}){let u=n===null,l=this.runtime.tensorFromTypedArray("uint32",[1,r],t);try{for(let c=0;c<a;++c){let d=l;if(d===null)throw new Error("streamForwardGreedy lost its next input tensor");let p=await this.forward({input_ids:d,past_key_values:n,use_cache:!0,logits_to_keep:1});n=p.past_key_values;let f;try{let m=await this.runtime.readTensor(p.logits),h=p.logits.shape[1];f=Ad(_n(m),(h-1)*this.config.vocab_size,this.config.vocab_size)}finally{p.logits.destroy()}if(d.destroy(),l=null,o&&Jn(f,s))return;i?.throwIfAborted(),yield f,c+1<a&&(l=this.runtime.tensorFromTypedArray("uint32",[1,1],new Uint32Array([f])))}}finally{l?.destroy(),u&&n?.dispose()}}async*#l({tokenIds:t,inputLength:r,cache:n,eosTokenId:a,generationArgs:s,sampling:o=null}){let i=this.specDecodeRunner();if(i){if(o)throw new Error("speculative decoding verifies greedy drafts; it cannot sample");for await(let l of i.generate(t.subarray(0,r),n,s,c=>this.#o(c),a??null))s.signal?.throwIfAborted(),yield l;return}let u=this.#g(n,{eosTokenId:a,pipelineDepth:In(s,vx,Ir),sampling:o});yield*Jm(u,t.subarray(0,r),s)}#u=null;attachSpecDecodeRunner(t){if(t&&this.mediaRows)throw new Error("speculative decoding does not take media rows");t!==this.#u&&this.#u?.dispose(),this.#u=t}specDecodeRunner(){return this.#u?.active?this.#u:null}makeGraphDecodeSession(t){return new Ui(this,t,{mediaRows:this.mediaRows!==null})}async prefillTokens(t,r){if(r.length===0)return;let n=t.get_seq_length();if(n+r.length>t.maxLength)throw new Error(`prefill of ${r.length} tokens at ${n} exceeds maxLength ${t.maxLength}`);let a=(this.mediaRows?.pendingRows.get(t)?.size??0)>0;await this.#m(t,r,n,void 0,null,!1),a&&await this.runtime.queueIdle(),t.advance(r.length)}async submitDecodeStep(t,r,{token:n=null,row:a=null,predict:s}){if(!this.mediaRows)throw new Error("submitDecodeStep needs a model loaded with media rows");if(r+1>t.maxLength)throw new Error(`decode step at ${r} exceeds maxLength ${t.maxLength}`);let o=await this.#o(t);return{next:o.submitWithRow(n,a,r,s),hidden:o.hiddenStates}}#o(t){return this.#e.getOrBuild(t,async()=>{let r=this.makeGraphDecodeSession(t);return await r.build(t.get_seq_length(),{fuse:!0}),r})}prefillSupported(){return Hn(this)}makeGraphPrefillSession(t,r){return new Hi(this,t,r,this.mediaRows)}#i(){return this.#n??=this.prefillSupported()}async#c(t,r){return(await this.#t.getOrBuild(t,()=>new on)).getOrBuild(r,async()=>{let a=this.makeGraphPrefillSession(t,r);return await a.build(),a})}async warmup(t,r){await this.#d(t,r),await this.specDecodeRunner()?.warmup(t)}async#d(t,r){if(t.maxLength<3)return;if(await t.mutableStateCheckpoint?.(),await t.alternateMutableStateCheckpoint?.(),!this.#i()){await Bl(this,t,[1]);return}let n=r?.promptTokens!==void 0?this.#p(r.promptTokens,t):Jc((s,o,i)=>this.planPrefillChunks(s,o,i),t.maxLength),a=n.find(s=>s+2<=t.maxLength);if(a!==void 0&&!this.specDecodeRunner()&&await this.#o(t),await Bl(this,t,[a??1]),!(this.specDecodeRunner()||r?.deferRemainingPrefill))for(let s of n)s!==a&&await this.#c(t,s)}async warmRemainingPrefill(t){if(!this.#i()||this.specDecodeRunner())return;let r=Jc((n,a,s)=>this.planPrefillChunks(n,a,s),t.maxLength);for(let n of r)await this.#c(t,n)}#p(t,r){return Yc(this.planPrefillChunks(t,0,r.maxLength))}async#f(t,r,n,a=null,s=!0){let o=await this.#o(t);o.setSampling(a);let i=s?r.length-1:r.length,u=!1;for(let l=0;l<i;++l)o.enqueueStep(r[l],n+l,{full:u});return s?o.submitStep(r[i],n+i):0}prefillMinTokens(){return this.weights.dtype==="uint32"?2:Xc/2}planPrefillChunks(t,r,n){return!this.mediaRows&&t>=16&&this.config.model_type==="llama"&&this.weights.dtype==="float16"&&Is(this.runtime.device)&&!Kn(this.config)&&!_r(this.config)&&!rr(this.config)&&this.config.hidden_size%256===0&&this.config.intermediate_size%256===0?Uh({tokenCount:t,position:r,maxLength:n,shortBlock:16,normalMinTokens:this.prefillMinTokens(),bucketMinTokens:23,bucketMaxTokens:32,bucketBlock:32,fitCache:!0}):Kh({tokenCount:t,position:r,maxLength:n,minGraphTokens:this.mediaRows?2:this.prefillMinTokens(),shortTail:this.shortPrefillBlocks(),fewestRows:this.prefillComputesPaddedRows()})}prefillChunks(t,r,n){return this.#i()?this.planPrefillChunks(t,r,n):[{kind:"replay",tokenCount:t}]}generationCapacity(t,r){let n=i=>this.prefillChunks(t,0,i).reduce((u,l)=>u+(l.kind==="replay"?l.tokenCount:0),0),a=n(Number.MAX_SAFE_INTEGER),s=ja(t,r),o=s;for(;n(o)>a;)++o;return o<=Ba(this)?o:s}prefillComputesPaddedRows(){return bd(this)!=="float32"?!1:!((this.packs?.lut2_32?.length??0)+(this.packs?.lut2_16?.length??0)>0)||Is(this.runtime.device)}shortPrefillBlocks(){return this.weights.dtype==="uint32"}async#m(t,r,n,a,s=null,o=!0){let i=this.prefillChunks(r.length,n,t.maxLength),u=a?.signal??null,l=a?.onProgress??null,c=u!==null||l!==null,d=null,p=0,f=n,m=0;for(let h=0;h<i.length;++h){h>0&&u?.throwIfAborted();let g=i[h],w=r.subarray(p,p+g.tokenCount),b=h===i.length-1,v=b&&o;if(g.kind==="graph"){let y=await this.#c(t,g.blockLen);v?(y.setSampling(v?s:null),m=await y.run(w,f)):y.enqueueBody(w,f)}else m=await this.#f(t,w,f,v?s:null,!b||o);if(p+=g.tokenCount,f+=g.tokenCount,c&&!b){let y=this.runtime.queueIdle(),S=p;y.then(()=>l?.(S),()=>{}),d&&await d,d=y}}return m}#g(t,{eosTokenId:r,pipelineDepth:n,sampling:a}){return{cache:t,eosTokenId:r,pipelineDepth:n,sampling:a,prefill:(s,o,i)=>this.#m(t,s,o,i,a),beginDecode:async()=>{let s=await this.#o(t);return s.setSampling(a),s}}}};var N1=["conv","full_attention","sliding_attention"],Ga=class{vocab_size;hidden_size;intermediate_size;num_hidden_layers;num_attention_heads;num_key_value_heads;head_dim;max_position_embeddings;norm_eps;rms_norm_eps;conv_bias;conv_L_cache;sliding_window;layer_types;rope_parameters_by_type;layer_rope;use_sandwich_norm;use_attention_gate;headless;attention_output_gate;attention_qk_norm;tie_word_embeddings;pad_token_id;bos_token_id;eos_token_id;model_type;constructor(t={}){let r="Lfm3Config";this.headless=t.headless??!1,this.vocab_size=this.headless?0:rt(t,"vocab_size",r),this.hidden_size=rt(t,"hidden_size",r),this.intermediate_size=rt(t,"intermediate_size",r),this.num_hidden_layers=rt(t,"num_hidden_layers",r),this.num_attention_heads=rt(t,"num_attention_heads",r),this.num_key_value_heads=rt(t,"num_key_value_heads",r),this.head_dim=t.head_dim??Math.trunc(this.hidden_size/this.num_attention_heads),this.max_position_embeddings=rt(t,"max_position_embeddings",r),this.norm_eps=t.norm_eps??1e-5,this.rms_norm_eps=this.norm_eps,this.conv_bias=t.conv_bias??!1,this.conv_L_cache=rt(t,"conv_L_cache",r),this.sliding_window=t.sliding_window??void 0,this.layer_types=Vc(t,"layer_types",r),this.rope_parameters_by_type=Vc(t,"rope_parameters",r),this.use_sandwich_norm=t.use_sandwich_norm??!1,this.use_attention_gate=t.use_attention_gate??!0,this.use_attention_gate&&(this.attention_output_gate="headwise"),this.attention_qk_norm=!0,this.tie_word_embeddings=t.tie_word_embeddings??!0,this.pad_token_id=t.pad_token_id,this.bos_token_id=t.bos_token_id,this.eos_token_id=t.eos_token_id,this.model_type="lfm3",this.layer_rope=_x(this),this.validate()}validate(){if(Ti(this,[...this.headless?[]:["vocab_size"],"hidden_size","intermediate_size","num_hidden_layers","num_attention_heads","num_key_value_heads","max_position_embeddings","head_dim","conv_L_cache"],"Lfm3Config"),this.num_attention_heads%this.num_key_value_heads!==0)throw new Error("Lfm3Config num_attention_heads must be divisible by num_key_value_heads");if(this.head_dim%2!==0)throw new Error("Lfm3Config head_dim must be even for RoPE");if(this.layer_types.length!==this.num_hidden_layers)throw new Error("Lfm3Config.layer_types length must match num_hidden_layers");let t=this.layer_types.find(r=>!N1.includes(r));if(t!==void 0)throw new Error(`Lfm3Config.layer_types entry ${JSON.stringify(t)} is not one of ${N1.join(", ")}`);if(this.layer_types.includes("sliding_attention")&&!(Number.isInteger(this.sliding_window)&&this.sliding_window>0))throw new Error("Lfm3Config has sliding_attention layers but no positive integer sliding_window");if(this.conv_bias)throw new Error("Lfm3Config: conv_bias is not supported")}toJSON(){let{layer_rope:t,rope_parameters_by_type:r,...n}=this;return{...n,rope_parameters:r}}};function _x(e){let t={},r=e.head_dim/2;for(let n of new Set(e.layer_types)){if(n==="conv")continue;let a=e.rope_parameters_by_type[n];if(!a)throw new Error(`Lfm3Config.rope_parameters has no entry for ${n}`);let s=a.rope_type??"default";if(s!=="default"&&s!=="proportional")throw new Error(`Lfm3Config: ${n} rope_type ${JSON.stringify(a.rope_type)} is not supported`);if(typeof a.rope_theta!="number"||!(a.rope_theta>0))throw new Error(`Lfm3Config.rope_parameters.${n}.rope_theta must be a positive number`);let o=a.partial_rotary_factor??1,i=a.freq_factors?yx(a.freq_factors,r,n):Math.floor(Math.trunc(o*e.head_dim)/2);if(!(i>0&&i<=r))throw new Error(`Lfm3Config: ${n} partial_rotary_factor ${o} rotates ${i} of ${r} pairs`);t[n]={rope_theta:a.rope_theta,rotated_pairs:i}}return t}function yx(e,t,r){if(e.length!==t)throw new Error(`Lfm3Config: ${r} rope freq_factors has ${e.length} entries, expected ${t}`);let n=e.findIndex(s=>s!==1),a=n<0?t:n;if(e.slice(a).some(s=>!(s>=1e30)))throw new Error(`Lfm3Config: ${r} rope freq_factors must be 1 for rotated pairs then >= 1e30 (identity); got ${JSON.stringify(e)}`);return a}var Ji=Yt(()=>Sx),Ra=class extends Yi{static ConfigClass=Ga;static MODEL_NAME="Lfm3";static expectedWeightEntries=z1;static WEIGHT_PRECISION="source";static prepare=Ji.prepare;static load=Ji.load;static loadGGUFFile=Ji.loadGGUFFile;static loadGGUF=Ji.loadGGUF;async forward(){throw new Error("Lfm3ForCausalLM.forward is not supported; use generate/streamTokenIds")}prefillMinTokens(){return Math.min(super.prefillMinTokens(),8)}shortPrefillBlocks(){return super.shortPrefillBlocks()||Je(this,"layers.0.gate_proj")}};function z1(e){let t=e.hidden_size,r=e.intermediate_size,n=e.headless?[]:[{id:"embed_tokens",name:"model.embed_tokens.weight",length:e.vocab_size*t}];for(let a=0;a<e.num_hidden_layers;++a){let s=`model.layers.${a}`;if(e.layer_types[a]==="conv")n.push(...Qh(e,a));else{let[i,u,l,c,d,p]=Fh(e,a);n.push(i,u,l),e.use_attention_gate&&n.push({id:`layers.${a}.attn_gate`,name:`${s}.self_attn.gate_proj.weight`,length:e.num_attention_heads*t,packedGroup:i.packedGroup}),n.push({...c,name:`${s}.self_attn.o_proj.weight`},{...d,name:`${s}.self_attn.q_norm.weight`},{...p,name:`${s}.self_attn.k_norm.weight`})}let o=`L${a}.mlp.gate_up`;n.push({id:`layers.${a}.gate_proj`,name:`${s}.feed_forward.w1.weight`,length:r*t,packedGroup:o},{id:`layers.${a}.up_proj`,name:`${s}.feed_forward.w3.weight`,length:r*t,packedGroup:o},{id:`layers.${a}.down_proj`,name:`${s}.feed_forward.w2.weight`,length:t*r},{id:`layers.${a}.input_layernorm`,name:`${s}.operator_norm.weight`,length:t},{id:`layers.${a}.post_attention_layernorm`,name:`${s}.ffn_norm.weight`,length:t}),e.use_sandwich_norm&&n.push({id:`layers.${a}.post_operator_norm`,name:`${s}.post_operator_norm.weight`,length:t},{id:`layers.${a}.post_ffn_norm`,name:`${s}.post_ffn_norm.weight`,length:t})}return n.push({id:"model_norm",name:"model.embedding_norm.weight",length:t}),e.headless||n.push({id:"lm_head",name:"lm_head.weight",length:e.vocab_size*t}),n}var Ax=new Set(["gate_proj","up_proj","down_proj","o_proj","conv_out_proj","conv_in_proj","q_proj","k_proj","v_proj","attn_gate"]),xd=["q_proj","k_proj","attn_gate"],eu=["q_proj","k_proj","v_proj","attn_gate"],kx={...tr,q_proj:xd,k_proj:xd,attn_gate:xd,v_proj:["v_proj"]},Ea="rope_freqs.weight",tu={configTensors:e=>e.has(Ea)?[Ea]:[],configFromGGUF:Td,entriesFor:z1,nameToGGUF:_s,packSuffixes:Ax,nativeFormats:vd,decodeGroups:e=>e.use_attention_gate?kx:tr,denseGroups:{q_proj:eu,k_proj:eu,v_proj:eu,attn_gate:eu},densePrecision:"source",emitGraph:Ra.DECODE_GRAPH_EMIT,modelName:Ra.MODEL_NAME,construct:e=>new Ra(e)},Sx=Ca(tu),xx=(e,t)=>Rs(e,t,0,{mediaRows:!0}),Tx=(e,t,r)=>Hn(e)?qs(e,t,r,void 0,!0):null;function ru(e){return Ca({...tu,emitGraph:xx,emitPrefillGraph:Tx,construct:t=>{let r=new Ra(t);return r.mediaRows=e,r}})}function Td(e,t,{headerOnly:r=!1}={}){let n=e.metadata,a=n["general.architecture"];if(a!=="lfm3")throw new Error(`Expected lfm3 GGUF architecture, got ${JSON.stringify(a)}`);let s=(v,y)=>qt(n,`lfm3.${v}`,"lfm3",y),{blockCount:o,layer_types:i,num_key_value_heads:u}=Wh(n,"lfm3"),l=n["lfm3.attention.sliding_window_pattern"]??[];if(!Array.isArray(l)||l.length!==0&&l.length!==o)throw new Error(`lfm3 GGUF: attention.sliding_window_pattern must be a ${o}-entry array`);let c=i.map((v,y)=>v!=="conv"&&l[y]?"sliding_attention":v),d=s("attention.head_count"),p=s("embedding_length"),f=s("attention.key_length",Math.trunc(p/d));if(s("attention.value_length",f)!==f)throw new Error("lfm3 GGUF: value_length must equal key_length");let m=s("rope.dimension_count",f);if(m!==f)throw new Error(`lfm3 GGUF: rope.dimension_count ${m} must equal the head dim ${f}`);let h=s("rope.freq_base"),g;if(e.has(Ea)&&!r){let v=t?.get(Ea);if(!v)throw new Error(`lfm3 GGUF: ${Ea} must be read before the config (configTensors)`);g=Array.from(Zr(e.info(Ea).type,v))}let w=e.has("blk.0.post_attention_norm.weight");if(w!==e.has("blk.0.post_ffw_norm.weight"))throw new Error("lfm3 GGUF: sandwich norms come in pairs");let b=c.findIndex(v=>v!=="conv");return new Ga({vocab_size:qt(n,"lfm3.vocab_size","lfm3",Jl(e)),hidden_size:p,intermediate_size:s("feed_forward_length"),num_hidden_layers:o,num_attention_heads:d,num_key_value_heads:u,head_dim:f,max_position_embeddings:s("context_length"),norm_eps:s("attention.layer_norm_rms_epsilon",1e-5),conv_L_cache:s("shortconv.l_cache"),conv_bias:!1,sliding_window:c.includes("sliding_attention")?s("attention.sliding_window"):void 0,layer_types:c,rope_parameters:{full_attention:{rope_type:"default",rope_theta:h,...g?{freq_factors:g}:{}},sliding_attention:{rope_type:"default",rope_theta:s("rope.freq_base_swa",h)}},use_sandwich_norm:w,use_attention_gate:b<0||e.has(`blk.${b}.attn_gate.weight`),tie_word_embeddings:!e.has("output.weight"),eos_token_id:cs(n),bos_token_id:Zo(n["tokenizer.ggml.bos_token_id"]),pad_token_id:Zo(n["tokenizer.ggml.padding_token_id"])??null})}var Cx=1e6,Px={attn_q:"attn_q",attn_k:"attn_k",attn_v:"attn_v",attn_output:"attn_out",attn_q_norm:"attn_q_norm",attn_k_norm:"attn_k_norm",attn_gate:"attn_gate",attn_norm:"ln1",ffn_norm:"ln2",post_attention_norm:"attn_post_norm",post_ffw_norm:"ffn_post_norm","shortconv.in_proj":"conv_pw1","shortconv.conv":"conv_dw","shortconv.out_proj":"conv_pw2",ffn_gate:"ffn_gate",ffn_up:"ffn_up",ffn_down:"ffn_down"};function Ix(e,t){return r=>{let n=_s(r);if(n==="output_norm.weight")return t;let a=/^blk\.(\d+)\.(.+)\.weight$/.exec(n),s=a?Px[a[2]]:void 0;if(!a||!s)throw new Error(`no mmproj name for LFM3 trunk weight ${r}`);return`${e}.${a[1]}.${s}.weight`}}function Bx(e,t){let{blockPrefix:r}=t;t.validate?.(e);let n=l=>{if(!e.has(l))throw new Error(`${t.modelName}: missing ${l}`);return e.info(l).shape},a=0;for(;e.has(`${r}.${a}.ln1.weight`);)++a;if(a===0)throw new Error(`${t.modelName}: no ${r}.N layers`);let s=Array.from({length:a},(l,c)=>e.has(`${r}.${c}.conv_pw1.weight`)?"conv":t.attentionType),o=s.indexOf(t.attentionType),i=s.indexOf("conv");if(o<0)throw new Error(`${t.modelName}: no attention layer`);let u=n(`${r}.${o}.attn_q_norm.weight`)[0];return new Ga({headless:!0,hidden_size:n(`${r}.0.ln1.weight`)[0],intermediate_size:n(`${r}.0.ffn_gate.weight`)[0],num_hidden_layers:a,num_attention_heads:n(`${r}.${o}.attn_q.weight`)[0]/u,num_key_value_heads:n(`${r}.${o}.attn_k.weight`)[0]/u,head_dim:u,max_position_embeddings:t.capacity,norm_eps:t.normEps(e),conv_L_cache:i<0?3:n(`${r}.${i}.conv_dw.weight`)[1],...t.slidingWindow?{sliding_window:t.slidingWindow(e)}:{},layer_types:s,rope_parameters:{[t.attentionType]:{rope_type:"default",rope_theta:Cx}},use_sandwich_norm:e.has(`${r}.0.attn_post_norm.weight`)||e.has(`${r}.0.ffn_post_norm.weight`),use_attention_gate:e.has(`${r}.${o}.attn_gate.weight`)})}function La(e){return Ca({...tu,configFromGGUF:t=>Bx(t,e),nameToGGUF:Ix(e.blockPrefix,e.finalNorm),emitGraph:e.emitGraph??Pa,emitPrefillGraph:Pa,prefillBlockLengths:e.blockLens?.slice(1)??[],modelName:e.modelName,construct:t=>new Vi({...t,blockLens:e.blockLens})})}function nu(e){if(!Number.isInteger(e)||e<=0)throw new Error(`periodicHannWindow: invalid length ${e}`);let t=new Float64Array(e);for(let r=0;r<e;++r)t[r]=.5-.5*Math.cos(2*Math.PI*r/e);return t}var Cd=1e3,Pd=15;function K1(e){return e>=Cd?Pd+Math.log(e/Cd)*27/Math.log(6.4):3*e/200}function Ex(e){return e>=Pd?Cd*Math.exp(Math.log(6.4)/27*(e-Pd)):200*e/3}function Rx(e,t,r){if(!Number.isInteger(e)||e<=0)throw new Error(`slaneyFilterFreqs: invalid nMels ${e}`);if(!(t>=0)||!(r>t))throw new Error(`slaneyFilterFreqs: invalid range [${t}, ${r}]`);let n=K1(t),a=K1(r),s=new Float64Array(e+2);for(let o=0;o<e+2;++o)s[o]=Ex(n+(a-n)*o/(e+1));return s}function U1(e,t,r,n,a){if(!Number.isInteger(t)||t<2)throw new Error(`slaneyMelFilterbank: invalid nFft ${t}`);if(!Number.isInteger(r)||r<=0)throw new Error(`slaneyMelFilterbank: invalid sampleRate ${r}`);let s=Math.floor(t/2)+1,o=Math.floor(r/2),i=Rx(e,n,a),u=new Float64Array(e*s);for(let l=0;l<e;++l){let c=i[l],d=i[l+1],p=i[l+2],f=2/(p-c);for(let m=0;m<s;++m){let h=m*o/(s-1),g=(h-c)/(d-c),w=(p-h)/(p-d);u[l*s+m]=Math.max(0,Math.min(g,w))*f}}return u}function $1(e,t,r){if(!Number.isInteger(t)||t<=0||!Number.isInteger(r)||r<=0||e.length!==t*r)throw new Error(`packMelBands: filterbank length ${e.length} does not match [${t}, ${r}]`);let n=new Uint32Array(t*2),a=[];for(let s=0;s<t;++s){let o=s*r,i=-1,u=-2;for(let l=0;l<r;++l){let c=e[o+l];if(!Number.isFinite(c)||c<0)throw new Error(`packMelBands: filter ${s} bin ${l} is not a finite nonnegative weight`);if(c>0){if(i<0)i=l;else if(l!==u+1)throw new Error(`packMelBands: filter ${s} is not a single contiguous band`);u=l}}n[s*2]=i<0?0:i,n[s*2+1]=i<0?0:u-i+1;for(let l=i;l<=u;++l)a.push(e[o+l])}return{spans:n,weights:Float32Array.from(a)}}var V1=24e3,Vn=6,Z1=1280,ar=320,At=1280,X1=At-ar,Oa=X1/2,un=Z1/2+1,Id=Float32Array.from(nu(At)),qa=10,au=qa*Vn,ln=At/ar-1,Rd=La({blockPrefix:"a.gen.wav.tfm.blk",finalNorm:"a.gen.wav.tfm.output_norm.weight",modelName:"LiquidCodecDecoder",attentionType:"sliding_attention",slidingWindow:()=>30,normEps:()=>1e-5,capacity:8192}),Gx=Yt(()=>Rd),Lx={projectOut:"a.gen.wav.quant.first.out_proj",fromQuantizer:"a.gen.wav.tfm.in_proj",lin:"a.gen.wav.tfm.out_proj"},Bd="a.gen.wav.quant.first.codebook.weight";async function Y1(e,t){let r=[];try{let{linears:n,tensors:a}=await Aa(e,t,Lx);r.push(...a);let s=(await Wn(e,t,[Bd],()=>!0)).get(Bd),o=pt(e,"float32",[At],()=>Id),i=pt(e,"float32",[1],()=>new Float32Array(1)),u=e.empty("float32",[ln,At],"liquid-codec-context");return r.push(s,o,i,u),{fsq:s,linears:n,window:o,zero:i,context:u,tensors:r}}catch(n){throw ve(r),n}}async function J1(e,t){let r=await Y1(e,t),n=r.linears.projectOut.inFeatures/r.fsq.shape[1];return[u3(r,qa,n).graph,l3(r).graph]}var e3="codes",t3="rows",r3="hidden",n3="real_rows",H1="context",a3="image_shape",s3="block_shape",o3="recip_envelope",Ed="frames",i3="audio",su=class{frames=0;tail=new Float32Array(0);finalized=!1};function u3(e,t,r){let n=new lt,{w:a,boundWeights:s}=vt(n),o=e.fsq.shape[1],i=n.op("ai.onnx.Gather",{data:a("fsq.table",e.fsq),indices:n.stepInput(e3,"int32",[t*r]),output:n.scratch("fsq.levels","float32",[t*r,o])},{attrs:{axis:0}}).output,u=vr(n,a,n.view(i,0,"float32",[t,r*o]),e.linears.projectOut),l=vr(n,a,u,e.linears.fromQuantizer),c=e.linears.fromQuantizer.outFeatures,d=n.scratch("upsampled","float32",[t*Vn,c]);for(let p=0;p<Vn;++p)d=n.op("com.xenova.StridedCopy",{srcT:l,dstT:d},{args:{rows:t,srcStride:c,srcStart:0,dstStride:Vn*c,dstStart:p*c,copyCols:c}}).dstT;return n.output(d,t3),{graph:n.finish({name:"liquid-codec-prologue",params:{frames:t}}),weights:s,meta:void 0}}function l3(e){let t=e.context,r=new lt,{w:n,boundWeights:a}=vt(r),s=au,o=e.linears.lin,i=vr(r,n,r.stepInput(r3,"float32",[s,o.inFeatures]),o),u=(C,k)=>r.op("com.xenova.StridedCopy",{srcT:i,dstT:r.scratch(k,"float32",[s,un])},{args:{rows:s,srcStride:2*un,srcStart:C,dstStride:un,dstStart:0,copyCols:un}}).dstT,l=r.op("ai.onnx.Exp",{x:u(0,"log_magnitude")}).y,c=u(un,"angle"),d=r.op("ai.onnx.Mul",{a:l,b:r.op("ai.onnx.Cos",{x:c}).y}).c,p=r.op("ai.onnx.Mul",{a:l,b:r.op("ai.onnx.Sin",{x:c}).y}).c,f=r.op("ai.onnx.Concat",{a:r.view(d,0,"float32",[s,un,1]),b:r.view(p,0,"float32",[s,un,1]),c:r.scratch("spectrum","float32",[s,un,2])},{attrs:{axis:2}}).c,m=r.op("ai.onnx.DFT",{input:f,output:r.scratch("frames","float32",[s,Z1,1])},{attrs:{inverse:1,onesided:1}}).output,h=r.op("ai.onnx.Mul",{a:r.view(m,0,"float32",[s,At]),b:n("window",e.window)}).c,g=r.op("ai.onnx.Where",{condition:r.stepInput(n3,"bool",[s,1]),x:h,y:n("zero",e.zero),output:r.scratch("windowed.real","float32",[s,At])}).output,w=r.op("ai.onnx.Concat",{a:r.state(H1,"float32",t.shape),b:g,c:r.scratch("windowed.all","float32",[ln+s,At])},{attrs:{axis:0}}).c;r.output(w,Ed);let b=r.op("ai.onnx.Transpose",{x:w,y:r.scratch("columns","float32",[At,ln+s])},{attrs:{perm:[1,0]}}).y,v=s*ar+Oa,y=r.op("ai.onnx.Col2Im",{input:r.view(b,0,"float32",[1,At,ln+s]),image_shape:r.stepInput(a3,"uint32",[2]),block_shape:r.stepInput(s3,"uint32",[2]),output:r.scratch("folded","float32",[1,1,1,v])},{attrs:{strides:[1,ar],pads:[0,ln*ar,0,X1-Oa]}}).output,S=r.op("ai.onnx.Mul",{a:r.view(y,0,"float32",[v]),b:r.stepInput(o3,"float32",[v])}).c;return r.output(S,i3),{graph:r.finish({name:"liquid-codec-epilogue",params:{rows:s}}),weights:a,states:{[H1]:t},meta:void 0}}function qx(e,t){let r=new Float32Array(au*ar+Oa);for(let n=Math.max(0,e-ln);n<e+t;++n)for(let a=0;a<At;++a){let s=(n-e)*ar+a;s>=0&&s<r.length&&(r[s]+=Id[a]*Id[a])}for(let n=0;n<r.length;++n)r[n]=r[n]>0?1/r[n]:0;return r}var ou=class e{#e;#t;#n;#r;#a=null;#s=Promise.resolve();numCodebooks;codebookSize;constructor(t,r,n){this.#e=t,this.#t=r,this.#n=n,this.#r=new tn(t),this.codebookSize=r.fsq.shape[0],this.numCodebooks=r.linears.projectOut.inFeatures/r.fsq.shape[1]}static async loadGGUF(t,r){if(!r.has(Bd))throw new Error("LiquidCodecDecoder: the GGUF holds no Liquid codec decoder (a.gen.wav.*)");let n=await Gx.loadGGUF(t,{gguf:r}),a=null;try{a=await Y1(t,r);let s=a.linears.fromQuantizer.outFeatures;if(s!==n.config.hidden_size)throw new Error(`LiquidCodecDecoder: from_quantizer writes ${s} features; the trunk is ${n.config.hidden_size} wide`);return new e(t,a,n)}catch(s){throw ve(a?.tensors??[]),n.dispose(),s}}createStreamingCache(){return new su}decode(t,{cache:r,finalize:n=!0}={}){let a=this.numCodebooks;if(t.length===0&&!r||t.length%a!==0)throw new Error(`LiquidCodecDecoder: ${t.length} codes are not whole ${a}-code frames`);let s=t.findIndex(u=>!(u>=0&&u<this.codebookSize));if(s>=0)throw new Error(`LiquidCodecDecoder: code ${t[s]} at ${s} is outside [0, ${this.codebookSize})`);let o=r??new su;if(o.finalized)throw new Error("LiquidCodecDecoder: the stream was finalized");let i=this.#s.then(()=>this.#l(t,o,n));return this.#s=i.catch(()=>{}),i}async#l(t,r,n){let a=this.numCodebooks;if(r.frames>0&&this.#a!==r)throw new Error("LiquidCodecDecoder: another stream has run since this one; start a new one");this.#a=r;let s=await this.#r.acquire("prologue",()=>u3(this.#t,qa,a)),o=await this.#r.acquire("epilogue",()=>l3(this.#t)),i=this.#t.context,u=this.#r,l=At*4,c=[];for(let p=0;p<t.length/a;p+=qa){let f=Math.min(qa,t.length/a-p),m=r.frames;m===0&&this.#e.host.writeBuffer(i.buffer,i.byteOffset??0,new Float32Array(ln*At));let h=new Int32Array(qa*a);h.set(t.subarray(p*a,(p+f)*a)),u.write(s,e3,h),u.enqueue(s);let g=f*Vn;await this.#n.encode(s.compiled.tensor(t3),g,o.compiled.tensor(r3),{append:m>0}),u.write(o,n3,Uint32Array.from({length:au},(w,b)=>b<g?1:0)),u.write(o,a3,new Uint32Array([1,au*ar+Oa])),u.write(o,s3,new Uint32Array([1,At])),u.write(o,o3,qx(m*Vn,g)),u.enqueue(o),c.push({first:m*Vn,count:g,audio:u.read(o,i3)}),this.#e.copyBufferToBuffer({src:o.compiled.tensor(Ed),dst:i,byteLength:ln*l,srcOffset:(o.compiled.tensor(Ed).byteOffset??0)+g*l,dstOffset:i.byteOffset??0}),r.frames+=f}let d=[];for(let{first:p,count:f,audio:m}of c){let h=await m;d.push(h.slice(p===0?Oa:0,f*ar)),r.tail=h.slice(f*ar,f*ar+Oa)}return n&&(d.push(r.tail),r.finalized=!0,this.#a===r&&(this.#a=null)),Wa(d,Float32Array)}dispose(){this.#r.dispose(),this.#n.dispose(),ve(this.#t.tensors)}};var Ox={blockPrefix:"a.blk",finalNorm:"a.post_ln.weight",modelName:"Lfm3AudioEncoder",validate:e=>{let t=e.metadata;if(t["general.architecture"]!=="clip"||t["clip.has_audio_encoder"]!==!0)throw new Error("LFM3-Omni speech encoder: expected an mmproj GGUF with an audio encoder");if(t["clip.audio.projector_type"]!=="lfm3a")throw new Error(`LFM3-Omni speech encoder: projector type ${JSON.stringify(t["clip.audio.projector_type"])} is not lfm3a`)},attentionType:"sliding_attention",slidingWindow:e=>qt(e.metadata,"clip.audio.window_size","LFM3-Omni mmproj"),normEps:e=>qt(e.metadata,"clip.audio.attention.layer_norm_epsilon","LFM3-Omni mmproj"),capacity:1024,blockLens:[64,128,256]},iu=La(Ox);var Da=16e3,cn=400,uu=160,Dx=1e-12,Mx=4,Wx=4,Fx=8,Qx=1e-5,Gd=2*Da,d3=32*Da;function p3(){let e=[Gd];for(;e[e.length-1]<d3;)e.push(2*e[e.length-1]);return e}function jx(e){let t=p3().find(r=>r>=e);if(t===void 0)throw new Error(`LFM3-Omni speech encoder: a clip holds at most ${d3/Da} s, got ${(e/Da).toFixed(2)} s`);return t}function cu(e){if(e<=cn)return 1;let t=(e-cn)%uu;return 1+(e+(t===0?0:uu-t)-cn)/uu}function Ds(e){return Math.ceil(cu(e)/Fx)}var c3={convIn:0,convMid:2,linMid:3,convEnd:5,linEnd:6},Nx={linOut:"a.pre_encode.out",linear1:"mm.a.mlp.1",linear2:"mm.a.mlp.3"},lu="mm.a.mlp.0";function f3(e){let t=r=>qt(e.metadata,`clip.audio.${r}`,"LFM3-Omni mmproj");return{numMelBins:t("num_mel_bins"),convChannels:e.info("a.conv1d.0.weight").shape[0],projectionDim:t("projection_dim")}}async function m3(e,t,r){let n=[],a=s=>(n.push(s),s);try{let s=$1(U1(t,cn,Da,0,Da/2),t,cn/2+1),o={window:a(pt(e,"float32",[cn],()=>Float32Array.from(nu(cn)))),spans:a(pt(e,"uint32",[s.spans.length],()=>s.spans)),bandWeights:a(pt(e,"float32",[s.weights.length],()=>s.weights))},{linears:i,tensors:u}=await Aa(e,r,Nx);n.push(...u);let l=[...Object.values(c3).flatMap(p=>[`a.conv1d.${p}.weight`,`a.conv1d.${p}.bias`]),`${lu}.weight`,`${lu}.bias`],c=await Wn(e,r,l,p=>p.startsWith("a.conv1d.")&&p.endsWith(".weight"));c.forEach(a);let d=p=>({weight:c.get(`${p}.weight`),bias:c.get(`${p}.bias`)});return{mel:o,convs:Object.fromEntries(Object.entries(c3).map(([p,f])=>[p,d(`a.conv1d.${f}`)])),linears:i,projectorNorm:d(lu),tensors:n}}catch(s){for(let o of n)o.destroy();throw s}}var g3="audio",zx=["mel_mask","conv_in_mask","conv_mid_mask"],h3="rows",w3="encoded",b3="audio_rows";function v3(e,t,r){let n=new lt,{w:a,boundWeights:s}=vt(n),o=cu(r),i=n.op("com.xenova.LogMelSpectrogram",{audioT:n.stepInput(g3,"float32",[r]),windowT:a("mel.window",t.mel.window),melSpansT:a("mel.spans",t.mel.spans),melWeightsT:a("mel.band_weights",t.mel.bandWeights),outT:n.scratch("mel","float32",[e.numMelBins,o])},{args:{n_fft:cn,hop:uu,n_mels:e.numMelBins,n_frames:o,audio_length:r,center:!1,clamp_to_peak:!1,log_floor:Dx,post_add:Mx,post_scale:1/Wx}}).outT,u=e.convChannels,l=(h,g,w,b,v)=>{let[,,y,S]=g.shape,C=w===2?[1,u,Math.ceil(y/2),Math.ceil(S/2)]:[1,u,y,S],k=t.convs[h],B=n.op("ai.onnx.Conv",{x:g,w:a(`${h}.weight`,k.weight),bias:a(`${h}.bias`,k.bias),y:n.scratch(`${h}.out`,"float32",C)},{attrs:{strides:[w,w],pads:w===2?[1,1,1,1]:[0,0,0,0],group:b?u:1}}).y;return v?n.op("com.xenova.Silu",{xT:B},{args:{count:C.reduce((P,_)=>P*_,1)}}).xT:B},c=(h,g)=>n.op("ai.onnx.Mul",{a:h,b:n.stepInput(g,"float32",[1,1,1,h.shape[3]])}).c,d=c(n.view(i,0,"float32",[1,1,e.numMelBins,o],"mel.nchw"),"mel_mask");d=c(l("convIn",d,2,!1,!0),"conv_in_mask"),d=l("convMid",d,2,!0,!1),d=c(l("linMid",d,1,!1,!0),"conv_mid_mask"),d=l("convEnd",d,2,!0,!1),d=l("linEnd",d,1,!1,!0);let[,,p,f]=d.shape,m=n.op("ai.onnx.Transpose",{x:n.view(d,0,"float32",[u*p,f],"front.flat"),y:n.scratch("front.tokens","float32",[f,u*p])},{attrs:{perm:[1,0]}}).y;return n.output(vr(n,a,m,t.linears.linOut),h3),{graph:n.finish({name:"lfm3-audio-front",params:{numSamples:r,frames:o,rows:f}}),weights:s,meta:void 0}}function _3(e,t,r){let n=new lt,{w:a,boundWeights:s}=vt(n),o=n.op("ai.onnx.LayerNormalization",{x:n.stepInput(w3,"float32",[t,r]),scale:a("projector.norm.weight",e.projectorNorm.weight),b:a("projector.norm.bias",e.projectorNorm.bias)},{attrs:{axis:-1,epsilon:Qx}}).y,i=vr(n,a,o,e.linears.linear1,{gelu:"erf"});return n.output(vr(n,a,i,e.linears.linear2),b3),{graph:n.finish({name:"lfm3-audio-projector",params:{rows:t}}),weights:s,meta:void 0}}var Kx=Yt(()=>iu);async function y3(e,t){let r=f3(t),n=await m3(e,r.numMelBins,t),a=t.info(`${lu}.weight`).shape[0];return p3().flatMap(s=>[v3(r,n,s).graph,_3(n,Ds(s),a).graph])}var du=class e{#e;config;#t;#n;#r;constructor(t,r,n,a){this.#e=t,this.config=r,this.#t=n,this.#n=a,this.#r=new tn(t,{maxSessions:4})}static async load(t,r){let n=f3(r),a=await Kx.loadGGUF(t,{gguf:r});try{return new e(t,n,await m3(t,n.numMelBins,r),a)}catch(s){throw a.dispose(),s}}async encode(t){if(t.length===0)throw new Error("LFM3-Omni speech encoder: empty clip");let r=Ds(t.length),n=this.#n.config.hidden_size,a=jx(t.length),s=Ds(a),o=await this.#r.acquire(`front:${a}`,()=>v3(this.config,this.#t,a)),i=await this.#r.acquire(`projector:${s}`,()=>_3(this.#t,s,n)),u=new Float32Array(a);u.set(t),this.#r.write(o,g3,u);let l=cu(t.length),c=cu(a);for(let p of zx)this.#r.write(o,p,Float32Array.from({length:c},(f,m)=>m<l?1:0)),l=Math.ceil(l/2),c=Math.ceil(c/2);this.#r.enqueue(o),await this.#n.encode(o.compiled.tensor(h3),r,i.compiled.tensor(w3)),this.#r.enqueue(i);let d=this.#e.empty("float32",[r,this.config.projectionDim],"lfm3-audio-rows");return await this.#e.copyBufferToBuffer({src:i.compiled.tensor(b3),dst:d,byteLength:d.byteLength}),d}prepare(){return this.#n.prepare()}dispose(){this.#r.dispose(),this.#n.dispose(),ve(this.#t.tensors)}};var Ux=[4,4,4],Se=12,k3=e=>qt(e.metadata,"clip.gen.audio.attention.layer_norm_epsilon","LFM3-Omni mmproj"),Md=La({blockPrefix:"a.gen.code.blk",finalNorm:"a.gen.code.output_norm.weight",modelName:"Lfm3DepthDecoder",attentionType:"full_attention",normEps:k3,capacity:Se,emitGraph:(e,t)=>Pa(e,t,Se,{bidirectional:!0})}),$x=Yt(()=>Md),Ms=class{settings;seed;lastSeen;#e=0;constructor(t,r,n){let{temperature:a,repetition_penalty:s,repetition_window:o}=t;if(t.do_sample&&!(a>0))throw new Error(`Lfm3DepthSampler: temperature must be positive, got ${a}`);if(!(s>0))throw new Error(`Lfm3DepthSampler: repetition_penalty must be positive, got ${s}`);if(!Number.isInteger(o)||o<0)throw new Error(`Lfm3DepthSampler: repetition_window must be a non-negative integer, got ${o}`);this.settings=t,this.seed=n>>>0,this.lastSeen=new Float32Array(Se*r).fill(-1/0)}get recorded(){return this.#e}windowStart(t){let{repetition_window:r,repetition_penalty:n}=this.settings;return r===0||n===1?1/0:t-r}record(t){let r=this.lastSeen.length/Se;for(let n=0;n<Se;++n)this.lastSeen[n*r+t[n]]=this.#e;++this.#e}},Dr="a.gen.code";async function A3(e,t,r,n,a,s){let o=`${Dr}.${r}.weight`;if(t.info(o).type===I.Q8_0){let{tensors:u,locations:l}=await Hc(e,t,[{id:r,name:o,length:n*a,shape:[n,a]}],new Set([r]));s.push(...u);let{bits:c,scales:d,blockOffset:p}=l.get(r);return{rows:n,width:a,format:"q8_0",bits:c,scales:d,blockOffset:p}}let i=(await Wn(e,t,[o])).get(o);return s.push(i),{rows:n,width:a,format:"f32",dense:i}}async function S3(e,t,r,n){let a=[];try{let{linears:s,tensors:o}=await Aa(e,t,{projIn:`${Dr}.proj_in`});a.push(...o);let i=Se*n,u=s.projIn.inFeatures,l=await A3(e,t,"embd",i,r,a),c=await A3(e,t,"out_embd",i,u,a),d=await Wn(e,t,[`${Dr}.mask_embd`,`${Dr}.head_norm.weight`]);a.push(...d.values());let p=pt(e,"uint32",[Se],()=>Uint32Array.from({length:Se},(h,g)=>g*n)),f=pt(e,"uint32",[Se],()=>Uint32Array.from({length:Se},(h,g)=>g)),m=pt(e,"float32",[Se],()=>new Float32Array(Se).fill(1));return a.push(p,f,m),{projIn:s.projIn,codes:l,frameInput:c,maskEmbedding:d.get(`${Dr}.mask_embd`),headNorm:d.get(`${Dr}.head_norm.weight`),codebookStarts:p,codebookRoutes:f,unitWeights:m,tensors:a}}catch(s){for(let o of a)o.destroy();throw s}}function x3(e,t,r,n,a,s){let o=e.scratch(`${n}.rows`,"float32",[s,r.width]),i={hiddenSize:r.width,vocabSize:r.rows,seqLen:s};return r.format==="f32"?e.op("com.xenova.LlamaEmbed",{inputT:a,weightsT:t(`${n}.table`,r.dense),hiddenT:o},{args:{...i,embedOffset:0}}).hiddenT:e.op("com.xenova.LlamaEmbed",{inputT:a,bitsT:t(`${n}.bits`,r.bits),scalesT:t(`${n}.scales`,r.scales),hiddenT:o},{args:{...i,embedOffset:r.blockOffset,format:"q8_0"}}).hiddenT}var T3="hidden",pu="depth_base",Wd="codes",C3="revealed",P3="pick",fu="last_seen",I3="sample_control",B3="sample_draw",Fd="codes_out",Ld="frame_codes",qd="record_index",Od="frame_embedding";function E3(e){let t=new lt,{w:r,boundWeights:n}=vt(t),a=vr(t,r,t.stepInput(T3,"float32",[1,e.projIn.inFeatures]),e.projIn);return t.output(a,pu),{graph:t.finish({name:"lfm3-depth-projection",params:{}}),weights:n,meta:void 0}}function R3(e,t){let r=new lt,{w:n,boundWeights:a}=vt(r),s=r.stepInput(Ld,"uint32",[Se]),o=x3(r,n,e.frameInput,"frame_input",s,Se),i=r.op("ai.onnx.ReduceSum",{x:o,y:r.scratch("frame.sum","float32",[1,e.frameInput.width])},{attrs:{axes:[0],keepdims:1}}).y;r.output(i,Od);let u=r.state(fu,"float32",t.shape),l=r.op("ai.onnx.ScatterElements",{data:u,indices:r.view(s,0,"int32",[Se]),updates:r.stepInput(qd,"float32",[Se]),output:r.scratch("frame.last_seen","float32",t.shape)},{attrs:{axis:0}}).output;return r.op("com.xenova.StridedCopy",{srcT:l,dstT:u},{args:{rows:1,srcStride:t.size,srcStart:0,dstStride:t.size,dstStart:0,copyCols:t.size}}),{graph:r.finish({name:"lfm3-depth-frame-embedding",params:{}}),weights:a,states:{[fu]:t},meta:void 0}}function G3(e,t,r,n){let a=Se,s=r.codes.width,o=x3(e,t,r.codes,"codes",n,a),i=e.op("ai.onnx.Where",{condition:e.stepInput(C3,"bool",[a,1]),x:o,y:t("mask_embedding",r.maskEmbedding),output:e.scratch("slots.codes","float32",[a,s])}).output;return e.op("ai.onnx.Add",{a:e.stepInput(pu,"float32",[a,s]),b:i,c:e.scratch("depth.hidden","float32",[a,s])}).c}function L3(e,t,r,n,a,s,o){let i=Se,u=r.codes.width,l=e.op("co.huggingface.RMSNorm",{xT:a,yT:e.scratch("depth.head_rms","float32",[i,u])},{args:{rows:i,dim:u,eps:o}}).yT,c=e.op("ai.onnx.Mul",{a:l,b:e.view(t("head_norm",r.headNorm),0,"float32",[i,u])}).c,d=r.codes,p=d.rows/i,f;if(d.format==="q8_0")f=e.scratch("depth.logits","float32",[i,p]),e.op("com.xenova.LlamaPrefillMoeDown",{intermediateT:c,bitsT:t("codes.bits",d.bits),scalesT:t("codes.scales",d.scales),routeIdxT:t("codebook_routes",r.codebookRoutes),routeWT:t("unit_weights",r.unitWeights),yT:f},{args:{hiddenSize:p,moeIntermediateSize:u,seqLen:i,numExperts:i,topK:1,downOffset:d.blockOffset,format:"q8_0"}});else{let w=e.op("com.xenova.LlamaPrefillMatmul",{aT:c,weightsT:t("codes.table",d.dense),yT:e.scratch("depth.logits","float32",[i,d.rows])},{args:{M:i,inFeatures:u,outFeatures:d.rows,outStride:d.rows,dstColStart:0,weightOffset:0}}).yT;f=e.op("com.xenova.StridedCopy",{srcT:w,dstT:e.scratch("depth.own","float32",[i,p])},{args:{rows:i,srcStride:d.rows+p,srcStart:0,dstStride:p,dstStart:0,copyCols:p}}).dstT}let m=e.op("com.xenova.SampleRows",{logitsT:f,lastSeenT:e.view(e.state(fu,"float32",n.shape),0,"float32",[i,p]),controlT:e.stepInput(I3,"float32",[3]),drawT:e.stepInput(B3,"uint32",[2]),idsT:e.scratch("depth.picks","uint32",[i])}).idsT,h=e.op("ai.onnx.Add",{a:m,b:t("codebook_starts",r.codebookStarts)}).c;return e.op("ai.onnx.Where",{condition:e.stepInput(P3,"bool",[i]),x:h,y:s,output:e.scratch("depth.codes","uint32",[i])}).output}function Hx(e,t,r,n){let a=Se,s=Gs(e,t,a),{g:o,w:i,H:u,eps:l,top:c,topWeights:d,boundWeights:p,states:f,realRowsT:m}=s,h=o.stepInput(Wd,"uint32",[a]),g=G3(o,i,r,h);Ls(s,g,{bidirectional:!0});let w=o.op("co.huggingface.RMSNorm",{xT:g,wT:d,yT:o.scratch("depth.normed","float32",[a,u])},{args:{rows:a,dim:u,eps:l,weightBaseOffset:c.model_norm}}).yT;return o.output(L3(o,i,r,n,w,h,l),Fd),{graph:o.finish({name:"lfm3-depth-pass",params:{T:a}}),weights:p,states:{...f,[fu]:n},paramsName:"params",...m?{realRowsInput:m.name}:{}}}async function q3(e,t){let[,r,n]=t.info(`${Dr}.embd.weight`).shape,a=await S3(e,t,n,r),s=e.empty("float32",[Se*r],"lfm3-depth-last-seen"),o=new lt,{w:i}=vt(o),u=o.stepInput(Wd,"uint32",[Se]),l=G3(o,i,a,u),c=k3(t),d=o.op("co.huggingface.RMSNorm",{xT:l,wT:i("final_norm",e.empty("float32",[n],"lfm3-depth-final-norm")),yT:o.scratch("depth.normed","float32",[Se,n])},{args:{rows:Se,dim:n,eps:c,weightBaseOffset:0}}).yT;return o.output(L3(o,i,a,s,d,u,c),Fd),[E3(a).graph,R3(a,s).graph,o.finish({name:"lfm3-depth-pass-head",params:{}})]}var Dd=class extends $n{#e;#t;constructor(t,r,n,a){super(t,r,Se),this.#e=n,this.#t=a}buildEmission(){return Hx(this.model,this.cache,this.#e,this.#t)}tensor(t){return this.compiled.tensor(t)}write(t,r){this.runtime.host.writeBuffer(this.compiled.tensor(t).buffer,0,r)}enqueuePass(t,r){this.write(C3,t),this.write(P3,r),this.writeRunInputs(Se,0),this.compiled.collector.enqueue(this.steps)}},mu=class e{#e;#t;#n;#r;#a;#s=null;numCodebooks=Se;codebookSize;hiddenSize;sampling;constructor(t,r,n,a,s){this.#e=t,this.#t=r,this.#n=n,this.#r=new tn(t),this.#a=t.empty("float32",[Se*a],"lfm3-depth-last-seen"),this.codebookSize=a,this.hiddenSize=r.projIn.inFeatures,this.sampling=s}static async loadGGUF(t,r){if(!r.has(`${Dr}.embd.weight`))throw new Error("Lfm3DepthDecoder: the mmproj holds no depth decoder (a.gen.code.*)");let n=r.info(`${Dr}.embd.weight`).shape;if(n.length!==3||n[0]!==Se)throw new Error(`Lfm3DepthDecoder: codebook table is [${n}], expected [${Se}, codes, width]`);let a=await $x.loadGGUF(t,{gguf:r});try{if(a.config.hidden_size!==n[2])throw new Error(`Lfm3DepthDecoder: the codebook table is ${n[2]} wide; the trunk is ${a.config.hidden_size}`);let s=(u,l)=>qt(r.metadata,`clip.gen.audio.sampling.${u}`,"LFM3-Omni mmproj",l),o=s("temp",1),i={do_sample:o>0,temperature:o,repetition_penalty:s("penalty_repeat",1),repetition_window:s("penalty_last_n",64)};return new e(t,await S3(t,r,n[2],n[1]),a,n[1],i)}catch(s){throw a.dispose(),s}}async submitFrame(t,r,n,a,s){let o=Se;if(r.length!==o||new Set(r).size!==o||r.some(b=>!(b>=0&&b<o)))throw new Error(`Lfm3DepthDecoder: order must be a permutation of 0..${o-1}`);let i=this.#e,u=await this.#r.acquire("projection",()=>E3(this.#t)),l=await this.#u(),c=await this.#l(),{settings:d}=n;l.write(I3,Float32Array.of(n.windowStart(s),d.repetition_penalty,d.do_sample?d.temperature:0)),l.write(B3,Uint32Array.of(n.seed,a));let p=this.hiddenSize*4;i.copyBufferToBuffer({src:t,dst:u.compiled.tensor(T3),byteLength:p}),this.#r.enqueue(u);let f=l.tensor(pu);i.copyBufferToBuffer({src:u.compiled.tensor(pu),dst:f,byteLength:f.byteLength});let m=l.tensor(Wd);i.host.writeBuffer(m.buffer,0,Uint32Array.from({length:o},(b,v)=>v*this.codebookSize));let h=new Uint32Array(o),g=0;for(let b of Ux){let v=r.slice(g,g+=b),y=new Uint32Array(o);for(let S of v)y[S]=1;l.enqueuePass(h,y),i.copyBufferToBuffer({src:l.tensor(Fd),dst:m,byteLength:o*4});for(let S of v)h[S]=1}let w=i.readTensor(m).then(b=>Int32Array.from(b,(v,y)=>v-y*this.codebookSize));return i.copyBufferToBuffer({src:m,dst:c.compiled.tensor(Ld),byteLength:o*4}),this.#r.write(c,qd,new Float32Array(o).fill(s)),this.#r.enqueue(c),{codes:w,row:c.compiled.tensor(Od)}}async embedFrame(t){let r=await this.#l();return this.#r.write(r,Ld,Uint32Array.from({length:Se},(n,a)=>t[a]+a*this.codebookSize)),this.#r.write(r,qd,new Float32Array(Se).fill(-1/0)),this.#r.enqueue(r),r.compiled.tensor(Od)}async embedFrames(t){let r=Se,n=t.length/r;if(!Number.isInteger(n)||n===0)throw new Error(`Lfm3DepthDecoder: ${t.length} codes are not whole ${r}-code frames`);let a=this.#e.empty("float32",[n,this.hiddenSize],"lfm3-frame-rows"),s=this.hiddenSize*4;for(let o=0;o<n;++o){let i=await this.embedFrame(t.subarray(o*r,(o+1)*r));this.#e.copyBufferToBuffer({src:i,dst:a,byteLength:s,srcOffset:i.byteOffset??0,dstOffset:(a.byteOffset??0)+o*s})}return a}syncPenalty(t){let r=this.#a;this.#e.host.writeBuffer(r.buffer,r.byteOffset??0,t.lastSeen)}#l(){return this.#r.acquire("frame-embedding",()=>R3(this.#t,this.#a))}async#u(){if(this.#s)return this.#s.session;let t=Dt.allocate(this.#e,this.#n.config,1,Se,this.#n.cacheDtype),r=new Dd(this.#n,t,this.#t,this.#a);try{await r.build()}catch(n){throw t.dispose(),n}return this.#s={session:r,cache:t},r}dispose(){this.#s?.session.dispose(),this.#s?.cache.dispose(),this.#s=null,this.#a.destroy(),this.#r.dispose(),this.#n.dispose(),ve(this.#t.tensors)}};function Vx(e,t){return zg({device:Ue(t),loadWeights:async()=>Ng(e),emitGraphs:async(r,n)=>[...await y3(r,n),...await q3(r,n),...await J1(r,n)]})}function Zx(e){let t=Ue(e);return`${t.features.has("shader-f16")?"f16":"f32"}|${wr(t.limits)}`}var O3=new WeakMap;function Xx(e,t){let r=O3.get(e);r||O3.set(e,r=new Map);let n=Zx(t),a=r.get(n);return a||(a=Vx(e,t).then(Kg),r.set(n,a),a.catch(()=>r.delete(n))),a}async function D3(e,t){let r=Td(e.model,void 0,{headerOnly:!0}),n=await ru({pendingRows:new WeakMap}).checkSupport(r,t,e.model),a=[["speech encoder",iu],["depth decoder",Md],["codec",Rd]];for(let[s,o]of a)n=_c(n,await o.checkSupport(o.configFromGGUF(e.mmproj),t,e.mmproj),s);return _c(n,(await Xx(e.mmproj,t))(t),"speech graphs")}async function*M3(e,t,r,n,a,s){let o=r.generation_state?.cache;if(!o)throw new Error("generating with media rows requires generation_state.cache");let i=o.get_seq_length(),u=i;for(let d of e.prefillChunks(gn(r.input_ids).length,i,o.maxLength)){let p=u+d.tokenCount;if(d.kind==="replay"&&n.some(f=>f.start<p&&f.start+f.gridHeight*f.gridWidth>u))throw new ea(u+jn(d.tokenCount),o.maxLength);u=p}let l=[],c=()=>{t.pendingRows.delete(o);for(let d of l.splice(0))d.release()};try{for(let f=0;f<n.length;++f)r.signal?.throwIfAborted(),l.push(await a(f));for(let f of n)o.ropePositions.addImageSpan(f);t.pendingRows.set(o,new Map(n.map((f,m)=>[f.start,l[m].features])));let d=r.on_prefill_done??null,p={...r,on_prefill_done:f=>{c(),d?.(f)}};yield*s?s(p):e.streamTokenIdsFromCache(p)}finally{c(),o.get_seq_length()<=i&&o.ropePositions.truncate(i)}}var Wt=(e,t)=>e<<t|e>>>32-t;function gu(e){return e^=e>>>16,e=Math.imul(e,2246822507),e^=e>>>13,e=Math.imul(e,3266489909),e^e>>>16}var Yx=new Uint8Array(new Uint32Array([1]).buffer)[0]===1;function Jx(e,t,r){let n=e[0],a=e[1],s=e[2],o=e[3];for(let i=0,u=r*4;i<u;i+=4)n^=Math.imul(Wt(Math.imul(t[i],597399067),15),2869860233),n=Math.imul(Wt(n,19)+a|0,5)+1444728091|0,a^=Math.imul(Wt(Math.imul(t[i+1],2869860233),16),951274213),a=Math.imul(Wt(a,17)+s|0,5)+197830471|0,s^=Math.imul(Wt(Math.imul(t[i+2],951274213),17),2716044179),s=Math.imul(Wt(s,15)+o|0,5)+2530024501|0,o^=Math.imul(Wt(Math.imul(t[i+3],2716044179),18),597399067),o=Math.imul(Wt(o,13)+n|0,5)+850148119|0;e[0]=n,e[1]=a,e[2]=s,e[3]=o}var Qd=4096;function W3(e,t){let r=new Int32Array([t,t,t,t]),n=e.byteLength,a=n>>>4,s=new Uint32Array(Math.min(Qd,a)*4),o=new Uint8Array(s.buffer);for(let w=0;w<a;w+=Qd){let b=Math.min(Qd,a-w),v=e.subarray(w*16,(w+b)*16);if(Yx)o.set(v);else for(let y=0;y<b*4;++y)s[y]=v[4*y]|v[4*y+1]<<8|v[4*y+2]<<16|v[4*y+3]<<24;Jx(r,s,b)}let i=r[0],u=r[1],l=r[2],c=r[3],d=a*16,p=n&15,f=0,m=0,h=0,g=0;for(let w=p-1;w>=0;--w){let b=e[d+w]<<(w&3)*8;w>=12?g^=b:w>=8?h^=b:w>=4?m^=b:f^=b}return p>12&&(c^=Math.imul(Wt(Math.imul(g,2716044179),18),597399067)),p>8&&(l^=Math.imul(Wt(Math.imul(h,951274213),17),2716044179)),p>4&&(u^=Math.imul(Wt(Math.imul(m,2869860233),16),951274213)),p>0&&(i^=Math.imul(Wt(Math.imul(f,597399067),15),2869860233)),i^=n,u^=n,l^=n,c^=n,i=i+u+l+c|0,u=u+i|0,l=l+i|0,c=c+i|0,i=gu(i),u=gu(u),l=gu(l),c=gu(c),i=i+u+l+c|0,u=u+i|0,l=l+i|0,c=c+i|0,[i>>>0,u>>>0,l>>>0,c>>>0]}var jd=16e3;function eT(e){let t=e;return typeof t=="object"&&t!==null&&typeof t.sampleRate=="number"&&typeof t.numberOfChannels=="number"&&typeof t.getChannelData=="function"}var F3=new WeakMap;function j3(e){if(e instanceof Float32Array)return{samples:e,digest:Q3(e)};if(!eT(e))throw new TypeError("an audio part holds a Float32Array of 16 kHz samples or an AudioBuffer");let t=Array.from({length:e.numberOfChannels},(i,u)=>e.getChannelData(u)),r=`${e.sampleRate}:${t.map(i=>Nd("channel",i)).join(",")}`,n=F3.get(e);if(n?.source===r)return n.clip;let a=new Float32Array(e.length);for(let i of t)for(let u=0;u<a.length;++u)a[u]+=i[u]/t.length;let s=nT(a,e.sampleRate,jd),o={samples:s,digest:Q3(s)};return F3.set(e,{source:r,clip:o}),o}var tT=16,rT=4096;function nT(e,t,r){if(t===r)return e;let n=r/t,a=.95*Math.min(1,n),s=tT/a,o=c=>{let d=Math.ceil(c-s),p=new Float64Array(Math.floor(c+s)-d+1);for(let f=0;f<p.length;++f){let m=d+f-c,h=m*a;p[f]=(h===0?1:Math.sin(Math.PI*h)/(Math.PI*h))*(.5+.5*Math.cos(Math.PI*m/s))*a}return{first:d,weights:p}},i=Number.isInteger(t)&&Number.isInteger(r)&&r/aT(t,r)<=rT,u=new Map,l=new Float32Array(Math.round(e.length*n));for(let c=0;c<l.length;++c){let d,p;if(i){d=Math.floor(c*t/r);let b=c*t-d*r;p=u.get(b),p||u.set(b,p=o(b/r))}else{let b=c/n;d=Math.floor(b),p=o(b-d)}let{first:f,weights:m}=p,h=d+f,g=Math.min(m.length,e.length-h),w=0;for(let b=Math.max(0,-h);b<g;++b)w+=e[h+b]*m[b];l[c]=w}return l}function aT(e,t){for(;t!==0;)[e,t]=[t,e%t];return e}function Nd(e,t){return`${e}:${W3(new Uint8Array(t.buffer,t.byteOffset,t.byteLength),0).map(r=>r.toString(16).padStart(8,"0")).join("")}:${t.length}`}function Q3(e){return Nd("audio",e)}function N3(e){return Nd("speech",e)}var sT=2,oT=10,Kd=3,iT=8,uT=e=>e.mode==="audio"||e.mode==="interleaved"&&e.speaking,lT=(e,t)=>e.mode===t.mode&&e.speaking===t.speaking&&e.slots===t.slots,zd=6,cT=12;function z3(e,t){if(e!=="interleaved")return t;if(t.slots>1)return{...t,slots:t.slots-1};let r=!t.speaking;return{...t,speaking:r,slots:r?cT:zd}}var hu=class e{language_model;audio_encoder;depth_decoder;codec;tokens;#e;constructor(t,r){this.language_model=t.language_model,this.audio_encoder=t.audio_encoder,this.depth_decoder=t.depth_decoder,this.codec=t.codec,this.tokens=t.tokens,this.#e=r}static async loadGGUF(t,r){let n=u=>tg(r.model.metadata,u),a={audio:n("<|audio|>"),audioOut:n("<|audio_out|>"),audioStart:n("<|audio_start|>"),mixedStart:n("<|mixed_start|>"),textEnd:n("<|text_end|>")},s={pendingRows:new WeakMap},o=await Yt(()=>ru(s)).loadGGUF(t,{gguf:r.model,onProgress:r.onProgress??null}),i=[o];try{let u=o.config.hidden_size,l=await du.load(t,r.mmproj);i.push(l);let c=await mu.loadGGUF(t,r.mmproj);i.push(c);let d=await ou.loadGGUF(t,r.mmproj);if(i.push(d),l.config.projectionDim!==u)throw new Error(`LFM3-Omni: the speech rows are ${l.config.projectionDim} wide; the language model is ${u}`);if(c.hiddenSize!==u)throw new Error(`LFM3-Omni: the depth decoder reads ${c.hiddenSize}-wide states; the language model is ${u}`);if(d.codebookSize!==c.codebookSize-1)throw new Error(`LFM3-Omni: the codec has ${d.codebookSize} codes; the depth decoder speaks ${c.codebookSize-1} plus end-of-audio`);return new e({language_model:o,audio_encoder:l,depth_decoder:c,codec:d,tokens:a},s)}catch(u){for(let l of i.reverse())l.dispose();throw u}}get audioEosCode(){return this.depth_decoder.codebookSize-1}createGenerationState(t={}){return this.language_model.createGenerationState(t)}disposeGenerationState(t){this.language_model.disposeGenerationState(t)}async*streamTokenIds(t){yield*this.#s(t,r=>this.streamTokenIdsFromCache({...t,generation_state:r}))}async generate(t){let r=[];for await(let n of this.streamTokenIds(t))r.push(n);return r}async*streamTokenIdsFromCache(t){let{on_speech:r,audio_chunk_frames:n=oT,...a}=t;if(!(Number.isInteger(n)&&n>0))throw new Error(`LFM3-Omni: audio_chunk_frames must be a positive integer, got ${n}`);let s=this.#l(a).primed,o=r?this.codec.createStreamingCache():null,i=[],u=[],l=c=>{u.push(this.decodeSpeech(i.splice(0),{cache:o,finalize:c}).then(d=>{d.length>0&&r.audio(d)}))};try{for await(let{output:c,finished:d}of this.#t(a)){if(s!==null&&(yield s),s=null,d)break;if(c.type==="text"){yield c.token;continue}r?.frame(c.codes),yield this.tokens.audioOut,!(!r||c.terminal)&&(i.push(c.codes),i.length>=(u.length===0?Math.min(sT,n):n)&&l(!1))}}finally{o&&(l(!0),await Promise.all(u))}}planAudio(t){let{samples:r,digest:n}=j3(t);if(r.length===0)throw new Error("LFM3-Omni: an audio part holds no samples");return{kind:"audio",tokens:Ds(r.length),digest:n,tokenId:this.tokens.audio,samples:r}}planSpeech(t){let r=this.depth_decoder.numCodebooks;if(t.length===0||t.length%r!==0)throw new Error(`LFM3-Omni: ${t.length} codes are not whole ${r}-code frames`);return{kind:"speech",tokens:t.length/r,digest:N3(t),tokenId:this.tokens.audioOut,codes:t}}async warmup(t,r){await this.language_model.warmup(t,r);let n=this.language_model.runtime,a=await this.audio_encoder.encode(new Float32Array(Gd)),s=n.empty("float32",[this.depth_decoder.hiddenSize],"lfm3-omni-warmup");try{let o=new Ms({do_sample:!1,temperature:1,repetition_penalty:1,repetition_window:0},this.depth_decoder.codebookSize,0);this.depth_decoder.syncPenalty(o);let i=Array.from({length:this.depth_decoder.numCodebooks},(l,c)=>c),{codes:u}=await this.depth_decoder.submitFrame(s,i,o,0,0);await this.depth_decoder.embedFrame(await u),await this.codec.decode(new Int32Array(this.codec.numCodebooks))}finally{s.destroy(),a.destroy()}}async warmRemainingPrefill(t){await Promise.all([this.language_model.warmRemainingPrefill(t),this.audio_encoder.prepare()])}async*streamOutputs(t){yield*this.#s(t,r=>this.streamOutputsFromCache({...t,generation_state:r}))}async generateOutputs(t){let r=[],n=[];for await(let a of this.streamOutputs(t))a.type==="text"?r.push(a.token):(r.push(this.tokens.audioOut),n.push(a.codes));return{sequences:r,audio_codes:n}}async*streamOutputsFromCache(t){for await(let{output:r}of this.#t(t))yield r}async decodeSpeech(t,{cache:r,finalize:n=!0}={}){let a=this.audioEosCode,s=t.filter(o=>!Array.from(o).every(i=>i===a));return s.length===0&&!r?new Float32Array(0):this.codec.decode(Int32Array.from(s.flatMap(o=>Array.from(o))),{cache:r,finalize:n})}dispose(){this.language_model.dispose(),this.audio_encoder.dispose(),this.depth_decoder.dispose(),this.codec.dispose()}async*#t(t){let{audio:r,images:n,generation_mode:a,tool_call_token:s,maskgit_order:o,...i}=t;if(Qr(t)!==null)throw new Error("LFM3-Omni: speech output decodes its text greedily (do_sample); the depth decoder samples per depth_decoder_*");let u=this.#a(t),{ids:l,last:c,mode:d,deferred:p}=this.#l(t),f=dT(t.seed??Math.random()*2**32),m=this.depth_decoder.sampling,h=new Ms({do_sample:t.depth_decoder_do_sample??m.do_sample,temperature:t.depth_decoder_temperature??m.temperature,repetition_penalty:t.depth_decoder_repetition_penalty??m.repetition_penalty,repetition_window:t.depth_decoder_repetition_window??m.repetition_window},this.depth_decoder.codebookSize,f()),g=[],w=o??(S=>{for(;g.length<=S;)g.push(pT(this.depth_decoder.numCodebooks,f));return g[S]});if(r&&n)throw new Error("LFM3-Omni: pass the prompt's clips as audio or its media as images, not both");let b=n??(r??[]).map(S=>this.planAudio(S)),v=l.subarray(0,-1),y=this.#o(v,b,u.get_seq_length());yield*M3(this.language_model,this.#e,{...i,input_ids:v},y,this.#u(b),S=>this.#n(S,c,d,p,w,h))}async*#n(t,r,n,a,s,o){let i=this.language_model,u=this.depth_decoder,l=this.#a(t),c=gn(t.input_ids),{maxNewTokens:d,eosTokenId:p,stopOnEos:f,onPrefillDone:m,signal:h}=Yn(t,{eosTokenId:i.config.eos_token_id}),g=Q=>f&&Jn(Q,p);await i.prefillTokens(l,c),m?.({tokens:c.length,cache_length:l.get_seq_length()});let w=l.get_seq_length(),b=Math.min(d,l.maxLength-w);if(b<=0)return;let v=In(t,Kd,iT),y=await l.stepCheckpoints(v+1);u.syncPenalty(o);let S=[],C={mode:n,speaking:!1,slots:zd},k=C,B={token:r,row:null},P=0,_=0,x=0,R=0,O=0,N=async()=>{let Q=uT(k);y[_%y.length]?.capture();let{next:V,hidden:$}=await i.submitDecodeStep(l,w+_++,{...B,predict:!Q}),M;if(Q){let j=await u.submitFrame($,s(R),o,R++,O++);M=j.codes,B={token:null,row:j.row}}else M=V,B={token:null,row:null};M.catch(()=>{}),k=z3(k.mode,k),S.push({frame:Q,result:M,phase:k})};try{for(;P<b;){for(;S.length<Math.min(v,P+1)&&_<b;)await N();let{frame:Q,result:V,phase:$}=S.shift(),M=await V;l.advance(1),++P,h?.throwIfAborted();let j;if(Q){let T=M,E=T.includes(this.audioEosCode);E?T.fill(this.audioEosCode):o.record(T),++x,j={type:"audio",codes:T,terminal:E}}else{let T=M;j={type:"text",token:a===null||T===a.toolCall?T:a.marker}}a=null;let A=this.#r(C,j,g);if(C=A.phase,!A.finished&&P<b&&!lT(C,$)&&(_>P&&y[P%y.length]?.restore(),S.length=0,_=P,k=C,R=x,O=o.recorded,u.syncPenalty(o),B=j.type==="audio"?{token:null,row:await u.embedFrame(j.codes)}:{token:j.token,row:null}),yield{output:j,finished:A.finished},A.finished)break}}finally{_>P&&(y[P%y.length]?.restore(),await Promise.allSettled(S.map(({result:Q})=>Q)))}}#r(t,r,n){let a=this.tokens,s=t.mode==="interleaved",{mode:o,speaking:i,slots:u}=t,l=!1;return r.type==="audio"?r.terminal&&(o="text"):r.token===a.textEnd?(l=!s,s&&(i=!0,u=1/0)):n(r.token)?l=!0:o==="text"&&r.token===a.audioStart?o="audio":o==="text"&&r.token===a.mixedStart&&(o="interleaved",i=!1,u=zd),{phase:z3(t.mode,{mode:o,speaking:i,slots:u}),finished:l}}#a(t){let r=t.generation_state?.cache;if(!r)throw new Error("Lfm3OmniForConditionalGeneration: generating from a cache requires generation_state.cache");return r}async*#s(t,r){let n=this.createGenerationState({maxLength:t.generation_state?.maxLength,max_length:t.generation_state?.max_length,minimumCapacity:ja(gn(t.input_ids).length+1,t.max_new_tokens)});try{yield*r(n)}finally{this.disposeGenerationState(n)}}#l(t){let r=this.tokens,n=gn(t.input_ids),a=t.generation_mode,s=a==="audio"?[r.audioStart,r.audioOut]:a==="interleaved"?[r.mixedStart]:[],o=s.length>0&&!s.includes(n[n.length-1])?s[0]:null,i=o!==null&&t.tool_call_token!==void 0?{marker:o,toolCall:t.tool_call_token}:null,u=i===null?o:null,l=u===null?n:Uint32Array.of(...n,u);if(l.length===0)throw new Error("LFM3-Omni: generation needs at least one prompt token");let c=l[l.length-1],d=c===r.audioStart||c===r.audioOut?"audio":c===r.mixedStart?"interleaved":"text";return{ids:l,last:c,mode:i===null?a??d:"text",primed:u,deferred:i}}#u(t){return async r=>{let n=t[r],a=n.kind==="audio"?await this.audio_encoder.encode(n.samples):await this.depth_decoder.embedFrames(n.codes);return{features:a,release:()=>a.destroy()}}}#o(t,r,n){let a=new Set([this.tokens.audio,this.tokens.audioOut]),s=[];for(let o=0;o<t.length;){if(!a.has(t[o])){++o;continue}let i=r[s.length];if(!i)throw new Error(`LFM3-Omni: the prompt holds more <|audio|> / <|audio_out|> runs than its ${r.length} media items`);if(i.tokenId!==t[o]||t.subarray(o,o+i.tokens).some(u=>u!==i.tokenId))throw new Error(`LFM3-Omni: media item ${s.length} (${i.kind}) needs ${i.tokens} consecutive placeholders at prompt token ${o}`);s.push({start:n+o,gridHeight:1,gridWidth:i.tokens}),o+=i.tokens}if(s.length!==r.length)throw new Error(`LFM3-Omni: ${s.length} media runs in the prompt but ${r.length} items`);return s}};function dT(e){let t=e>>>0;return()=>{t=t+1831565813>>>0;let r=t;return r=Math.imul(r^r>>>15,r|1),r^=r+Math.imul(r^r>>>7,r|61),(r^r>>>14)>>>0}}function pT(e,t){let r=Array.from({length:e},(n,a)=>a);for(let n=e-1;n>0;--n){let a=Math.floor(t()/4294967296*(n+1));[r[n],r[a]]=[r[a],r[n]]}return r}var fT="LiquidAI/LFM3-Omni-2B-GGUF",mT=4096,K3=e=>(e.split("/").pop()??e).replace(/-GGUF$/i,"");function gT(e){return{audioTokenId:e.tokens.audio,audioOutTokenId:e.tokens.audioOut,planAudio:t=>e.planAudio(t),planSpeech:t=>e.planSpeech(t)}}var Ud={defaultModelId:fT,defaultFile:e=>`${K3(e)}-Q4_K_M.gguf`,defaultMmproj:e=>`mmproj-${K3(e)}-Q8_0.gguf`,unsupportedReason:async(e,t,r)=>r?(await D3({model:e,mmproj:r},t)).reason:"LFM3-Omni needs its mmproj (the speech encoder, depth decoder and codec)",loadModel:async(e,t,r,n)=>{if(!n)throw new Error("LFM3-Omni needs its mmproj (the speech encoder, depth decoder and codec)");return hu.loadGGUF(e,{model:t,mmproj:n,onProgress:r})},createGenerationState:(e,t)=>{let r=Ba(e.language_model);return e.createGenerationState({maxLength:Math.max(1,Math.min(t??Xi(e.language_model),r))})},disposeGenerationState:(e,t)=>e.disposeGenerationState(t),defaultMaxNewTokens:mT,decodePipelineDepth:Kd,speech:gT};var $d=class e extends si{input_sampling_rate=jd;audio_sampling_rate=V1;constructor(t){super(t)}static checkAvailability(t,r={}){return Eg(Ud,t,r)}static async from_pretrained(t,r={}){let n=await Gg(Ud,a=>new Mo(a),t,r);return new e(n)}},TW=$d;export{hn as BusyError,ea as ContextOverflowError,ta as DeviceLostError,$d as Lfm3OmniForConditionalGeneration,TW as default};