VickyRDP commited on
Commit
32cb8a0
·
verified ·
1 Parent(s): 35f4ea6

Add KV dtype adapter: align KV-cache feed dtypes with session input metadata (handles both fp16- and fp32-resolving ORT builds)

Browse files
Files changed (1) hide show
  1. index.html +29 -0
index.html CHANGED
@@ -136,6 +136,35 @@ async function loadModel() {
136
  }
137
  }
138
  });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
139
  barWrap.style.display = 'none';
140
  statusEl.firstChild.textContent = '🟢 GPU Mart model loaded — ask away (type or tap the mic). Replies stream in as they are generated.';
141
  addMsg("Namaste! I'm the GPU Mart assistant — trained on RDP GPU Mart's catalog and policies. Ask me about products, CARINA/QUASAR/DRACO series, pricing, quotes, shipping, warranty or support.", 'bot');
 
136
  }
137
  }
138
  });
139
+ // ---- KV dtype adapter: align feed dtypes with the session's expected input types.
140
+ // Different ONNX Runtime builds resolve this fp16 graph to fp16 or fp32 inputs;
141
+ // this makes generation work in both cases (verified both directions).
142
+ try {
143
+ const HasF16 = typeof Float16Array !== 'undefined';
144
+ for (const sess of Object.values(generator.model.sessions || {})) {
145
+ const md = sess.inputMetadata;
146
+ const want = {};
147
+ const entries = Array.isArray(md) ? md : Object.entries(md || {}).map(([n, v]) => ({ name: n, ...v }));
148
+ for (const e of entries) if (e && e.name && e.type) want[e.name] = e.type;
149
+ const orig = sess.run.bind(sess);
150
+ sess.run = async (feeds, opts) => {
151
+ let Tctor = null;
152
+ for (const k in feeds) if (feeds[k] && feeds[k].constructor) { Tctor = feeds[k].constructor; break; }
153
+ for (const k in feeds) {
154
+ const t = feeds[k], w = want[k];
155
+ if (!t || !w || !Tctor || !t.type || t.type === w) continue;
156
+ if (t.type === 'float32' && w === 'float16' && HasF16) {
157
+ feeds[k] = new Tctor('float16', Float16Array.from(t.data), t.dims);
158
+ } else if (t.type === 'float16' && w === 'float32') {
159
+ feeds[k] = new Tctor('float32', Float32Array.from(t.data), t.dims);
160
+ }
161
+ }
162
+ return orig(feeds, opts);
163
+ };
164
+ }
165
+ console.log('[GM] kv dtype adapter installed');
166
+ } catch (e) { console.warn('[GM] kv dtype adapter install failed', e); }
167
+
168
  barWrap.style.display = 'none';
169
  statusEl.firstChild.textContent = '🟢 GPU Mart model loaded — ask away (type or tap the mic). Replies stream in as they are generated.';
170
  addMsg("Namaste! I'm the GPU Mart assistant — trained on RDP GPU Mart's catalog and policies. Ask me about products, CARINA/QUASAR/DRACO series, pricing, quotes, shipping, warranty or support.", 'bot');