Add KV dtype adapter: align KV-cache feed dtypes with session input metadata (handles both fp16- and fp32-resolving ORT builds)
Browse files- index.html +29 -0
index.html
CHANGED
|
@@ -136,6 +136,35 @@ async function loadModel() {
|
|
| 136 |
}
|
| 137 |
}
|
| 138 |
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
barWrap.style.display = 'none';
|
| 140 |
statusEl.firstChild.textContent = '🟢 GPU Mart model loaded — ask away (type or tap the mic). Replies stream in as they are generated.';
|
| 141 |
addMsg("Namaste! I'm the GPU Mart assistant — trained on RDP GPU Mart's catalog and policies. Ask me about products, CARINA/QUASAR/DRACO series, pricing, quotes, shipping, warranty or support.", 'bot');
|
|
|
|
| 136 |
}
|
| 137 |
}
|
| 138 |
});
|
| 139 |
+
// ---- KV dtype adapter: align feed dtypes with the session's expected input types.
|
| 140 |
+
// Different ONNX Runtime builds resolve this fp16 graph to fp16 or fp32 inputs;
|
| 141 |
+
// this makes generation work in both cases (verified both directions).
|
| 142 |
+
try {
|
| 143 |
+
const HasF16 = typeof Float16Array !== 'undefined';
|
| 144 |
+
for (const sess of Object.values(generator.model.sessions || {})) {
|
| 145 |
+
const md = sess.inputMetadata;
|
| 146 |
+
const want = {};
|
| 147 |
+
const entries = Array.isArray(md) ? md : Object.entries(md || {}).map(([n, v]) => ({ name: n, ...v }));
|
| 148 |
+
for (const e of entries) if (e && e.name && e.type) want[e.name] = e.type;
|
| 149 |
+
const orig = sess.run.bind(sess);
|
| 150 |
+
sess.run = async (feeds, opts) => {
|
| 151 |
+
let Tctor = null;
|
| 152 |
+
for (const k in feeds) if (feeds[k] && feeds[k].constructor) { Tctor = feeds[k].constructor; break; }
|
| 153 |
+
for (const k in feeds) {
|
| 154 |
+
const t = feeds[k], w = want[k];
|
| 155 |
+
if (!t || !w || !Tctor || !t.type || t.type === w) continue;
|
| 156 |
+
if (t.type === 'float32' && w === 'float16' && HasF16) {
|
| 157 |
+
feeds[k] = new Tctor('float16', Float16Array.from(t.data), t.dims);
|
| 158 |
+
} else if (t.type === 'float16' && w === 'float32') {
|
| 159 |
+
feeds[k] = new Tctor('float32', Float32Array.from(t.data), t.dims);
|
| 160 |
+
}
|
| 161 |
+
}
|
| 162 |
+
return orig(feeds, opts);
|
| 163 |
+
};
|
| 164 |
+
}
|
| 165 |
+
console.log('[GM] kv dtype adapter installed');
|
| 166 |
+
} catch (e) { console.warn('[GM] kv dtype adapter install failed', e); }
|
| 167 |
+
|
| 168 |
barWrap.style.display = 'none';
|
| 169 |
statusEl.firstChild.textContent = '🟢 GPU Mart model loaded — ask away (type or tap the mic). Replies stream in as they are generated.';
|
| 170 |
addMsg("Namaste! I'm the GPU Mart assistant — trained on RDP GPU Mart's catalog and policies. Ask me about products, CARINA/QUASAR/DRACO series, pricing, quotes, shipping, warranty or support.", 'bot');
|