Spaces:
Running
Running
Sync from GitHub
Browse files- classic/engines/gemma4_moe_ssd.js +16 -7
- classic/engines/lfm2_moe_ssd.js +11 -139
- classic/engines/lora.js +122 -0
- classic/engines/qwen35_moe_ssd.js +24 -0
- classic/engines/qwen3_moe_ssd.js +46 -2
- classic/refs/gemma-lora-test-base.json +126 -0
- classic/refs/gemma-lora-test-lora.json +126 -0
- classic/refs/qwen-lora-test-base.json +126 -0
- classic/refs/qwen-lora-test-lora.json +126 -0
classic/engines/gemma4_moe_ssd.js
CHANGED
|
@@ -588,13 +588,6 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 588 |
this.chunkTokens = Math.max(2, Math.floor(opts.prefillChunk || 256));
|
| 589 |
}
|
| 590 |
|
| 591 |
-
// A bind group whose entries are buffers or { buffer, offset, size } ranges.
|
| 592 |
-
bindR(name, entries) {
|
| 593 |
-
return this.device.createBindGroup({
|
| 594 |
-
layout: this.pipeline(name).getBindGroupLayout(0),
|
| 595 |
-
entries: entries.map((e, i) => ({ binding: i, resource: e.buffer ? e : { buffer: e } })),
|
| 596 |
-
});
|
| 597 |
-
}
|
| 598 |
|
| 599 |
// Dense upload; the Q6_K token table split into aligned planes on the GPU (the raw copy is then freed);
|
| 600 |
// the full-attention layers' RoPE frequency factors on the CPU side.
|
|
@@ -674,6 +667,12 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 674 |
for (let l = 0; l < Lc; l++) {
|
| 675 |
const p = (n) => W[`blk.${l}.${n}`], s = shapeOf(l), rope = c.swa[l] ? a.ropeS : a.ropeF;
|
| 676 |
const hasV = !!p('attn_v.weight');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
const ly = {
|
| 678 |
swa: c.swa[l], hasV,
|
| 679 |
rmsA: this.bind('rmsnorm', [a.x, p('attn_norm.weight').raw, a.xn, u.rmsH]),
|
|
@@ -762,6 +761,8 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 762 |
this.gB = { embed: this.bind('embedQ6KB', [this.q6.ql, this.q6.qh, this.q6.sc, this.q6.dd, b.ids, b.x, u.embed, qb]) };
|
| 763 |
for (let l = 0; l < c.layers; l++) {
|
| 764 |
const n = (t) => `blk.${l}.${t}`, raw = (t) => W[n(t)].raw, sh = shapeOf(l), hasV = !!W[n('attn_v.weight')];
|
|
|
|
|
|
|
| 765 |
const rope = c.swa[l] ? b.ropeS : b.ropeF;
|
| 766 |
this.layers[l].B = {
|
| 767 |
rmsA: this.bind('rmsnormB', [b.x, raw('attn_norm.weight'), b.xn, u.rmsH]),
|
|
@@ -803,6 +804,7 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 803 |
d('rmsnormB', g.rmsA, T);
|
| 804 |
d('matmulQ4T', g.q, mm(QN), tt); d('matmulQ4T', g.k, mm(kvn), tt);
|
| 805 |
if (g.v) d('matmulQ4T', g.v, mm(kvn), tt);
|
|
|
|
| 806 |
d('vNormB', g.vn, c.kvHeads[l], T);
|
| 807 |
d('qkNormRopeB', g.ropeQ, c.heads, T); d('qkNormRopeB', g.ropeK, c.kvHeads[l], T);
|
| 808 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
|
@@ -810,11 +812,14 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 810 |
d('softmaxB', g.soft, c.heads * T);
|
| 811 |
d('attnOutB', g.attOut, Math.ceil(QN / 256), T);
|
| 812 |
d('matmulQ4T', g.o, mm(H), tt);
|
|
|
|
| 813 |
d('rmsnormAddB', g.postA, T);
|
| 814 |
d('rmsnormB', g.rmsF, T);
|
| 815 |
d('matmulQ4T', g.mg, mm(DF), tt); d('matmulQ4T', g.mu, mm(DF), tt);
|
|
|
|
| 816 |
d('gegluSplitB', g.mact, Math.ceil(DF / 256), T);
|
| 817 |
d('matmulQ4T', g.md, mm(H), tt);
|
|
|
|
| 818 |
d('rmsnormB', g.mpost, T);
|
| 819 |
d('rmsnormB', g.xm, T);
|
| 820 |
d('rmsnormKB', g.rp, T);
|
|
@@ -940,6 +945,7 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 940 |
d('rmsnorm', g.rmsA, 1);
|
| 941 |
d('matmulQ4', g.q, Math.ceil(QN / 4)); d('matmulQ4', g.k, Math.ceil(kvn / 4));
|
| 942 |
if (g.hasV) d('matmulQ4', g.v, Math.ceil(kvn / 4));
|
|
|
|
| 943 |
d('vNorm', g.vn, c.kvHeads[l]);
|
| 944 |
d('qkNormRope', g.ropeQ, c.heads); d('qkNormRope', g.ropeK, c.kvHeads[l]);
|
| 945 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
|
@@ -947,12 +953,15 @@ export class Gemma4MoeSsd extends Qwen3MoeSsd {
|
|
| 947 |
d('softmaxW', g.soft, c.heads);
|
| 948 |
d('attnOutW', g.attOut, Math.ceil(QN / 256));
|
| 949 |
d('matmulQ4', g.o, Math.ceil(c.hidden / 4));
|
|
|
|
| 950 |
d('rmsnormAdd', g.postA, 1);
|
| 951 |
if (g.capA) d('copyAt', g.capA, Math.ceil(c.hidden / 256));
|
| 952 |
d('rmsnorm', g.rmsF, 1);
|
| 953 |
d('matmulQ4', g.mg, Math.ceil(c.denseFf / 4)); d('matmulQ4', g.mu, Math.ceil(c.denseFf / 4));
|
|
|
|
| 954 |
d('geglu', g.mact, Math.ceil(c.denseFf / 256));
|
| 955 |
d('matmulQ4', g.md, Math.ceil(c.hidden / 4));
|
|
|
|
| 956 |
d('rmsnorm', g.mpost, 1);
|
| 957 |
d('rmsnorm', g.xm, 1);
|
| 958 |
if (this.prefetch) for (const p of g.pf) { d('rmsnormK', p.rp, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
|
|
|
| 588 |
this.chunkTokens = Math.max(2, Math.floor(opts.prefillChunk || 256));
|
| 589 |
}
|
| 590 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 591 |
|
| 592 |
// Dense upload; the Q6_K token table split into aligned planes on the GPU (the raw copy is then freed);
|
| 593 |
// the full-attention layers' RoPE frequency factors on the CPU side.
|
|
|
|
| 667 |
for (let l = 0; l < Lc; l++) {
|
| 668 |
const p = (n) => W[`blk.${l}.${n}`], s = shapeOf(l), rope = c.swa[l] ? a.ropeS : a.ropeF;
|
| 669 |
const hasV = !!p('attn_v.weight');
|
| 670 |
+
// run-time LoRA hooks (engines/lora.js): the attention projections and the dense FFN
|
| 671 |
+
for (const [t, site, x, y] of [['attn_q', 'q', a.xn, a.q], ['attn_k', 'k', a.xn, a.k], ...(hasV ? [['attn_v', 'v', a.xn, a.vr]] : []),
|
| 672 |
+
['attn_output', 'o', a.att, a.o], ['ffn_gate', 'mg', a.xn, { buffer: a.mgu, offset: 0, size: gateBytes }],
|
| 673 |
+
['ffn_up', 'mu', a.xn, { buffer: a.mgu, offset: gateBytes, size: gateBytes }], ['ffn_down', 'md', a.ma, a.md]]) {
|
| 674 |
+
this.loraIo(`blk.${l}.${t}.weight`, l, site, x, y);
|
| 675 |
+
}
|
| 676 |
const ly = {
|
| 677 |
swa: c.swa[l], hasV,
|
| 678 |
rmsA: this.bind('rmsnorm', [a.x, p('attn_norm.weight').raw, a.xn, u.rmsH]),
|
|
|
|
| 761 |
this.gB = { embed: this.bind('embedQ6KB', [this.q6.ql, this.q6.qh, this.q6.sc, this.q6.dd, b.ids, b.x, u.embed, qb]) };
|
| 762 |
for (let l = 0; l < c.layers; l++) {
|
| 763 |
const n = (t) => `blk.${l}.${t}`, raw = (t) => W[n(t)].raw, sh = shapeOf(l), hasV = !!W[n('attn_v.weight')];
|
| 764 |
+
for (const [t, x, y] of [['attn_q', b.xn, b.q], ['attn_k', b.xn, b.k], ...(hasV ? [['attn_v', b.xn, b.vr]] : []), ['attn_output', b.att, b.o],
|
| 765 |
+
['ffn_gate', b.xn, b.mg], ['ffn_up', b.xn, b.mu], ['ffn_down', b.ma, b.md]]) this.loraIoB(n(`${t}.weight`), x, y);
|
| 766 |
const rope = c.swa[l] ? b.ropeS : b.ropeF;
|
| 767 |
this.layers[l].B = {
|
| 768 |
rmsA: this.bind('rmsnormB', [b.x, raw('attn_norm.weight'), b.xn, u.rmsH]),
|
|
|
|
| 804 |
d('rmsnormB', g.rmsA, T);
|
| 805 |
d('matmulQ4T', g.q, mm(QN), tt); d('matmulQ4T', g.k, mm(kvn), tt);
|
| 806 |
if (g.v) d('matmulQ4T', g.v, mm(kvn), tt);
|
| 807 |
+
this.encodeLoraB(pass, l, 'q', T); this.encodeLoraB(pass, l, 'k', T); this.encodeLoraB(pass, l, 'v', T);
|
| 808 |
d('vNormB', g.vn, c.kvHeads[l], T);
|
| 809 |
d('qkNormRopeB', g.ropeQ, c.heads, T); d('qkNormRopeB', g.ropeK, c.kvHeads[l], T);
|
| 810 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
|
|
|
| 812 |
d('softmaxB', g.soft, c.heads * T);
|
| 813 |
d('attnOutB', g.attOut, Math.ceil(QN / 256), T);
|
| 814 |
d('matmulQ4T', g.o, mm(H), tt);
|
| 815 |
+
this.encodeLoraB(pass, l, 'o', T);
|
| 816 |
d('rmsnormAddB', g.postA, T);
|
| 817 |
d('rmsnormB', g.rmsF, T);
|
| 818 |
d('matmulQ4T', g.mg, mm(DF), tt); d('matmulQ4T', g.mu, mm(DF), tt);
|
| 819 |
+
this.encodeLoraB(pass, l, 'mg', T); this.encodeLoraB(pass, l, 'mu', T);
|
| 820 |
d('gegluSplitB', g.mact, Math.ceil(DF / 256), T);
|
| 821 |
d('matmulQ4T', g.md, mm(H), tt);
|
| 822 |
+
this.encodeLoraB(pass, l, 'md', T);
|
| 823 |
d('rmsnormB', g.mpost, T);
|
| 824 |
d('rmsnormB', g.xm, T);
|
| 825 |
d('rmsnormKB', g.rp, T);
|
|
|
|
| 945 |
d('rmsnorm', g.rmsA, 1);
|
| 946 |
d('matmulQ4', g.q, Math.ceil(QN / 4)); d('matmulQ4', g.k, Math.ceil(kvn / 4));
|
| 947 |
if (g.hasV) d('matmulQ4', g.v, Math.ceil(kvn / 4));
|
| 948 |
+
this.encodeLora(pass, l, 'q'); this.encodeLora(pass, l, 'k'); this.encodeLora(pass, l, 'v');
|
| 949 |
d('vNorm', g.vn, c.kvHeads[l]);
|
| 950 |
d('qkNormRope', g.ropeQ, c.heads); d('qkNormRope', g.ropeK, c.kvHeads[l]);
|
| 951 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
|
|
|
| 953 |
d('softmaxW', g.soft, c.heads);
|
| 954 |
d('attnOutW', g.attOut, Math.ceil(QN / 256));
|
| 955 |
d('matmulQ4', g.o, Math.ceil(c.hidden / 4));
|
| 956 |
+
this.encodeLora(pass, l, 'o');
|
| 957 |
d('rmsnormAdd', g.postA, 1);
|
| 958 |
if (g.capA) d('copyAt', g.capA, Math.ceil(c.hidden / 256));
|
| 959 |
d('rmsnorm', g.rmsF, 1);
|
| 960 |
d('matmulQ4', g.mg, Math.ceil(c.denseFf / 4)); d('matmulQ4', g.mu, Math.ceil(c.denseFf / 4));
|
| 961 |
+
this.encodeLora(pass, l, 'mg'); this.encodeLora(pass, l, 'mu');
|
| 962 |
d('geglu', g.mact, Math.ceil(c.denseFf / 256));
|
| 963 |
d('matmulQ4', g.md, Math.ceil(c.hidden / 4));
|
| 964 |
+
this.encodeLora(pass, l, 'md');
|
| 965 |
d('rmsnorm', g.mpost, 1);
|
| 966 |
d('rmsnorm', g.xm, 1);
|
| 967 |
if (this.prefetch) for (const p of g.pf) { d('rmsnormK', p.rp, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
classic/engines/lfm2_moe_ssd.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
| 23 |
|
| 24 |
import { Qwen3MoeSsd, BpeTokenizer, GGML, Q4_BLOCK, tensorBytes, TOK, STORAGE, COPY_SRC, COPY_DST, MAP_READ } from './qwen3_moe_ssd.js';
|
| 25 |
import { Gemma4MoeSsd } from './gemma4_moe_ssd.js';
|
| 26 |
-
import { Q8_BLOCK
|
| 27 |
import { QWEN35_BATCH_KERNELS } from './qwen35_moe_ssd.js';
|
| 28 |
|
| 29 |
export function configFromGgufLfm2Moe(kv) {
|
|
@@ -225,34 +225,6 @@ struct P { H: u32 }
|
|
| 225 |
let conv = w[3u * i] * s0 + w[3u * i + 1u] * s1 + w[3u * i + 2u] * bx;
|
| 226 |
y[i] = bcx[p.H + i] * conv;
|
| 227 |
st[2u * i] = s1; st[2u * i + 1u] = bx;
|
| 228 |
-
}`,
|
| 229 |
-
// Run-time LoRA, first half: t = AΒ·x, A [R Γ N] f16 (a GGUF `lora_a`), one workgroup per rank row.
|
| 230 |
-
loraA: `enable f16;
|
| 231 |
-
struct P { R: u32, N: u32 }
|
| 232 |
-
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 233 |
-
@group(0) @binding(1) var<storage, read> a: array<f16>;
|
| 234 |
-
@group(0) @binding(2) var<storage, read_write> t: array<f32>;
|
| 235 |
-
@group(0) @binding(3) var<uniform> p: P;
|
| 236 |
-
var<workgroup> red: array<f32, 256>;
|
| 237 |
-
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 238 |
-
let r = wg.x; let i = l.x; var acc = 0.0;
|
| 239 |
-
for (var k = i; k < p.N; k += 256u) { acc += f32(a[r * p.N + k]) * x[k]; }
|
| 240 |
-
red[i] = acc; workgroupBarrier();
|
| 241 |
-
for (var s = 128u; s > 0u; s >>= 1u) { if (i < s) { red[i] += red[i + s]; } workgroupBarrier(); }
|
| 242 |
-
if (i == 0u) { t[r] = red[0]; }
|
| 243 |
-
}`,
|
| 244 |
-
// Second half: y += scale Β· BΒ·t, B [M Γ R] f16 (a GGUF `lora_b`), one thread per output row.
|
| 245 |
-
loraB: `enable f16;
|
| 246 |
-
struct P { M: u32, R: u32, s: f32 }
|
| 247 |
-
@group(0) @binding(0) var<storage, read> t: array<f32>;
|
| 248 |
-
@group(0) @binding(1) var<storage, read> b: array<f16>;
|
| 249 |
-
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 250 |
-
@group(0) @binding(3) var<uniform> p: P;
|
| 251 |
-
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 252 |
-
let m = g.x; if (m >= p.M) { return; }
|
| 253 |
-
var acc = 0.0;
|
| 254 |
-
for (var j = 0u; j < p.R; j++) { acc += f32(b[m * p.R + j]) * t[j]; }
|
| 255 |
-
y[m] = y[m] + p.s * acc;
|
| 256 |
}`,
|
| 257 |
// Debug capture: y[off + i] = x[i].
|
| 258 |
copyAt: `
|
|
@@ -291,39 +263,6 @@ ${QB}
|
|
| 291 |
s0 = s1; s1 = bx;
|
| 292 |
}
|
| 293 |
st[2u * i] = s0; st[2u * i + 1u] = s1;
|
| 294 |
-
}`,
|
| 295 |
-
// Run-time LoRA over the chunk: t[tok] = AΒ·x[tok], one workgroup per (rank row, token).
|
| 296 |
-
loraAT: `enable f16;
|
| 297 |
-
struct P { R: u32, N: u32 }
|
| 298 |
-
${QB}
|
| 299 |
-
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 300 |
-
@group(0) @binding(1) var<storage, read> a: array<f16>;
|
| 301 |
-
@group(0) @binding(2) var<storage, read_write> t: array<f32>;
|
| 302 |
-
@group(0) @binding(3) var<uniform> p: P;
|
| 303 |
-
@group(0) @binding(4) var<uniform> qd: Q;
|
| 304 |
-
var<workgroup> red: array<f32, 256>;
|
| 305 |
-
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 306 |
-
let r = wg.x; let tok = wg.y; let i = l.x; if (tok >= qd.T) { return; }
|
| 307 |
-
let xb = tok * p.N; var acc = 0.0;
|
| 308 |
-
for (var k = i; k < p.N; k += 256u) { acc += f32(a[r * p.N + k]) * x[xb + k]; }
|
| 309 |
-
red[i] = acc; workgroupBarrier();
|
| 310 |
-
for (var s = 128u; s > 0u; s >>= 1u) { if (i < s) { red[i] += red[i + s]; } workgroupBarrier(); }
|
| 311 |
-
if (i == 0u) { t[tok * p.R + r] = red[0]; }
|
| 312 |
-
}`,
|
| 313 |
-
// y[tok] += scale Β· BΒ·t[tok], one thread per (output row, token).
|
| 314 |
-
loraBT: `enable f16;
|
| 315 |
-
struct P { M: u32, R: u32, s: f32 }
|
| 316 |
-
${QB}
|
| 317 |
-
@group(0) @binding(0) var<storage, read> t: array<f32>;
|
| 318 |
-
@group(0) @binding(1) var<storage, read> b: array<f16>;
|
| 319 |
-
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 320 |
-
@group(0) @binding(3) var<uniform> p: P;
|
| 321 |
-
@group(0) @binding(4) var<uniform> qd: Q;
|
| 322 |
-
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 323 |
-
let m = g.x; let tok = g.y; if (m >= p.M || tok >= qd.T) { return; }
|
| 324 |
-
var acc = 0.0;
|
| 325 |
-
for (var j = 0u; j < p.R; j++) { acc += f32(b[m * p.R + j]) * t[tok * p.R + j]; }
|
| 326 |
-
y[tok * p.M + m] = y[tok * p.M + m] + p.s * acc;
|
| 327 |
}`,
|
| 328 |
// act = SiLU(g) Β· u over T Γ n (the dense FFN's separate gate and up).
|
| 329 |
siluSplitB: `
|
|
@@ -394,7 +333,6 @@ export class Lfm2MoeSsd extends Gemma4MoeSsd {
|
|
| 394 |
constructor(device, manifest, gguf, opts) {
|
| 395 |
super(device, manifest, gguf, opts);
|
| 396 |
this.lookahead = Math.max(1, Math.floor(opts.lookahead ?? 1));
|
| 397 |
-
this.arch = gguf.kv['general.architecture']; // a LoRA must be for the same architecture (loadAdapter)
|
| 398 |
}
|
| 399 |
|
| 400 |
// Kernel names for the matrix and expert weights: Q4_0 here; an engine reading a Q8_0 file (hy_v3_moe_ssd.js) swaps
|
|
@@ -478,6 +416,12 @@ export class Lfm2MoeSsd extends Gemma4MoeSsd {
|
|
| 478 |
this.mlayers = [];
|
| 479 |
for (let L = 0; L < ML; L++) {
|
| 480 |
const p = (n) => W[`blk.${L}.${n}`];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 481 |
const ly = { attn: c.attn[L], rmsA: this.bind('rmsnorm', [a.x, p('attn_norm.weight').raw, a.xn, u.rmsH]) };
|
| 482 |
if (ly.attn) {
|
| 483 |
Object.assign(ly, {
|
|
@@ -574,6 +518,10 @@ export class Lfm2MoeSsd extends Gemma4MoeSsd {
|
|
| 574 |
this.gB = { embed: this.q6 ? this.bind('embedQ6KB', [this.q6.ql, this.q6.qh, this.q6.sc, this.q6.dd, b.ids, b.x, u.embed, qb]) : null };
|
| 575 |
for (let L = 0; L < c.modelLayers; L++) {
|
| 576 |
const n = (t) => `blk.${L}.${t}`, raw = (t) => W[n(t)].raw, ly = this.mlayers[L];
|
|
|
|
|
|
|
|
|
|
|
|
|
| 577 |
const B_ = { rmsA: this.bind('rmsnormB', [b.x, raw('attn_norm.weight'), b.xn, u.rmsH]) };
|
| 578 |
if (ly.attn) {
|
| 579 |
Object.assign(B_, {
|
|
@@ -747,82 +695,6 @@ export class Lfm2MoeSsd extends Gemma4MoeSsd {
|
|
| 747 |
}
|
| 748 |
}
|
| 749 |
|
| 750 |
-
// ββ Run-time LoRA ββ
|
| 751 |
-
// A LoRA GGUF (llama.cpp's convert_lora_to_gguf.py: per adapted matrix W, `W.lora_a` [N β R] and `W.lora_b` [R β M],
|
| 752 |
-
// F16), applied as y += scale Β· B(AΒ·x) right after W's matmul, with scale = alpha / R (as llama.cpp's --lora) times
|
| 753 |
-
// `scale`. The weights stay as loaded: setAdapter() switches between adapters, or back to none, with no reload.
|
| 754 |
-
// Adapters on the experts are refused (the paged experts have no hook).
|
| 755 |
-
async loadAdapter(source, { scale = 1 } = {}) {
|
| 756 |
-
// a URL, a Blob (a File from OPFS or a picker), an ArrayBuffer or a Uint8Array
|
| 757 |
-
const bytes = source instanceof Uint8Array ? source : new Uint8Array(typeof source === 'string'
|
| 758 |
-
? await (await fetch(source)).arrayBuffer() : source instanceof Blob ? await source.arrayBuffer() : source);
|
| 759 |
-
const gguf = parseGguf(bytes), kv = gguf.kv;
|
| 760 |
-
if (kv['adapter.type'] !== 'lora') throw new Error(`not a LoRA GGUF (adapter.type ${kv['adapter.type']})`);
|
| 761 |
-
if (kv['general.architecture'] !== this.arch) throw new Error(`adapter for ${kv['general.architecture']}, model is ${this.arch}`);
|
| 762 |
-
const SITES = { attn_q: 'q', attn_k: 'k', attn_v: 'v', attn_output: 'o', 'shortconv.in_proj': 'inProj', 'shortconv.out_proj': 'o', ffn_gate: 'mg', ffn_up: 'mu', ffn_down: 'md' };
|
| 763 |
-
const byName = new Map(gguf.tensors.map((t) => [t.name, t]));
|
| 764 |
-
const sites = Array.from({ length: this.cfg.modelLayers }, () => ({}));
|
| 765 |
-
const STO = 0x80 | COPY_DST, a = this.a, b = this.b, qb = this.qb, gateBytes = this.cfg.denseFf * 4;
|
| 766 |
-
let R = 0, bytesOnGpu = 0;
|
| 767 |
-
const upload = (t) => {
|
| 768 |
-
if (t.type !== GGML.F16) throw new Error(`${t.name}: type ${t.type}, LoRA tensors must be F16`);
|
| 769 |
-
const n = t.dims.reduce((x, y) => x * y, 1), buf = this.buffer(n * 2, STO, 'adapter');
|
| 770 |
-
this.device.queue.writeBuffer(buf, 0, bytes, gguf.dataStart + t.offset, n * 2);
|
| 771 |
-
bytesOnGpu += buf.size;
|
| 772 |
-
return buf;
|
| 773 |
-
};
|
| 774 |
-
for (const ta of gguf.tensors) {
|
| 775 |
-
const m = /^blk\.(\d+)\.(.+)\.weight\.lora_a$/.exec(ta.name);
|
| 776 |
-
if (!m) continue;
|
| 777 |
-
const L = Number(m[1]), site = SITES[m[2]], tb = byName.get(ta.name.replace(/lora_a$/, 'lora_b')), w = this.w[`blk.${L}.${m[2]}.weight`];
|
| 778 |
-
if (!site || !tb || !w) throw new Error(`${ta.name}: no matching matrix in this model (adapters on the experts are not supported)`);
|
| 779 |
-
const [N, r] = ta.dims, [r2, M] = tb.dims;
|
| 780 |
-
if (r !== r2 || N !== w.cols || M !== w.rows) throw new Error(`${ta.name}: shape ${N}Γ${r} / ${r2}Γ${M} does not fit ${w.cols}Γ${w.rows}`);
|
| 781 |
-
R = Math.max(R, r);
|
| 782 |
-
sites[L][site] = { r, N, M, A: upload(ta), B: upload(tb) };
|
| 783 |
-
}
|
| 784 |
-
if (!R) throw new Error('the adapter has no lora_a / lora_b pairs');
|
| 785 |
-
const s = (kv['adapter.lora.alpha'] ?? R) / R * scale;
|
| 786 |
-
// One t buffer per adapter (R floats per token); binds against the engine's activations.
|
| 787 |
-
const lt = this.buffer(R * 4, 0x80 | COPY_SRC | COPY_DST, 'adapter'), ltB = b ? this.buffer(this.chunkTokens * R * 4, 0x80 | COPY_SRC | COPY_DST, 'adapter') : null;
|
| 788 |
-
const io = (L, site) => {
|
| 789 |
-
const attn = this.cfg.attn[L];
|
| 790 |
-
const d = { q: [a.xn, a.q], k: [a.xn, a.k], v: [a.xn, a.v], o: attn ? [a.att, a.o] : [a.y, a.o], inProj: [a.xn, a.bcx],
|
| 791 |
-
mg: [a.xn, { buffer: a.mgu, offset: 0, size: gateBytes }], mu: [a.xn, { buffer: a.mgu, offset: gateBytes, size: gateBytes }], md: [a.ma, a.o] }[site];
|
| 792 |
-
const bt = b && { q: [b.xn, b.q], k: [b.xn, b.k], v: [b.xn, b.v], o: attn ? [b.att, b.o] : [b.y, b.o], inProj: [b.xn, b.bcx],
|
| 793 |
-
mg: [b.xn, b.mg], mu: [b.xn, b.mu], md: [b.ma, b.o] }[site];
|
| 794 |
-
return { d, bt };
|
| 795 |
-
};
|
| 796 |
-
sites.forEach((ss, L) => {
|
| 797 |
-
for (const [site, e] of Object.entries(ss)) {
|
| 798 |
-
const { d, bt } = io(L, site), uA = this.uniform([e.r, e.N]), uB = this.uniform([e.M, e.r, { f: s }]);
|
| 799 |
-
e.a = this.bind('loraA', [d[0], e.A, lt, uA]);
|
| 800 |
-
e.b = this.bindR('loraB', [lt, e.B, d[1], uB]);
|
| 801 |
-
if (bt) { e.aB = this.bind('loraAT', [bt[0], e.A, ltB, uA, qb]); e.bB = this.bind('loraBT', [ltB, e.B, bt[1], uB, qb]); }
|
| 802 |
-
}
|
| 803 |
-
});
|
| 804 |
-
const n = sites.reduce((x, ss) => x + Object.keys(ss).length, 0);
|
| 805 |
-
return { name: kv['general.name'] || 'lora', rank: R, scale: s, matrices: n, bytes: bytesOnGpu, sites };
|
| 806 |
-
}
|
| 807 |
-
// The adapter for the next turns (null: the plain model). The cached prompt was computed with the old weights, so
|
| 808 |
-
// the next turn starts from token 0.
|
| 809 |
-
async setAdapter(adapter) {
|
| 810 |
-
const release = await this.acquire();
|
| 811 |
-
try { this.adapter = adapter || null; this.reset(); } finally { release(); }
|
| 812 |
-
}
|
| 813 |
-
encodeLora(pass, L, site) {
|
| 814 |
-
const e = this.adapter && this.adapter.sites[L][site];
|
| 815 |
-
if (!e) return;
|
| 816 |
-
this.dispatch(pass, 'loraA', e.a, e.r);
|
| 817 |
-
this.dispatch(pass, 'loraB', e.b, Math.ceil(e.M / 256));
|
| 818 |
-
}
|
| 819 |
-
encodeLoraB(pass, L, site, T) {
|
| 820 |
-
const e = this.adapter && this.adapter.sites[L][site];
|
| 821 |
-
if (!e) return;
|
| 822 |
-
this.dispatch(pass, 'loraAT', e.aB, e.r, T);
|
| 823 |
-
this.dispatch(pass, 'loraBT', e.bB, Math.ceil(e.M / 256), T);
|
| 824 |
-
}
|
| 825 |
-
|
| 826 |
// The conv state cannot rewind, so any change before the cached end restarts from token 0.
|
| 827 |
reset() {
|
| 828 |
super.reset();
|
|
|
|
| 23 |
|
| 24 |
import { Qwen3MoeSsd, BpeTokenizer, GGML, Q4_BLOCK, tensorBytes, TOK, STORAGE, COPY_SRC, COPY_DST, MAP_READ } from './qwen3_moe_ssd.js';
|
| 25 |
import { Gemma4MoeSsd } from './gemma4_moe_ssd.js';
|
| 26 |
+
import { Q8_BLOCK } from '../src/gguf.js';
|
| 27 |
import { QWEN35_BATCH_KERNELS } from './qwen35_moe_ssd.js';
|
| 28 |
|
| 29 |
export function configFromGgufLfm2Moe(kv) {
|
|
|
|
| 225 |
let conv = w[3u * i] * s0 + w[3u * i + 1u] * s1 + w[3u * i + 2u] * bx;
|
| 226 |
y[i] = bcx[p.H + i] * conv;
|
| 227 |
st[2u * i] = s1; st[2u * i + 1u] = bx;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
}`,
|
| 229 |
// Debug capture: y[off + i] = x[i].
|
| 230 |
copyAt: `
|
|
|
|
| 263 |
s0 = s1; s1 = bx;
|
| 264 |
}
|
| 265 |
st[2u * i] = s0; st[2u * i + 1u] = s1;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
}`,
|
| 267 |
// act = SiLU(g) Β· u over T Γ n (the dense FFN's separate gate and up).
|
| 268 |
siluSplitB: `
|
|
|
|
| 333 |
constructor(device, manifest, gguf, opts) {
|
| 334 |
super(device, manifest, gguf, opts);
|
| 335 |
this.lookahead = Math.max(1, Math.floor(opts.lookahead ?? 1));
|
|
|
|
| 336 |
}
|
| 337 |
|
| 338 |
// Kernel names for the matrix and expert weights: Q4_0 here; an engine reading a Q8_0 file (hy_v3_moe_ssd.js) swaps
|
|
|
|
| 416 |
this.mlayers = [];
|
| 417 |
for (let L = 0; L < ML; L++) {
|
| 418 |
const p = (n) => W[`blk.${L}.${n}`];
|
| 419 |
+
// run-time LoRA hooks (engines/lora.js): the operator's projections and, in the dense layers, the FFN
|
| 420 |
+
const hooks = c.attn[L] ? [['attn_q', 'q', a.xn, a.q], ['attn_k', 'k', a.xn, a.k], ['attn_v', 'v', a.xn, a.v], ['attn_output', 'o', a.att, a.o]]
|
| 421 |
+
: [['shortconv.in_proj', 'inProj', a.xn, a.bcx], ['shortconv.out_proj', 'o', a.y, a.o]];
|
| 422 |
+
if (L < c.lead) hooks.push(['ffn_gate', 'mg', a.xn, { buffer: a.mgu, offset: 0, size: gateBytes }],
|
| 423 |
+
['ffn_up', 'mu', a.xn, { buffer: a.mgu, offset: gateBytes, size: gateBytes }], ['ffn_down', 'md', a.ma, a.o]);
|
| 424 |
+
for (const [t, site, x, y] of hooks) this.loraIo(`blk.${L}.${t}.weight`, L, site, x, y);
|
| 425 |
const ly = { attn: c.attn[L], rmsA: this.bind('rmsnorm', [a.x, p('attn_norm.weight').raw, a.xn, u.rmsH]) };
|
| 426 |
if (ly.attn) {
|
| 427 |
Object.assign(ly, {
|
|
|
|
| 518 |
this.gB = { embed: this.q6 ? this.bind('embedQ6KB', [this.q6.ql, this.q6.qh, this.q6.sc, this.q6.dd, b.ids, b.x, u.embed, qb]) : null };
|
| 519 |
for (let L = 0; L < c.modelLayers; L++) {
|
| 520 |
const n = (t) => `blk.${L}.${t}`, raw = (t) => W[n(t)].raw, ly = this.mlayers[L];
|
| 521 |
+
const hooks = ly.attn ? [['attn_q', b.xn, b.q], ['attn_k', b.xn, b.k], ['attn_v', b.xn, b.v], ['attn_output', b.att, b.o]]
|
| 522 |
+
: [['shortconv.in_proj', b.xn, b.bcx], ['shortconv.out_proj', b.y, b.o]];
|
| 523 |
+
if (L < c.lead) hooks.push(['ffn_gate', b.xn, b.mg], ['ffn_up', b.xn, b.mu], ['ffn_down', b.ma, b.o]);
|
| 524 |
+
for (const [t, x, y] of hooks) this.loraIoB(n(`${t}.weight`), x, y);
|
| 525 |
const B_ = { rmsA: this.bind('rmsnormB', [b.x, raw('attn_norm.weight'), b.xn, u.rmsH]) };
|
| 526 |
if (ly.attn) {
|
| 527 |
Object.assign(B_, {
|
|
|
|
| 695 |
}
|
| 696 |
}
|
| 697 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 698 |
// The conv state cannot rewind, so any change before the cached end restarts from token 0.
|
| 699 |
reset() {
|
| 700 |
super.reset();
|
classic/engines/lora.js
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Run-time LoRA for the native engines: a LoRA GGUF (llama.cpp's convert_lora_to_gguf.py format: per adapted matrix
|
| 2 |
+
// W, `W.lora_a` [N β R] and `W.lora_b` [R β M], F16) applied as y += scale Β· B(AΒ·x) right after W's matmul, with
|
| 3 |
+
// scale = alpha / R (as llama.cpp's --lora) times the caller's scale. The model's weights stay as loaded, so an engine
|
| 4 |
+
// switches between adapters, or back to none, between turns with no reload (Qwen3MoeSsd.loadAdapter / setAdapter).
|
| 5 |
+
//
|
| 6 |
+
// An engine says which of its matmuls an adapter may follow by calling loraIo() as it binds them: the matrix's tensor
|
| 7 |
+
// name, its input and output buffers for one token, and (in initBatch) for a chunk. Adapters on the paged experts are
|
| 8 |
+
// refused: nothing reads them through a bound matmul.
|
| 9 |
+
import { GGML, parseGguf } from '../src/gguf.js';
|
| 10 |
+
|
| 11 |
+
const QB = 'struct Q { T: u32, pos0: u32, S: u32, pad: u32 }';
|
| 12 |
+
export const LORA_KERNELS = {
|
| 13 |
+
// t = AΒ·x, A [R Γ N] f16, one workgroup per rank row.
|
| 14 |
+
loraA: `enable f16;
|
| 15 |
+
struct P { R: u32, N: u32 }
|
| 16 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 17 |
+
@group(0) @binding(1) var<storage, read> a: array<f16>;
|
| 18 |
+
@group(0) @binding(2) var<storage, read_write> t: array<f32>;
|
| 19 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 20 |
+
var<workgroup> red: array<f32, 256>;
|
| 21 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 22 |
+
let r = wg.x; let i = l.x; var acc = 0.0;
|
| 23 |
+
for (var k = i; k < p.N; k += 256u) { acc += f32(a[r * p.N + k]) * x[k]; }
|
| 24 |
+
red[i] = acc; workgroupBarrier();
|
| 25 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (i < s) { red[i] += red[i + s]; } workgroupBarrier(); }
|
| 26 |
+
if (i == 0u) { t[r] = red[0]; }
|
| 27 |
+
}`,
|
| 28 |
+
// y += scale Β· BΒ·t, B [M Γ R] f16, one thread per output row.
|
| 29 |
+
loraB: `enable f16;
|
| 30 |
+
struct P { M: u32, R: u32, s: f32 }
|
| 31 |
+
@group(0) @binding(0) var<storage, read> t: array<f32>;
|
| 32 |
+
@group(0) @binding(1) var<storage, read> b: array<f16>;
|
| 33 |
+
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 34 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 35 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 36 |
+
let m = g.x; if (m >= p.M) { return; }
|
| 37 |
+
var acc = 0.0;
|
| 38 |
+
for (var j = 0u; j < p.R; j++) { acc += f32(b[m * p.R + j]) * t[j]; }
|
| 39 |
+
y[m] = y[m] + p.s * acc;
|
| 40 |
+
}`,
|
| 41 |
+
// The chunk forms: t[tok] = AΒ·x[tok], one workgroup per (rank row, token) ...
|
| 42 |
+
loraAT: `enable f16;
|
| 43 |
+
struct P { R: u32, N: u32 }
|
| 44 |
+
${QB}
|
| 45 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 46 |
+
@group(0) @binding(1) var<storage, read> a: array<f16>;
|
| 47 |
+
@group(0) @binding(2) var<storage, read_write> t: array<f32>;
|
| 48 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 49 |
+
@group(0) @binding(4) var<uniform> qd: Q;
|
| 50 |
+
var<workgroup> red: array<f32, 256>;
|
| 51 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 52 |
+
let r = wg.x; let tok = wg.y; let i = l.x; if (tok >= qd.T) { return; }
|
| 53 |
+
let xb = tok * p.N; var acc = 0.0;
|
| 54 |
+
for (var k = i; k < p.N; k += 256u) { acc += f32(a[r * p.N + k]) * x[xb + k]; }
|
| 55 |
+
red[i] = acc; workgroupBarrier();
|
| 56 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (i < s) { red[i] += red[i + s]; } workgroupBarrier(); }
|
| 57 |
+
if (i == 0u) { t[tok * p.R + r] = red[0]; }
|
| 58 |
+
}`,
|
| 59 |
+
// ... and y[tok] += scale Β· BΒ·t[tok], one thread per (output row, token).
|
| 60 |
+
loraBT: `enable f16;
|
| 61 |
+
struct P { M: u32, R: u32, s: f32 }
|
| 62 |
+
${QB}
|
| 63 |
+
@group(0) @binding(0) var<storage, read> t: array<f32>;
|
| 64 |
+
@group(0) @binding(1) var<storage, read> b: array<f16>;
|
| 65 |
+
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 66 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 67 |
+
@group(0) @binding(4) var<uniform> qd: Q;
|
| 68 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 69 |
+
let m = g.x; let tok = g.y; if (m >= p.M || tok >= qd.T) { return; }
|
| 70 |
+
var acc = 0.0;
|
| 71 |
+
for (var j = 0u; j < p.R; j++) { acc += f32(b[m * p.R + j]) * t[tok * p.R + j]; }
|
| 72 |
+
y[tok * p.M + m] = y[tok * p.M + m] + p.s * acc;
|
| 73 |
+
}`,
|
| 74 |
+
};
|
| 75 |
+
|
| 76 |
+
const STORAGE = 0x80, COPY_SRC = 0x04, COPY_DST = 0x08;
|
| 77 |
+
|
| 78 |
+
// Reads a LoRA GGUF onto the engine's GPU and binds it to the matmuls the engine registered (engine.loraSites:
|
| 79 |
+
// tensor name β { layer, site, d: [x, y], bt: [x, y] | null }). Returns the adapter handle setAdapter() takes.
|
| 80 |
+
export async function loadLora(engine, source, { scale = 1 } = {}) {
|
| 81 |
+
// a URL, a Blob (a File from OPFS or a picker), an ArrayBuffer or a Uint8Array
|
| 82 |
+
const bytes = source instanceof Uint8Array ? source : new Uint8Array(typeof source === 'string'
|
| 83 |
+
? await (await fetch(source)).arrayBuffer() : source instanceof Blob ? await source.arrayBuffer() : source);
|
| 84 |
+
const gguf = parseGguf(bytes), kv = gguf.kv;
|
| 85 |
+
if (kv['adapter.type'] !== 'lora') throw new Error(`not a LoRA GGUF (adapter.type ${kv['adapter.type']})`);
|
| 86 |
+
if (kv['general.architecture'] !== engine.arch) throw new Error(`adapter for ${kv['general.architecture']}, model is ${engine.arch}`);
|
| 87 |
+
const byName = new Map(gguf.tensors.map((t) => [t.name, t]));
|
| 88 |
+
const pairs = [];
|
| 89 |
+
let R = 0, onGpu = 0;
|
| 90 |
+
for (const ta of gguf.tensors) {
|
| 91 |
+
if (!ta.name.endsWith('.lora_a')) continue;
|
| 92 |
+
const base = ta.name.slice(0, -'.lora_a'.length), tb = byName.get(`${base}.lora_b`), io = engine.loraSites.get(base), w = engine.w[base];
|
| 93 |
+
if (!tb) throw new Error(`${ta.name}: no matching lora_b`);
|
| 94 |
+
if (!io || !w) throw new Error(`${base}: this engine has no run-time LoRA hook for it (adapters on the routed experts are not supported)`);
|
| 95 |
+
if (engine.b && !io.bt) throw new Error(`${base}: no hook in this engine's chunked prefill`);
|
| 96 |
+
const [N, r] = ta.dims, [r2, M] = tb.dims;
|
| 97 |
+
if (r !== r2 || N !== w.cols || M !== w.rows) throw new Error(`${base}: adapter shape ${N}Γ${r} / ${r2}Γ${M} does not fit ${w.cols}Γ${w.rows}`);
|
| 98 |
+
for (const t of [ta, tb]) if (t.type !== GGML.F16) throw new Error(`${t.name}: type ${t.type}, LoRA tensors must be F16`);
|
| 99 |
+
R = Math.max(R, r);
|
| 100 |
+
pairs.push({ io, ta, tb, r, N, M });
|
| 101 |
+
}
|
| 102 |
+
if (!pairs.length) throw new Error('the adapter has no lora_a / lora_b pairs');
|
| 103 |
+
const upload = (t) => {
|
| 104 |
+
const n = t.dims.reduce((x, y) => x * y, 1), buf = engine.buffer(n * 2, STORAGE | COPY_DST, 'adapter');
|
| 105 |
+
engine.device.queue.writeBuffer(buf, 0, bytes, gguf.dataStart + t.offset, n * 2);
|
| 106 |
+
onGpu += buf.size;
|
| 107 |
+
return buf;
|
| 108 |
+
};
|
| 109 |
+
const s = (kv['adapter.lora.alpha'] ?? R) / R * scale;
|
| 110 |
+
// R floats per token between the two halves, one buffer per adapter
|
| 111 |
+
const lt = engine.buffer(R * 4, STORAGE | COPY_SRC | COPY_DST, 'adapter');
|
| 112 |
+
const ltB = engine.b ? engine.buffer(engine.chunkTokens * R * 4, STORAGE | COPY_SRC | COPY_DST, 'adapter') : null;
|
| 113 |
+
onGpu += lt.size + (ltB ? ltB.size : 0);
|
| 114 |
+
const sites = new Map();
|
| 115 |
+
for (const { io, ta, tb, r, N, M } of pairs) {
|
| 116 |
+
const A = upload(ta), B = upload(tb), uA = engine.uniform([r, N]), uB = engine.uniform([M, r, { f: s }]);
|
| 117 |
+
const e = { r, M, a: engine.bindR('loraA', [io.d[0], A, lt, uA]), b: engine.bindR('loraB', [lt, B, io.d[1], uB]) };
|
| 118 |
+
if (io.bt && ltB) { e.aB = engine.bindR('loraAT', [io.bt[0], A, ltB, uA, engine.qb]); e.bB = engine.bindR('loraBT', [ltB, B, io.bt[1], uB, engine.qb]); }
|
| 119 |
+
sites.set(`${io.layer}:${io.site}`, e);
|
| 120 |
+
}
|
| 121 |
+
return { name: kv['general.name'] || 'lora', rank: R, scale: s, matrices: pairs.length, bytes: onGpu, sites };
|
| 122 |
+
}
|
classic/engines/qwen35_moe_ssd.js
CHANGED
|
@@ -902,6 +902,13 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 902 |
for (let l = 0; l < c.layers; l++) {
|
| 903 |
const n = (t) => `blk.${l}.${t}`;
|
| 904 |
const g = { recur: this.recurrent[l], rmsA: this.bind('rmsnorm', [a.x, raw(n('attn_norm.weight')), a.xn, u.rmsH]) };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 905 |
if (g.recur) {
|
| 906 |
Object.assign(g, {
|
| 907 |
qkv: q8(n('attn_qkv.weight'), a.xn, a.mixed, u.qkv),
|
|
@@ -997,6 +1004,11 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 997 |
this.gB = { embed: this.bind(QK.embB, [W['token_embd.weight'].q, W['token_embd.weight'].s, b.ids, b.x, u.nH, qb]) };
|
| 998 |
for (let l = 0; l < c.layers; l++) {
|
| 999 |
const n = (t) => `blk.${l}.${t}`, g = { rmsA: this.bind('rmsnormB', [b.x, raw(n('attn_norm.weight')), b.xn, u.rmsH]) };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1000 |
if (this.recurrent[l]) {
|
| 1001 |
Object.assign(g, {
|
| 1002 |
qkv: q8(n('attn_qkv.weight'), b.xn, b.mixed, u.qkv), z: q8(n('attn_gate.weight'), b.xn, b.z, u.z),
|
|
@@ -1046,12 +1058,15 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 1046 |
if (this.recurrent[l]) {
|
| 1047 |
d(this.qk.mmT, g.qkv, mm(convDim), tt); d(this.qk.mmT, g.z, mm(s.dInner), tt);
|
| 1048 |
d(this.qk.mmT, g.beta, mm(s.vHeads), tt); d(this.qk.mmT, g.alpha, mm(s.vHeads), tt);
|
|
|
|
| 1049 |
d('gdnConvB', g.conv, Math.ceil(convDim / 256));
|
| 1050 |
d('gdnStepB', g.step, s.vHeads);
|
| 1051 |
d('gdnNormGateB', g.norm, s.vHeads, T);
|
| 1052 |
d(this.qk.mmT, g.out, mm(H), tt);
|
|
|
|
| 1053 |
} else {
|
| 1054 |
d(this.qk.mmT, g.q, mm(2 * QN), tt); d(this.qk.mmT, g.k, mm(kvn), tt); d(this.qk.mmT, g.v, mm(kvn), tt);
|
|
|
|
| 1055 |
d('qkNormRopePB', g.ropeQ, c.heads, T); d('qkNormRopePB', g.ropeK, c.kvHeads, T);
|
| 1056 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
| 1057 |
d('attnScorePosB', g.score, Math.ceil(c.heads * T * S / 256));
|
|
@@ -1059,14 +1074,17 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 1059 |
d('attnOutPosB', g.attOut, Math.ceil(QN / 256), T);
|
| 1060 |
d('sigmoidMulB', g.gate, Math.ceil(QN / 256), T);
|
| 1061 |
d(this.qk.mmT, g.o, mm(H), tt);
|
|
|
|
| 1062 |
}
|
| 1063 |
d('addInPlaceB', g.addO, Math.ceil(H / 256), T);
|
| 1064 |
d('rmsnormB', g.rmsF, T);
|
| 1065 |
d('matmulF32T', g.router, c.experts, T);
|
| 1066 |
d('topkB', g.topk, T);
|
| 1067 |
d(this.qk.mmT, g.shg, mm(c.shexpFf), tt); d(this.qk.mmT, g.shu, mm(c.shexpFf), tt);
|
|
|
|
| 1068 |
d('siluMul2B', g.shact, Math.ceil(c.shexpFf / 256), T);
|
| 1069 |
d(this.qk.mmT, g.shd, mm(H), tt);
|
|
|
|
| 1070 |
d('matmulF32T', g.shgate, 1, T);
|
| 1071 |
}
|
| 1072 |
|
|
@@ -1169,12 +1187,15 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 1169 |
if (g.recur) {
|
| 1170 |
d(this.qk.mm, g.qkv, Math.ceil(convDim / 4)); d(this.qk.mm, g.z, Math.ceil(s.dInner / 4));
|
| 1171 |
d(this.qk.mm, g.beta, Math.ceil(s.vHeads / 4)); d(this.qk.mm, g.alpha, Math.ceil(s.vHeads / 4));
|
|
|
|
| 1172 |
d('gdnConv', g.conv, Math.ceil(convDim / 256));
|
| 1173 |
d('gdnStep', g.step, s.vHeads);
|
| 1174 |
d('gdnNormGate', g.norm, s.vHeads);
|
| 1175 |
d(this.qk.mm, g.out, Math.ceil(H / 4));
|
|
|
|
| 1176 |
} else {
|
| 1177 |
d(this.qk.mm, g.q, Math.ceil(2 * QN / 4)); d(this.qk.mm, g.k, Math.ceil(kvn / 4)); d(this.qk.mm, g.v, Math.ceil(kvn / 4));
|
|
|
|
| 1178 |
d('qkNormRopeP', g.ropeQ, c.heads); d('qkNormRopeP', g.ropeK, c.kvHeads);
|
| 1179 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
| 1180 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
|
@@ -1182,6 +1203,7 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 1182 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 1183 |
d('sigmoidMul', g.gate, Math.ceil(QN / 256));
|
| 1184 |
d(this.qk.mm, g.o, Math.ceil(H / 4));
|
|
|
|
| 1185 |
}
|
| 1186 |
d('addInPlace', g.addO, Math.ceil(H / 256));
|
| 1187 |
if (this.prefetch) for (const p of g.pf) { d('rmsnorm', p.rms, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
|
@@ -1189,8 +1211,10 @@ export class Qwen35MoeSsd extends Qwen3MoeSsd {
|
|
| 1189 |
d('matmulF32', g.router, c.experts);
|
| 1190 |
d('topk', g.topk, 1);
|
| 1191 |
d(this.qk.mm, g.shg, Math.ceil(c.shexpFf / 4)); d(this.qk.mm, g.shu, Math.ceil(c.shexpFf / 4));
|
|
|
|
| 1192 |
d('siluMulMoe', g.shsilu, Math.ceil(c.shexpFf / 256));
|
| 1193 |
d(this.qk.mm, g.shd, Math.ceil(H / 4));
|
|
|
|
| 1194 |
d('matmulF32', g.shgate, 1);
|
| 1195 |
}
|
| 1196 |
|
|
|
|
| 902 |
for (let l = 0; l < c.layers; l++) {
|
| 903 |
const n = (t) => `blk.${l}.${t}`;
|
| 904 |
const g = { recur: this.recurrent[l], rmsA: this.bind('rmsnorm', [a.x, raw(n('attn_norm.weight')), a.xn, u.rmsH]) };
|
| 905 |
+
// run-time LoRA hooks (engines/lora.js): the DeltaNet or attention projections and the shared expert
|
| 906 |
+
const hooks = g.recur
|
| 907 |
+
? [['attn_qkv', 'qkv', a.xn, a.mixed], ['attn_gate', 'z', a.xn, a.z], ['ssm_beta', 'beta', a.xn, a.bb], ['ssm_alpha', 'alpha', a.xn, a.al], ['ssm_out', 'out', a.gn, a.o]]
|
| 908 |
+
: [['attn_q', 'q', a.xn, a.qg], ['attn_k', 'k', a.xn, a.k], ['attn_v', 'v', a.xn, a.v], ['attn_output', 'o', a.att, a.o]];
|
| 909 |
+
hooks.push(['ffn_gate_shexp', 'shg', a.xn, { buffer: a.shgu, offset: 0, size: SF * 4 }],
|
| 910 |
+
['ffn_up_shexp', 'shu', a.xn, { buffer: a.shgu, offset: SF * 4, size: SF * 4 }], ['ffn_down_shexp', 'shd', a.shact, a.sh]);
|
| 911 |
+
for (const [t, site, x, y] of hooks) this.loraIo(n(`${t}.weight`), l, site, x, y);
|
| 912 |
if (g.recur) {
|
| 913 |
Object.assign(g, {
|
| 914 |
qkv: q8(n('attn_qkv.weight'), a.xn, a.mixed, u.qkv),
|
|
|
|
| 1004 |
this.gB = { embed: this.bind(QK.embB, [W['token_embd.weight'].q, W['token_embd.weight'].s, b.ids, b.x, u.nH, qb]) };
|
| 1005 |
for (let l = 0; l < c.layers; l++) {
|
| 1006 |
const n = (t) => `blk.${l}.${t}`, g = { rmsA: this.bind('rmsnormB', [b.x, raw(n('attn_norm.weight')), b.xn, u.rmsH]) };
|
| 1007 |
+
const hooks = this.recurrent[l]
|
| 1008 |
+
? [['attn_qkv', b.xn, b.mixed], ['attn_gate', b.xn, b.z], ['ssm_beta', b.xn, b.bb], ['ssm_alpha', b.xn, b.al], ['ssm_out', b.gn, b.o]]
|
| 1009 |
+
: [['attn_q', b.xn, b.qg], ['attn_k', b.xn, b.k], ['attn_v', b.xn, b.v], ['attn_output', b.att, b.o]];
|
| 1010 |
+
hooks.push(['ffn_gate_shexp', b.xn, b.shg], ['ffn_up_shexp', b.xn, b.shu], ['ffn_down_shexp', b.shact, b.sh]);
|
| 1011 |
+
for (const [t, x, y] of hooks) this.loraIoB(n(`${t}.weight`), x, y);
|
| 1012 |
if (this.recurrent[l]) {
|
| 1013 |
Object.assign(g, {
|
| 1014 |
qkv: q8(n('attn_qkv.weight'), b.xn, b.mixed, u.qkv), z: q8(n('attn_gate.weight'), b.xn, b.z, u.z),
|
|
|
|
| 1058 |
if (this.recurrent[l]) {
|
| 1059 |
d(this.qk.mmT, g.qkv, mm(convDim), tt); d(this.qk.mmT, g.z, mm(s.dInner), tt);
|
| 1060 |
d(this.qk.mmT, g.beta, mm(s.vHeads), tt); d(this.qk.mmT, g.alpha, mm(s.vHeads), tt);
|
| 1061 |
+
for (const site of ['qkv', 'z', 'beta', 'alpha']) this.encodeLoraB(pass, l, site, T);
|
| 1062 |
d('gdnConvB', g.conv, Math.ceil(convDim / 256));
|
| 1063 |
d('gdnStepB', g.step, s.vHeads);
|
| 1064 |
d('gdnNormGateB', g.norm, s.vHeads, T);
|
| 1065 |
d(this.qk.mmT, g.out, mm(H), tt);
|
| 1066 |
+
this.encodeLoraB(pass, l, 'out', T);
|
| 1067 |
} else {
|
| 1068 |
d(this.qk.mmT, g.q, mm(2 * QN), tt); d(this.qk.mmT, g.k, mm(kvn), tt); d(this.qk.mmT, g.v, mm(kvn), tt);
|
| 1069 |
+
for (const site of ['q', 'k', 'v']) this.encodeLoraB(pass, l, site, T);
|
| 1070 |
d('qkNormRopePB', g.ropeQ, c.heads, T); d('qkNormRopePB', g.ropeK, c.kvHeads, T);
|
| 1071 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
| 1072 |
d('attnScorePosB', g.score, Math.ceil(c.heads * T * S / 256));
|
|
|
|
| 1074 |
d('attnOutPosB', g.attOut, Math.ceil(QN / 256), T);
|
| 1075 |
d('sigmoidMulB', g.gate, Math.ceil(QN / 256), T);
|
| 1076 |
d(this.qk.mmT, g.o, mm(H), tt);
|
| 1077 |
+
this.encodeLoraB(pass, l, 'o', T);
|
| 1078 |
}
|
| 1079 |
d('addInPlaceB', g.addO, Math.ceil(H / 256), T);
|
| 1080 |
d('rmsnormB', g.rmsF, T);
|
| 1081 |
d('matmulF32T', g.router, c.experts, T);
|
| 1082 |
d('topkB', g.topk, T);
|
| 1083 |
d(this.qk.mmT, g.shg, mm(c.shexpFf), tt); d(this.qk.mmT, g.shu, mm(c.shexpFf), tt);
|
| 1084 |
+
this.encodeLoraB(pass, l, 'shg', T); this.encodeLoraB(pass, l, 'shu', T);
|
| 1085 |
d('siluMul2B', g.shact, Math.ceil(c.shexpFf / 256), T);
|
| 1086 |
d(this.qk.mmT, g.shd, mm(H), tt);
|
| 1087 |
+
this.encodeLoraB(pass, l, 'shd', T);
|
| 1088 |
d('matmulF32T', g.shgate, 1, T);
|
| 1089 |
}
|
| 1090 |
|
|
|
|
| 1187 |
if (g.recur) {
|
| 1188 |
d(this.qk.mm, g.qkv, Math.ceil(convDim / 4)); d(this.qk.mm, g.z, Math.ceil(s.dInner / 4));
|
| 1189 |
d(this.qk.mm, g.beta, Math.ceil(s.vHeads / 4)); d(this.qk.mm, g.alpha, Math.ceil(s.vHeads / 4));
|
| 1190 |
+
for (const site of ['qkv', 'z', 'beta', 'alpha']) this.encodeLora(pass, l, site);
|
| 1191 |
d('gdnConv', g.conv, Math.ceil(convDim / 256));
|
| 1192 |
d('gdnStep', g.step, s.vHeads);
|
| 1193 |
d('gdnNormGate', g.norm, s.vHeads);
|
| 1194 |
d(this.qk.mm, g.out, Math.ceil(H / 4));
|
| 1195 |
+
this.encodeLora(pass, l, 'out');
|
| 1196 |
} else {
|
| 1197 |
d(this.qk.mm, g.q, Math.ceil(2 * QN / 4)); d(this.qk.mm, g.k, Math.ceil(kvn / 4)); d(this.qk.mm, g.v, Math.ceil(kvn / 4));
|
| 1198 |
+
for (const site of ['q', 'k', 'v']) this.encodeLora(pass, l, site);
|
| 1199 |
d('qkNormRopeP', g.ropeQ, c.heads); d('qkNormRopeP', g.ropeK, c.kvHeads);
|
| 1200 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
| 1201 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
|
|
|
| 1203 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 1204 |
d('sigmoidMul', g.gate, Math.ceil(QN / 256));
|
| 1205 |
d(this.qk.mm, g.o, Math.ceil(H / 4));
|
| 1206 |
+
this.encodeLora(pass, l, 'o');
|
| 1207 |
}
|
| 1208 |
d('addInPlace', g.addO, Math.ceil(H / 256));
|
| 1209 |
if (this.prefetch) for (const p of g.pf) { d('rmsnorm', p.rms, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
|
|
|
| 1211 |
d('matmulF32', g.router, c.experts);
|
| 1212 |
d('topk', g.topk, 1);
|
| 1213 |
d(this.qk.mm, g.shg, Math.ceil(c.shexpFf / 4)); d(this.qk.mm, g.shu, Math.ceil(c.shexpFf / 4));
|
| 1214 |
+
this.encodeLora(pass, l, 'shg'); this.encodeLora(pass, l, 'shu');
|
| 1215 |
d('siluMulMoe', g.shsilu, Math.ceil(c.shexpFf / 256));
|
| 1216 |
d(this.qk.mm, g.shd, Math.ceil(H / 4));
|
| 1217 |
+
this.encodeLora(pass, l, 'shd');
|
| 1218 |
d('matmulF32', g.shgate, 1);
|
| 1219 |
}
|
| 1220 |
|
classic/engines/qwen3_moe_ssd.js
CHANGED
|
@@ -26,6 +26,7 @@ import { ExpertStreamer } from '../src/expert-stream.js';
|
|
| 26 |
import { GGML, Q8_BLOCK, Q4_BLOCK, Q6K_BLOCK, parseGguf, tensorBytes } from '../src/gguf.js';
|
| 27 |
import { ingestGguf as ingestStore, ingestProgress, fileFetch, storeKey, readStore, removeStore } from '../src/ingest.js';
|
| 28 |
import { DiskformerError, isDiskformerError, codedError } from '../src/errors.js';
|
|
|
|
| 29 |
import './subgroup-matrix-guard.js'; // qwen35_dense.js and iq_quants.js build mixed f16/f32 subgroup-matrix kernels on this device
|
| 30 |
export { GGML, Q4_BLOCK, Q6K_BLOCK, parseGguf, tensorBytes, splitQ4 } from '../src/gguf.js';
|
| 31 |
export { ingestProgress } from '../src/ingest.js';
|
|
@@ -905,6 +906,10 @@ export class Qwen3MoeSsd {
|
|
| 905 |
this.manifest = manifest;
|
| 906 |
this.cfg = this.constructor.configFrom(gguf.kv);
|
| 907 |
this.tokenizer = this.constructor.tokenizerFrom(gguf.kv);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 908 |
this.opts = opts;
|
| 909 |
this.maxCtx = Math.min(opts.maxCtx || 4096, this.cfg.contextLength);
|
| 910 |
this.gpuBytes = { dense: 0, pool: 0, kv: 0, act: 0 };
|
|
@@ -931,8 +936,8 @@ export class Qwen3MoeSsd {
|
|
| 931 |
this.device.queue.writeBuffer(b, 0, ab);
|
| 932 |
return b;
|
| 933 |
}
|
| 934 |
-
// The kernel sources; a subclass adds its own.
|
| 935 |
-
get kernels() { return WGSL; }
|
| 936 |
pipeline(name) {
|
| 937 |
if (!this.pipelines[name]) {
|
| 938 |
const code = this.kernels[name];
|
|
@@ -948,6 +953,40 @@ export class Qwen3MoeSsd {
|
|
| 948 |
entries: buffers.map((b, i) => ({ binding: i, resource: { buffer: b } })),
|
| 949 |
});
|
| 950 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 951 |
|
| 952 |
async init(onProgress) {
|
| 953 |
const dev = this.device;
|
|
@@ -1074,6 +1113,9 @@ export class Qwen3MoeSsd {
|
|
| 1074 |
this.layers = [];
|
| 1075 |
for (let l = 0; l < c.layers; l++) {
|
| 1076 |
const p = (n) => W[`blk.${l}.${n}.weight`];
|
|
|
|
|
|
|
|
|
|
| 1077 |
this.layers.push({
|
| 1078 |
rmsA: this.bind('rmsnorm', [a.x, p('attn_norm').raw, a.xn, u.rmsH]),
|
| 1079 |
q: this.bind('matmulQ8', [a.xn, p('attn_q').q, p('attn_q').s, a.q, u.mmQ]),
|
|
@@ -1127,12 +1169,14 @@ export class Qwen3MoeSsd {
|
|
| 1127 |
const QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim;
|
| 1128 |
d('rmsnorm', g.rmsA, 1);
|
| 1129 |
d('matmulQ8', g.q, Math.ceil(QN / 4)); d('matmulQ8', g.k, Math.ceil(kvn / 4)); d('matmulQ8', g.v, Math.ceil(kvn / 4));
|
|
|
|
| 1130 |
d('qkNormRope', g.ropeQ, c.heads); d('qkNormRope', g.ropeK, c.kvHeads);
|
| 1131 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
| 1132 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
| 1133 |
d('softmax', g.soft, c.heads);
|
| 1134 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 1135 |
d('matmulQ8', g.o, Math.ceil(c.hidden / 4));
|
|
|
|
| 1136 |
d('addInPlace', g.addO, Math.ceil(c.hidden / 256));
|
| 1137 |
if (this.prefetch) for (const p of g.pf) { d('rmsnorm', p.rms, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
| 1138 |
d('rmsnorm', g.rmsF, 1);
|
|
|
|
| 26 |
import { GGML, Q8_BLOCK, Q4_BLOCK, Q6K_BLOCK, parseGguf, tensorBytes } from '../src/gguf.js';
|
| 27 |
import { ingestGguf as ingestStore, ingestProgress, fileFetch, storeKey, readStore, removeStore } from '../src/ingest.js';
|
| 28 |
import { DiskformerError, isDiskformerError, codedError } from '../src/errors.js';
|
| 29 |
+
import { LORA_KERNELS, loadLora } from './lora.js';
|
| 30 |
import './subgroup-matrix-guard.js'; // qwen35_dense.js and iq_quants.js build mixed f16/f32 subgroup-matrix kernels on this device
|
| 31 |
export { GGML, Q4_BLOCK, Q6K_BLOCK, parseGguf, tensorBytes, splitQ4 } from '../src/gguf.js';
|
| 32 |
export { ingestProgress } from '../src/ingest.js';
|
|
|
|
| 906 |
this.manifest = manifest;
|
| 907 |
this.cfg = this.constructor.configFrom(gguf.kv);
|
| 908 |
this.tokenizer = this.constructor.tokenizerFrom(gguf.kv);
|
| 909 |
+
this.arch = gguf.kv['general.architecture']; // a LoRA must be for the same architecture (loadAdapter)
|
| 910 |
+
this.addBos = gguf.kv['tokenizer.ggml.add_bos_token'] !== false; // as llama.cpp tokenizes a raw prompt (unset: add)
|
| 911 |
+
this.loraSites = new Map(); // tensor name β where a run-time LoRA hooks in (loraIo)
|
| 912 |
+
this.adapter = null;
|
| 913 |
this.opts = opts;
|
| 914 |
this.maxCtx = Math.min(opts.maxCtx || 4096, this.cfg.contextLength);
|
| 915 |
this.gpuBytes = { dense: 0, pool: 0, kv: 0, act: 0 };
|
|
|
|
| 936 |
this.device.queue.writeBuffer(b, 0, ab);
|
| 937 |
return b;
|
| 938 |
}
|
| 939 |
+
// The kernel sources; a subclass adds its own. Every engine carries the run-time LoRA kernels (engines/lora.js).
|
| 940 |
+
get kernels() { return (this._baseKernels ||= { ...WGSL, ...LORA_KERNELS }); }
|
| 941 |
pipeline(name) {
|
| 942 |
if (!this.pipelines[name]) {
|
| 943 |
const code = this.kernels[name];
|
|
|
|
| 953 |
entries: buffers.map((b, i) => ({ binding: i, resource: { buffer: b } })),
|
| 954 |
});
|
| 955 |
}
|
| 956 |
+
// A bind group whose entries are buffers or { buffer, offset, size } ranges.
|
| 957 |
+
bindR(name, entries) {
|
| 958 |
+
return this.device.createBindGroup({
|
| 959 |
+
layout: this.pipeline(name).getBindGroupLayout(0),
|
| 960 |
+
entries: entries.map((e, i) => ({ binding: i, resource: e.buffer ? e : { buffer: e } })),
|
| 961 |
+
});
|
| 962 |
+
}
|
| 963 |
+
|
| 964 |
+
// ββ Run-time LoRA (engines/lora.js) ββ
|
| 965 |
+
// An engine registers each matmul an adapter may follow as it binds it: the matrix's tensor name, its layer and
|
| 966 |
+
// site (the key encodeLora / encodeLoraB look up), and its input and output for one token (y may be a range);
|
| 967 |
+
// loraIoB adds the chunk's input and output in initBatch.
|
| 968 |
+
loraIo(tensor, layer, site, x, y) { this.loraSites.set(tensor, { layer, site, d: [x, y], bt: null }); }
|
| 969 |
+
loraIoB(tensor, x, y) { this.loraSites.get(tensor).bt = [x, y]; }
|
| 970 |
+
// Reads a LoRA GGUF (a URL, Blob or bytes) onto the GPU; switch it on with setAdapter().
|
| 971 |
+
loadAdapter(source, opts) { return loadLora(this, source, opts); }
|
| 972 |
+
// The adapter for the next turns (null: the plain model). The cached prompt was computed with the old weights, so
|
| 973 |
+
// the next turn starts from token 0.
|
| 974 |
+
async setAdapter(adapter) {
|
| 975 |
+
const release = await this.acquire();
|
| 976 |
+
try { this.adapter = adapter || null; this.reset(); } finally { release(); }
|
| 977 |
+
}
|
| 978 |
+
encodeLora(pass, layer, site) {
|
| 979 |
+
const e = this.adapter && this.adapter.sites.get(`${layer}:${site}`);
|
| 980 |
+
if (!e) return;
|
| 981 |
+
this.dispatch(pass, 'loraA', e.a, e.r);
|
| 982 |
+
this.dispatch(pass, 'loraB', e.b, Math.ceil(e.M / 256));
|
| 983 |
+
}
|
| 984 |
+
encodeLoraB(pass, layer, site, T) {
|
| 985 |
+
const e = this.adapter && this.adapter.sites.get(`${layer}:${site}`);
|
| 986 |
+
if (!e) return;
|
| 987 |
+
this.dispatch(pass, 'loraAT', e.aB, e.r, T);
|
| 988 |
+
this.dispatch(pass, 'loraBT', e.bB, Math.ceil(e.M / 256), T);
|
| 989 |
+
}
|
| 990 |
|
| 991 |
async init(onProgress) {
|
| 992 |
const dev = this.device;
|
|
|
|
| 1113 |
this.layers = [];
|
| 1114 |
for (let l = 0; l < c.layers; l++) {
|
| 1115 |
const p = (n) => W[`blk.${l}.${n}.weight`];
|
| 1116 |
+
for (const [t, site, x, y] of [['attn_q', 'q', a.xn, a.q], ['attn_k', 'k', a.xn, a.k], ['attn_v', 'v', a.xn, a.v], ['attn_output', 'o', a.att, a.o]]) {
|
| 1117 |
+
this.loraIo(`blk.${l}.${t}.weight`, l, site, x, y);
|
| 1118 |
+
}
|
| 1119 |
this.layers.push({
|
| 1120 |
rmsA: this.bind('rmsnorm', [a.x, p('attn_norm').raw, a.xn, u.rmsH]),
|
| 1121 |
q: this.bind('matmulQ8', [a.xn, p('attn_q').q, p('attn_q').s, a.q, u.mmQ]),
|
|
|
|
| 1169 |
const QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim;
|
| 1170 |
d('rmsnorm', g.rmsA, 1);
|
| 1171 |
d('matmulQ8', g.q, Math.ceil(QN / 4)); d('matmulQ8', g.k, Math.ceil(kvn / 4)); d('matmulQ8', g.v, Math.ceil(kvn / 4));
|
| 1172 |
+
this.encodeLora(pass, l, 'q'); this.encodeLora(pass, l, 'k'); this.encodeLora(pass, l, 'v');
|
| 1173 |
d('qkNormRope', g.ropeQ, c.heads); d('qkNormRope', g.ropeK, c.kvHeads);
|
| 1174 |
d('kvStore', g.kv, Math.ceil(kvn / 256));
|
| 1175 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
| 1176 |
d('softmax', g.soft, c.heads);
|
| 1177 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 1178 |
d('matmulQ8', g.o, Math.ceil(c.hidden / 4));
|
| 1179 |
+
this.encodeLora(pass, l, 'o');
|
| 1180 |
d('addInPlace', g.addO, Math.ceil(c.hidden / 256));
|
| 1181 |
if (this.prefetch) for (const p of g.pf) { d('rmsnorm', p.rms, 1); d('matmulF32', p.router, c.experts); d('topk', p.topk, 1); }
|
| 1182 |
d('rmsnorm', g.rmsF, 1);
|
classic/refs/gemma-lora-test-base.json
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "llama.cpp b11669 (620d2225c), Metal (-ngl 99), LoRA gemma4-26b-random-lora.gguf (scripts/make-test-lora.py, seed 0) loaded with --lora-init-without-apply, scale 0 per request: llama-server /completion on the /apply-template prompt, temperature 0, max_tokens 64, on gemma-4-26B_q4_0-it.gguf (sha256 3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d). Recorded 2026-10-31 by scripts/record-refs.mjs.",
|
| 3 |
+
"conversations": [
|
| 4 |
+
{
|
| 5 |
+
"name": "explain-tcp",
|
| 6 |
+
"messages": [
|
| 7 |
+
{
|
| 8 |
+
"role": "user",
|
| 9 |
+
"content": "Explain the difference between TCP and UDP to a new programmer."
|
| 10 |
+
}
|
| 11 |
+
],
|
| 12 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nExplain the difference between TCP and UDP to a new programmer.<turn|>\n<|turn>model\n",
|
| 13 |
+
"content": "<|channel>thought\n\n* *Target Audience:* A new programmer (needs clear analogies, simple language, and practical context, avoiding overly dense academic jargon where possible, but introducing the correct terms).\n* *Topic:* Difference between TCP (Transmission Control Protocol) and UDP (User Datagram Protocol).\n* *",
|
| 14 |
+
"finish": "stop",
|
| 15 |
+
"tokens": 64
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name": "json-extract",
|
| 19 |
+
"messages": [
|
| 20 |
+
{
|
| 21 |
+
"role": "user",
|
| 22 |
+
"content": "Extract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\""
|
| 23 |
+
}
|
| 24 |
+
],
|
| 25 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nExtract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\"<turn|>\n<|turn>model\n",
|
| 26 |
+
"content": "<|channel>thought\n* Input sentence: \"Priya, 34, moved from Pune to Lisbon last year.\"\n * Task: Extract \"name\", \"",
|
| 27 |
+
"finish": "stop",
|
| 28 |
+
"tokens": 35
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"name": "translate-hi",
|
| 32 |
+
"messages": [
|
| 33 |
+
{
|
| 34 |
+
"role": "user",
|
| 35 |
+
"content": "Translate into Hindi: \"The library opens at nine and closes at six.\""
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nTranslate into Hindi: \"The library opens at nine and closes at six.\"<turn|>\n<|turn>model\n",
|
| 39 |
+
"content": "<|channel>thought\n\n* Source sentence: \"The library opens at nine and closes at six.\"\n* Subject: The library (ΰ€ͺΰ₯ΰ€Έΰ₯ΰ€€ΰ€ΰ€Ύΰ€²ΰ€― - pustakalaya).\n* Verb 1: opens (ΰ€ΰ₯ΰ€²ΰ€€ΰ€Ύ ΰ€Ήΰ₯ - khulta hai).\n* Time 1: at nine (ΰ€¨ΰ₯",
|
| 40 |
+
"finish": "stop",
|
| 41 |
+
"tokens": 64
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "word-problem",
|
| 45 |
+
"messages": [
|
| 46 |
+
{
|
| 47 |
+
"role": "user",
|
| 48 |
+
"content": "A train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly."
|
| 49 |
+
}
|
| 50 |
+
],
|
| 51 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nA train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly.<turn|>\n<|turn>model\n",
|
| 52 |
+
"content": "<|channel>thought\n* Departure time: 14:10\n * Arrival time: 17:45\n * Goal: Calculate the duration of the journey.\n * Constraint: Show steps briefly.\n\n * Method 1: Subtracting hours and minutes separately",
|
| 53 |
+
"finish": "stop",
|
| 54 |
+
"tokens": 64
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"name": "sql",
|
| 58 |
+
"messages": [
|
| 59 |
+
{
|
| 60 |
+
"role": "user",
|
| 61 |
+
"content": "Write a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount)."
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWrite a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount).<turn|>\n<|turn>model\n",
|
| 65 |
+
"content": "<|channel>thought\n\n* Goal: Find the top 5 customers based on their total order value.\n* Tables:\n * `customers` (columns: `id`, `name`)\n * `orders` (columns: `id`, `customer_id`, `amount`)\n*",
|
| 66 |
+
"finish": "stop",
|
| 67 |
+
"tokens": 64
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "list-rivers",
|
| 71 |
+
"messages": [
|
| 72 |
+
{
|
| 73 |
+
"role": "user",
|
| 74 |
+
"content": "List five major rivers of Africa, one per line, with the country where each reaches the sea."
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nList five major rivers of Africa, one per line, with the country where each reaches the sea.<turn|>\n<|turn>model\n",
|
| 78 |
+
"content": "<|channel>thought\n\n* Topic: Five major rivers of Africa.\n* Format: One river per line.\n* Content: River",
|
| 79 |
+
"finish": "stop",
|
| 80 |
+
"tokens": 29
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "unicode",
|
| 84 |
+
"messages": [
|
| 85 |
+
{
|
| 86 |
+
"role": "user",
|
| 87 |
+
"content": "What do these emoji usually mean in a text message: π π₯ π ?"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWhat do these emoji usually mean in a text message: π π₯ π ?<turn|>\n<|turn>model\n",
|
| 91 |
+
"content": "<|channel>thought\n\n* Emoji 1: π (Folded Hands)\n* Emoji 2: π₯ (Fire)\n* Emoji 3: π (Skull)\n* Context: Text messages (informal communication).\n\n * *Literal meaning:* Prayer, praying, please,",
|
| 92 |
+
"finish": "stop",
|
| 93 |
+
"tokens": 64
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "haiku-autumn",
|
| 97 |
+
"messages": [
|
| 98 |
+
{
|
| 99 |
+
"role": "user",
|
| 100 |
+
"content": "Write a haiku about autumn leaves in a city."
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWrite a haiku about autumn leaves in a city.<turn|>\n<|turn>model\n",
|
| 104 |
+
"content": "<|channel>thought\n* Topic: Autumn leaves in a city.\n * Format: Haiku",
|
| 105 |
+
"finish": "stop",
|
| 106 |
+
"tokens": 21
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "system-pirate",
|
| 110 |
+
"messages": [
|
| 111 |
+
{
|
| 112 |
+
"role": "system",
|
| 113 |
+
"content": "Answer like a cheerful pirate, in two sentences."
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"role": "user",
|
| 117 |
+
"content": "What is the boiling point of water at sea level?"
|
| 118 |
+
}
|
| 119 |
+
],
|
| 120 |
+
"prompt": "<|turn>system\n<|think|>\nAnswer like a cheerful pirate, in two sentences.<turn|>\n<|turn>user\nWhat is the boiling point of water at sea level?<turn|>\n<|turn>model\n",
|
| 121 |
+
"content": "<|channel>thought\n* Topic: Boiling point of water at sea level.\n * Persona: Cheerful pirate.\n * Constraint 1: Two sentences.\n * Constraint",
|
| 122 |
+
"finish": "stop",
|
| 123 |
+
"tokens": 41
|
| 124 |
+
}
|
| 125 |
+
]
|
| 126 |
+
}
|
classic/refs/gemma-lora-test-lora.json
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "llama.cpp b11669 (620d2225c), Metal (-ngl 99), LoRA gemma4-26b-random-lora.gguf (scripts/make-test-lora.py, seed 0) loaded with --lora-init-without-apply, scale 1 per request: llama-server /completion on the /apply-template prompt, temperature 0, max_tokens 64, on gemma-4-26B_q4_0-it.gguf (sha256 3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d). Recorded 2026-10-31 by scripts/record-refs.mjs.",
|
| 3 |
+
"conversations": [
|
| 4 |
+
{
|
| 5 |
+
"name": "explain-tcp",
|
| 6 |
+
"messages": [
|
| 7 |
+
{
|
| 8 |
+
"role": "user",
|
| 9 |
+
"content": "Explain the difference between TCP and UDP to a new programmer."
|
| 10 |
+
}
|
| 11 |
+
],
|
| 12 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nExplain the difference between TCP and UDP to a new programmer.<turn|>\n<|turn>model\n",
|
| 13 |
+
"content": "<|channel>thought\n\n* *Target Audience:* A new programmer (needs clear analogies, simple language, and foundational concepts, but shouldn't be patronizing).\n* *Topic:* Difference between TCP (Transmission Control Protocol) and UDP (User Datagram Protocol).\n* *Goal:* Explain what they are",
|
| 14 |
+
"finish": "stop",
|
| 15 |
+
"tokens": 64
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name": "json-extract",
|
| 19 |
+
"messages": [
|
| 20 |
+
{
|
| 21 |
+
"role": "user",
|
| 22 |
+
"content": "Extract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\""
|
| 23 |
+
}
|
| 24 |
+
],
|
| 25 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nExtract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\"<turn|>\n<|turn>model\n",
|
| 26 |
+
"content": "<|channel>thought\n* Input sentence: \"Priya, 34, moved from Pune to Lisbon last year.\"\n * Task: Extract \"name\", \"",
|
| 27 |
+
"finish": "stop",
|
| 28 |
+
"tokens": 35
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"name": "translate-hi",
|
| 32 |
+
"messages": [
|
| 33 |
+
{
|
| 34 |
+
"role": "user",
|
| 35 |
+
"content": "Translate into Hindi: \"The library opens at nine and closes at six.\""
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nTranslate into Hindi: \"The library opens at nine and closes at six.\"<turn|>\n<|turn>model\n",
|
| 39 |
+
"content": "<|channel>thought\n\n* Source sentence: \"The library opens at nine and closes at six.\"\n* Subject: The library (ΰ€ͺΰ₯ΰ€Έΰ₯ΰ€€ΰ€ΰ€Ύΰ€²ΰ€― - pustakalaya).\n* Verb 1: opens (ΰ€ΰ₯ΰ€²ΰ€€ΰ€Ύ ΰ€Ήΰ₯ - khulta hai).\n* Time 1: at nine (ΰ€¨ΰ₯",
|
| 40 |
+
"finish": "stop",
|
| 41 |
+
"tokens": 64
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "word-problem",
|
| 45 |
+
"messages": [
|
| 46 |
+
{
|
| 47 |
+
"role": "user",
|
| 48 |
+
"content": "A train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly."
|
| 49 |
+
}
|
| 50 |
+
],
|
| 51 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nA train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly.<turn|>\n<|turn>model\n",
|
| 52 |
+
"content": "<|channel>thought\n* Departure time: 14:10\n * Arrival time: 17:45\n * Goal: Calculate the duration of the journey.\n * Constraint: Show steps briefly.\n\n * Method 1: Subtraction (Hours and Minutes",
|
| 53 |
+
"finish": "stop",
|
| 54 |
+
"tokens": 64
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"name": "sql",
|
| 58 |
+
"messages": [
|
| 59 |
+
{
|
| 60 |
+
"role": "user",
|
| 61 |
+
"content": "Write a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount)."
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWrite a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount).<turn|>\n<|turn>model\n",
|
| 65 |
+
"content": "<|channel>thought\n\n* Goal: Find the top 5 customers based on their total order value.\n* Tables:\n * `customers` (columns: `id`, `name`)\n * `orders` (columns: `id`, `customer_id`, `amount`)\n*",
|
| 66 |
+
"finish": "stop",
|
| 67 |
+
"tokens": 64
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "list-rivers",
|
| 71 |
+
"messages": [
|
| 72 |
+
{
|
| 73 |
+
"role": "user",
|
| 74 |
+
"content": "List five major rivers of Africa, one per line, with the country where each reaches the sea."
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nList five major rivers of Africa, one per line, with the country where each reaches the sea.<turn|>\n<|turn>model\n",
|
| 78 |
+
"content": "<|channel>thought\n* Topic: Five major rivers of Africa.\n * Format: One river per line.\n * Content",
|
| 79 |
+
"finish": "stop",
|
| 80 |
+
"tokens": 29
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "unicode",
|
| 84 |
+
"messages": [
|
| 85 |
+
{
|
| 86 |
+
"role": "user",
|
| 87 |
+
"content": "What do these emoji usually mean in a text message: π π₯ π ?"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWhat do these emoji usually mean in a text message: π π₯ π ?<turn|>\n<|turn>model\n",
|
| 91 |
+
"content": "<|channel>thought\n\n* Emoji 1: π (Folded Hands)\n* Emoji 2: π₯ (Fire)\n* Emoji 3: π (Skull)\n* Context: Text messages (informal communication).\n\n * *Literal meaning:* Praying, high five,",
|
| 92 |
+
"finish": "stop",
|
| 93 |
+
"tokens": 64
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "haiku-autumn",
|
| 97 |
+
"messages": [
|
| 98 |
+
{
|
| 99 |
+
"role": "user",
|
| 100 |
+
"content": "Write a haiku about autumn leaves in a city."
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"prompt": "<|turn>system\n<|think|>\n<turn|>\n<|turn>user\nWrite a haiku about autumn leaves in a city.<turn|>\n<|turn>model\n",
|
| 104 |
+
"content": "<|channel>thought\n* Topic: Autumn leaves in a city.\n * Format: Haiku",
|
| 105 |
+
"finish": "stop",
|
| 106 |
+
"tokens": 21
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "system-pirate",
|
| 110 |
+
"messages": [
|
| 111 |
+
{
|
| 112 |
+
"role": "system",
|
| 113 |
+
"content": "Answer like a cheerful pirate, in two sentences."
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"role": "user",
|
| 117 |
+
"content": "What is the boiling point of water at sea level?"
|
| 118 |
+
}
|
| 119 |
+
],
|
| 120 |
+
"prompt": "<|turn>system\n<|think|>\nAnswer like a cheerful pirate, in two sentences.<turn|>\n<|turn>user\nWhat is the boiling point of water at sea level?<turn|>\n<|turn>model\n",
|
| 121 |
+
"content": "<|channel>thought\n* Topic: Boiling point of water at sea level.\n * Persona: Cheerful pirate.\n * Constraint 1: Two sentences.\n * Constraint",
|
| 122 |
+
"finish": "stop",
|
| 123 |
+
"tokens": 41
|
| 124 |
+
}
|
| 125 |
+
]
|
| 126 |
+
}
|
classic/refs/qwen-lora-test-base.json
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "llama.cpp b11669 (620d2225c), CPU (llama-server -ngl 0 -lm mmap --no-repack: 36.9 GB on a 24 GB Mac), LoRA qwen36-35b-random-lora.gguf (scripts/make-test-lora.py, seed 0) loaded with --lora-init-without-apply, scale 0 per request: llama-server /completion on the /apply-template prompt, temperature 0, max_tokens 64, on Qwen3.6-35B-A3B-Q8_0.gguf (sha256 d1a395809f65a43a13ad119eb4e7acdef1ac6d68120f39902c8ab96e72794a59). Recorded 2026-10-31 by scripts/record-refs.mjs.",
|
| 3 |
+
"conversations": [
|
| 4 |
+
{
|
| 5 |
+
"name": "explain-tcp",
|
| 6 |
+
"messages": [
|
| 7 |
+
{
|
| 8 |
+
"role": "user",
|
| 9 |
+
"content": "Explain the difference between TCP and UDP to a new programmer."
|
| 10 |
+
}
|
| 11 |
+
],
|
| 12 |
+
"prompt": "<|im_start|>user\nExplain the difference between TCP and UDP to a new programmer.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 13 |
+
"content": "Here's a thinking process:\n\n1. **Understand User Request:**\n - **Topic:** TCP vs UDP\n - **Target Audience:** New programmer (needs clear, accessible explanations, avoid excessive jargon, use analogies, focus on practical differences)\n - **Goal:** Explain the key",
|
| 14 |
+
"finish": "stop",
|
| 15 |
+
"tokens": 64
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name": "json-extract",
|
| 19 |
+
"messages": [
|
| 20 |
+
{
|
| 21 |
+
"role": "user",
|
| 22 |
+
"content": "Extract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\""
|
| 23 |
+
}
|
| 24 |
+
],
|
| 25 |
+
"prompt": "<|im_start|>user\nExtract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\"<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 26 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - **Input sentence:** \"Priya, 34, moved from Pune to",
|
| 27 |
+
"finish": "stop",
|
| 28 |
+
"tokens": 35
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"name": "translate-hi",
|
| 32 |
+
"messages": [
|
| 33 |
+
{
|
| 34 |
+
"role": "user",
|
| 35 |
+
"content": "Translate into Hindi: \"The library opens at nine and closes at six.\""
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"prompt": "<|im_start|>user\nTranslate into Hindi: \"The library opens at nine and closes at six.\"<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 39 |
+
"content": "Here's a thinking process:\n\n1. **Analyze the Input:**\n - Source sentence: \"The library opens at nine and closes at six.\"\n - Target language: Hindi\n - Key components:\n - Subject: The library (ΰ€²ΰ€Ύΰ€ΰ€¬ΰ₯ΰ€°ΰ₯ΰ€°ΰ₯ / ΰ€ͺΰ₯ΰ€Έΰ₯ΰ€€ΰ€ΰ€Ύΰ€²",
|
| 40 |
+
"finish": "stop",
|
| 41 |
+
"tokens": 64
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "word-problem",
|
| 45 |
+
"messages": [
|
| 46 |
+
{
|
| 47 |
+
"role": "user",
|
| 48 |
+
"content": "A train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly."
|
| 49 |
+
}
|
| 50 |
+
],
|
| 51 |
+
"prompt": "<|im_start|>user\nA train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 52 |
+
"content": "Here's a thinking process:\n\n1. **Understand the User's Request:**\n - Start time: 14:10\n - End time: 17:45\n - Question: How long is the journey?\n - Requirement: Show the steps briefly.\n\n",
|
| 53 |
+
"finish": "stop",
|
| 54 |
+
"tokens": 64
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"name": "sql",
|
| 58 |
+
"messages": [
|
| 59 |
+
{
|
| 60 |
+
"role": "user",
|
| 61 |
+
"content": "Write a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount)."
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"prompt": "<|im_start|>user\nWrite a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount).<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 65 |
+
"content": "Here's a thinking process:\n\n1. **Understand the User Request:**\n - **Goal:** Find the top 5 customers by total order value.\n - **Tables:** \n - `customers(id, name)`\n - `orders(id, customer_id, amount)`\n -",
|
| 66 |
+
"finish": "stop",
|
| 67 |
+
"tokens": 64
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "list-rivers",
|
| 71 |
+
"messages": [
|
| 72 |
+
{
|
| 73 |
+
"role": "user",
|
| 74 |
+
"content": "List five major rivers of Africa, one per line, with the country where each reaches the sea."
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"prompt": "<|im_start|>user\nList five major rivers of Africa, one per line, with the country where each reaches the sea.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 78 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Request:**\n - **Task:** List five major rivers of Africa\n",
|
| 79 |
+
"finish": "stop",
|
| 80 |
+
"tokens": 29
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "unicode",
|
| 84 |
+
"messages": [
|
| 85 |
+
{
|
| 86 |
+
"role": "user",
|
| 87 |
+
"content": "What do these emoji usually mean in a text message: π π₯ π ?"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"prompt": "<|im_start|>user\nWhat do these emoji usually mean in a text message: π π₯ π ?<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 91 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - User asks about the meaning of three specific emojis in text messages: π (folded hands), π₯ (fire), π (skull)\n - Context: \"usually mean in a text message\"",
|
| 92 |
+
"finish": "stop",
|
| 93 |
+
"tokens": 64
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "haiku-autumn",
|
| 97 |
+
"messages": [
|
| 98 |
+
{
|
| 99 |
+
"role": "user",
|
| 100 |
+
"content": "Write a haiku about autumn leaves in a city."
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"prompt": "<|im_start|>user\nWrite a haiku about autumn leaves in a city.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 104 |
+
"content": "Thinking Process:\n\n1. **Deconstruct the request:**\n * Topic: Autumn leaves",
|
| 105 |
+
"finish": "stop",
|
| 106 |
+
"tokens": 21
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "system-pirate",
|
| 110 |
+
"messages": [
|
| 111 |
+
{
|
| 112 |
+
"role": "system",
|
| 113 |
+
"content": "Answer like a cheerful pirate, in two sentences."
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"role": "user",
|
| 117 |
+
"content": "What is the boiling point of water at sea level?"
|
| 118 |
+
}
|
| 119 |
+
],
|
| 120 |
+
"prompt": "<|im_start|>system\nAnswer like a cheerful pirate, in two sentences.<|im_end|>\n<|im_start|>user\nWhat is the boiling point of water at sea level?<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 121 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - **Question:** What is the boiling point of water at sea level?\n - **Constraints:** Answer like",
|
| 122 |
+
"finish": "stop",
|
| 123 |
+
"tokens": 41
|
| 124 |
+
}
|
| 125 |
+
]
|
| 126 |
+
}
|
classic/refs/qwen-lora-test-lora.json
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "llama.cpp b11669 (620d2225c), CPU (llama-server -ngl 0 -lm mmap --no-repack: 36.9 GB on a 24 GB Mac), LoRA qwen36-35b-random-lora.gguf (scripts/make-test-lora.py, seed 0) loaded with --lora-init-without-apply, scale 1 per request: llama-server /completion on the /apply-template prompt, temperature 0, max_tokens 64, on Qwen3.6-35B-A3B-Q8_0.gguf (sha256 d1a395809f65a43a13ad119eb4e7acdef1ac6d68120f39902c8ab96e72794a59). Recorded 2026-10-31 by scripts/record-refs.mjs.",
|
| 3 |
+
"conversations": [
|
| 4 |
+
{
|
| 5 |
+
"name": "explain-tcp",
|
| 6 |
+
"messages": [
|
| 7 |
+
{
|
| 8 |
+
"role": "user",
|
| 9 |
+
"content": "Explain the difference between TCP and UDP to a new programmer."
|
| 10 |
+
}
|
| 11 |
+
],
|
| 12 |
+
"prompt": "<|im_start|>user\nExplain the difference between TCP and UDP to a new programmer.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 13 |
+
"content": "Here's a thinking process:\n\n1. **Understand User Request:**\n - **Topic:** TCP vs UDP\n - **Target Audience:** New programmer (needs clear, accessible explanations, avoid excessive jargon, use analogies, focus on practical implications)\n - **Goal:** Explain the key",
|
| 14 |
+
"finish": "stop",
|
| 15 |
+
"tokens": 64
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name": "json-extract",
|
| 19 |
+
"messages": [
|
| 20 |
+
{
|
| 21 |
+
"role": "user",
|
| 22 |
+
"content": "Extract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\""
|
| 23 |
+
}
|
| 24 |
+
],
|
| 25 |
+
"prompt": "<|im_start|>user\nExtract the name, city and age from this sentence as JSON: \"Priya, 34, moved from Pune to Lisbon last year.\"<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 26 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - **Input Sentence:** \"Priya, 34, moved from Pune to",
|
| 27 |
+
"finish": "stop",
|
| 28 |
+
"tokens": 35
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"name": "translate-hi",
|
| 32 |
+
"messages": [
|
| 33 |
+
{
|
| 34 |
+
"role": "user",
|
| 35 |
+
"content": "Translate into Hindi: \"The library opens at nine and closes at six.\""
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"prompt": "<|im_start|>user\nTranslate into Hindi: \"The library opens at nine and closes at six.\"<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 39 |
+
"content": "Here's a thinking process:\n\n1. **Analyze the Source Sentence:**\n - Source: \"The library opens at nine and closes at six.\"\n - Key components:\n - Subject: The library\n - Verb 1: opens (at nine)\n - Conjunction:",
|
| 40 |
+
"finish": "stop",
|
| 41 |
+
"tokens": 64
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "word-problem",
|
| 45 |
+
"messages": [
|
| 46 |
+
{
|
| 47 |
+
"role": "user",
|
| 48 |
+
"content": "A train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly."
|
| 49 |
+
}
|
| 50 |
+
],
|
| 51 |
+
"prompt": "<|im_start|>user\nA train leaves at 14:10 and arrives at 17:45. How long is the journey? Show the steps briefly.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 52 |
+
"content": "Here's a thinking process:\n\n1. **Understand the User's Request:**\n - Start time: 14:10 (2:10 PM)\n - End time: 17:45 (5:45 PM)\n - Question: How long is",
|
| 53 |
+
"finish": "stop",
|
| 54 |
+
"tokens": 64
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"name": "sql",
|
| 58 |
+
"messages": [
|
| 59 |
+
{
|
| 60 |
+
"role": "user",
|
| 61 |
+
"content": "Write a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount)."
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"prompt": "<|im_start|>user\nWrite a SQL query that returns the five customers with the highest total order value from tables customers(id, name) and orders(id, customer_id, amount).<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 65 |
+
"content": "Here's a thinking process:\n\n1. **Understand the User Request:**\n - **Goal:** Find the top 5 customers by total order value.\n - **Tables:** \n - `customers(id, name)`\n - `orders(id, customer_id, amount)`\n -",
|
| 66 |
+
"finish": "stop",
|
| 67 |
+
"tokens": 64
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"name": "list-rivers",
|
| 71 |
+
"messages": [
|
| 72 |
+
{
|
| 73 |
+
"role": "user",
|
| 74 |
+
"content": "List five major rivers of Africa, one per line, with the country where each reaches the sea."
|
| 75 |
+
}
|
| 76 |
+
],
|
| 77 |
+
"prompt": "<|im_start|>user\nList five major rivers of Africa, one per line, with the country where each reaches the sea.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 78 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Request:**\n - **Task:** List five major rivers of Africa\n",
|
| 79 |
+
"finish": "stop",
|
| 80 |
+
"tokens": 29
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"name": "unicode",
|
| 84 |
+
"messages": [
|
| 85 |
+
{
|
| 86 |
+
"role": "user",
|
| 87 |
+
"content": "What do these emoji usually mean in a text message: π π₯ π ?"
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
"prompt": "<|im_start|>user\nWhat do these emoji usually mean in a text message: π π₯ π ?<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 91 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - User asks about the meaning of three specific emojis in text messages: π π₯ π\n - Context: Text messaging/social media communication\n - Need to explain each emoji's common meanings, especially in modern",
|
| 92 |
+
"finish": "stop",
|
| 93 |
+
"tokens": 64
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "haiku-autumn",
|
| 97 |
+
"messages": [
|
| 98 |
+
{
|
| 99 |
+
"role": "user",
|
| 100 |
+
"content": "Write a haiku about autumn leaves in a city."
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"prompt": "<|im_start|>user\nWrite a haiku about autumn leaves in a city.<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 104 |
+
"content": "Thinking Process:\n\n1. **Deconstruct the request:**\n * Topic: Autumn leaves",
|
| 105 |
+
"finish": "stop",
|
| 106 |
+
"tokens": 21
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "system-pirate",
|
| 110 |
+
"messages": [
|
| 111 |
+
{
|
| 112 |
+
"role": "system",
|
| 113 |
+
"content": "Answer like a cheerful pirate, in two sentences."
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"role": "user",
|
| 117 |
+
"content": "What is the boiling point of water at sea level?"
|
| 118 |
+
}
|
| 119 |
+
],
|
| 120 |
+
"prompt": "<|im_start|>system\nAnswer like a cheerful pirate, in two sentences.<|im_end|>\n<|im_start|>user\nWhat is the boiling point of water at sea level?<|im_end|>\n<|im_start|>assistant\n<think>\n",
|
| 121 |
+
"content": "Here's a thinking process:\n\n1. **Analyze User Input:**\n - **Question:** What is the boiling point of water at sea level?\n - **Constraint:** Answer like",
|
| 122 |
+
"finish": "stop",
|
| 123 |
+
"tokens": 41
|
| 124 |
+
}
|
| 125 |
+
]
|
| 126 |
+
}
|