MiniCPM-V-4.7 WebBrain v2 release
7dbe69f verified 8 days ago
25.5 kB
// MiniCPM-V-4.7 (WebBrain fine-tune) in-browser runtime: ONNX Runtime Web (WebGPU) + a Transformers.js tokenizer.
//
// The ONNX bundle has three graphs:
//   onnx/embed_tokens_q4f16.onnx          input_ids int64[1,S]            -> inputs_embeds float32[1,S,1024]
//   onnx/vision_encoder_fp16.onnx         pixel_values float32[n,3,H,W]    -> image_features float32[n,T,1024]
//   onnx/decoder_model_merged_q4f16.onnx  Qwen3.5-0.8B topology (GatedDeltaNet + gated attention, M-RoPE),
//                                         chunked GDN prefill, last-position logits + decision_hidden
// This module reproduces MiniCPM-V-4.7's processor (slicing, PIL-compatible bicubic resize, normalization, image
// placeholders) and its canvas M-RoPE position ids exactly, so the browser builds the same inputs as Transformers.
//
// Everything here is plain JS; `ort` (onnxruntime-web with WebGPU) and a tokenizer with `encode(text)` are injected.

export const MINICPMV = {
  imageTokenId: 248056,
  imageStartId: 248078,   // <image>
  imageEndId: 248079,     // </image>
  sliceStartId: 248088,   // <slice>
  sliceEndId: 248089,     // </slice>
  newlineId: 198,         // "\n"
  imagePad: '<|image_pad|>',
  hidden: 1024,
  layers: 24,
  fullAttentionInterval: 4,
  gdnConv: [6144, 4],
  gdnState: [16, 128, 128],
  kvHeads: 2,
  headDim: 256,
  eosTokenIds: [248044, 248046],
  scaleResolution: 448,
  patchSize: 14,
  maxSliceNums: 9,
  mergeFactor: 4,           // "16x": 4x4 patches per LLM token
};

// ------------------------------------------------------------------------------------------ python numerics
/** Python 3 round(): ties to even. */
export function pyRound(x) {
  const r = Math.round(x);
  return (Math.abs(x % 1) === 0.5) ? 2 * Math.round(x / 2) : r;
}

const roundHalfEven = pyRound;

/** torch.linspace(start, end, steps) in float32 (same symmetric formula on CPU and CUDA). */
function linspaceF32(start, end, steps) {
  if (steps === 1) return [Math.fround(start)];
  const f = Math.fround;
  const s = f(start), e = f(end);
  const step = f(f(e - s) / (steps - 1));
  const half = Math.floor(steps / 2);
  const out = new Array(steps);
  for (let i = 0; i < steps; i++) {
    out[i] = i < half ? f(s + f(step * i)) : f(e - f(step * (steps - i - 1)));
  }
  return out;
}

// ------------------------------------------------------------------------------------------ slicing (processor)
function ensureDivide(length, divisor) {
  return Math.max(pyRound(length / divisor) * divisor, divisor);
}

function findBestResize(height, width, scaleResolution, patchSize, allowUpscale = false) {
  if (height * width > scaleResolution * scaleResolution || allowUpscale) {
    const aspect = width / height;
    height = Math.trunc(scaleResolution / Math.sqrt(aspect));
    width = Math.trunc(height * aspect);
  }
  return [ensureDivide(height, patchSize * 4), ensureDivide(width, patchSize * 4)];
}

function getRefineSize(height, width, grid, scaleResolution, patchSize, allowUpscale = false) {
  const [gridY, gridX] = grid;
  const refineWidth = ensureDivide(width, gridX);
  const refineHeight = ensureDivide(height, gridY);
  const [bh, bw] = findBestResize(refineHeight / gridY, refineWidth / gridX, scaleResolution, patchSize, allowUpscale);
  return [bh * gridY, bw * gridX];
}

export function getSlicedGrid(height, width, maxSliceNums, scaleResolution) {
  const logRatio = Math.log(width / height);
  const ratio = (width * height) / (scaleResolution * scaleResolution);
  const multiple = Math.min(Math.ceil(ratio), maxSliceNums);
  if (multiple <= 1) return null;
  let best = [1, 1], minError = Infinity;
  for (const numSlices of [multiple - 1, multiple, multiple + 1]) {
    if (numSlices === 1 || numSlices > maxSliceNums) continue;
    for (let rows = 1; rows <= numSlices; rows++) {
      if (numSlices % rows !== 0) continue;
      const cols = numSlices / rows;
      const error = Math.abs(logRatio - Math.log(cols / rows));
      if (error < minError) { best = [rows, cols]; minError = error; }
      else if (error === minError && rows > best[0]) best = [rows, cols];
    }
  }
  return best;
}

// ------------------------------------------------------------------------------------------ PIL bicubic resample
// Port of Pillow's ImagingResample (BICUBIC, a = -0.5, support 2, antialiased when downscaling, 22-bit fixed-point
// coefficients, horizontal pass then vertical pass with uint8 rounding in between). Torch's uint8 antialiased bicubic
// (what the Transformers torchvision backend calls) reproduces Pillow's integer pipeline.
const PRECISION_BITS = 32 - 8 - 2;

function bicubic(x) {
  const a = -0.5;
  x = Math.abs(x);
  if (x < 1.0) return ((a + 2.0) * x - (a + 3.0)) * x * x + 1;
  if (x < 2.0) return (((x - 5) * x + 8) * x - 4) * a;
  return 0.0;
}

function precomputeCoeffs(inSize, outSize) {
  const scale = inSize / outSize;
  const filterscale = Math.max(scale, 1.0);
  const support = 2.0 * filterscale;
  const ksize = Math.ceil(support) * 2 + 1;
  const bounds = new Int32Array(outSize * 2);
  const kk = new Float64Array(outSize * ksize);
  const ss = 1.0 / filterscale;
  for (let xx = 0; xx < outSize; xx++) {
    const center = (xx + 0.5) * scale;
    let ww = 0.0;
    let xmin = Math.trunc(center - support + 0.5);
    if (xmin < 0) xmin = 0;
    let xmax = Math.trunc(center + support + 0.5);
    if (xmax > inSize) xmax = inSize;
    xmax -= xmin;
    const base = xx * ksize;
    let x = 0;
    for (; x < xmax; x++) {
      const w = bicubic((x + xmin - center + 0.5) * ss);
      kk[base + x] = w;
      ww += w;
    }
    for (x = 0; x < xmax; x++) if (ww !== 0.0) kk[base + x] /= ww;
    for (; x < ksize; x++) kk[base + x] = 0;
    bounds[xx * 2] = xmin;
    bounds[xx * 2 + 1] = xmax;
  }
  // fixed point (Pillow normalize_coeffs_8bpc)
  const kint = new Int32Array(outSize * ksize);
  for (let i = 0; i < kk.length; i++) {
    const v = kk[i] * (1 << PRECISION_BITS);
    kint[i] = v < 0 ? Math.trunc(-0.5 + v) : Math.trunc(0.5 + v);
  }
  return { ksize, bounds, kint };
}

function clip8(v) {
  // Pillow: clip8(in) = in >> PRECISION_BITS clamped to [0, 255] after adding the rounding half
  const r = Math.floor(v / (1 << PRECISION_BITS));
  return r < 0 ? 0 : (r > 255 ? 255 : r);
}

/** Resize an RGB uint8 image (Uint8Array, 3 channels interleaved) with Pillow's BICUBIC. */
export function resizeBicubicRGB(src, width, height, outW, outH) {
  const half = 1 << (PRECISION_BITS - 1);
  let cur = src, cw = width, ch = height;
  if (outW !== cw) {
    const { ksize, bounds, kint } = precomputeCoeffs(cw, outW);
    const dst = new Uint8Array(outW * ch * 3);
    for (let y = 0; y < ch; y++) {
      for (let xx = 0; xx < outW; xx++) {
        const xmin = bounds[xx * 2], xmax = bounds[xx * 2 + 1], k = xx * ksize;
        let s0 = half, s1 = half, s2 = half;
        for (let x = 0; x < xmax; x++) {
          const p = (y * cw + x + xmin) * 3, w = kint[k + x];
          s0 += cur[p] * w; s1 += cur[p + 1] * w; s2 += cur[p + 2] * w;
        }
        const o = (y * outW + xx) * 3;
        dst[o] = clip8(s0); dst[o + 1] = clip8(s1); dst[o + 2] = clip8(s2);
      }
    }
    cur = dst; cw = outW;
  }
  if (outH !== ch) {
    const { ksize, bounds, kint } = precomputeCoeffs(ch, outH);
    const dst = new Uint8Array(cw * outH * 3);
    for (let yy = 0; yy < outH; yy++) {
      const ymin = bounds[yy * 2], ymax = bounds[yy * 2 + 1], k = yy * ksize;
      for (let x = 0; x < cw; x++) {
        let s0 = half, s1 = half, s2 = half;
        for (let y = 0; y < ymax; y++) {
          const p = ((y + ymin) * cw + x) * 3, w = kint[k + y];
          s0 += cur[p] * w; s1 += cur[p + 1] * w; s2 += cur[p + 2] * w;
        }
        const o = (yy * cw + x) * 3;
        dst[o] = clip8(s0); dst[o + 1] = clip8(s1); dst[o + 2] = clip8(s2);
      }
    }
    cur = dst; ch = outH;
  }
  return cur;
}

function cropRGB(src, width, x0, y0, w, h) {
  const out = new Uint8Array(w * h * 3);
  for (let y = 0; y < h; y++) out.set(src.subarray(((y0 + y) * width + x0) * 3, ((y0 + y) * width + x0 + w) * 3), y * w * 3);
  return out;
}

// The processor fuses rescale (1/255) and normalize (mean = std = 0.5) into (x - 127.5) / 127.5 in float32.
export const NORM_LUT = (() => {
  const t = new Float32Array(256);
  for (let x = 0; x < 256; x++) t[x] = Math.fround((x - 127.5) / 127.5);
  return t;
})();

/** RGB uint8 HWC -> normalized float32 CHW. */
function normalizeCHW(rgb, w, h) {
  const out = new Float32Array(3 * h * w);
  const plane = h * w;
  for (let i = 0; i < plane; i++) {
    for (let c = 0; c < 3; c++) out[c * plane + i] = NORM_LUT[rgb[i * 3 + c]];
  }
  return out;
}

/**
 * MiniCPMV4_6ImageProcessor for one image. `rgb` is uint8 RGB (HWC). Returns the crops in processor order
 * (thumbnail, then slices row-major) with their patch grids, and the slice grid ([rows, cols] or null).
 */
export function preprocessImage(rgb, width, height, { maxSliceNums = MINICPMV.maxSliceNums,
  scaleResolution = MINICPMV.scaleResolution, patchSize = MINICPMV.patchSize, sliceMode = true } = {}) {
  const grid = sliceMode ? getSlicedGrid(height, width, maxSliceNums, scaleResolution) : null;
  const [sh, sw] = findBestResize(height, width, scaleResolution, patchSize, grid === null);
  const crops = [];
  const source = resizeBicubicRGB(rgb, width, height, sw, sh);
  crops.push({ h: sh / patchSize, w: sw / patchSize, width: sw, height: sh, pixels: normalizeCHW(source, sw, sh) });
  if (grid) {
    const [rh, rw] = getRefineSize(height, width, grid, scaleResolution, patchSize, true);
    const refine = resizeBicubicRGB(rgb, width, height, rw, rh);
    const ph = Math.floor(rh / grid[0]), pw = Math.floor(rw / grid[1]);
    for (let i = 0; i < rh; i += ph) {
      for (let j = 0; j < rw; j += pw) {
        const piece = cropRGB(refine, rw, j, i, pw, ph);
        crops.push({ h: ph / patchSize, w: pw / patchSize, width: pw, height: ph, pixels: normalizeCHW(piece, pw, ph) });
      }
    }
  }
  return { crops, grid };
}

/** The processor's text replacement for the image placeholder (use_image_id, 16x downsampling). */
export function imagePlaceholder(image, localIndex = 0) {
  const div = MINICPMV.mergeFactor * MINICPMV.mergeFactor;
  const [thumb, ...slices] = image.crops;
  let text = `<image_id>${localIndex}</image_id><image>${MINICPMV.imagePad.repeat((thumb.h * thumb.w) / div)}</image>`;
  if (image.grid && slices.length) {
    const [rows, cols] = image.grid;
    const per = (slices[0].h * slices[0].w) / div;
    const slice = `<slice>${MINICPMV.imagePad.repeat(per)}</slice>`;
    text += Array.from({ length: rows }, () => slice.repeat(cols)).join('\n');
  }
  return text;
}

/** Replace each `<|image_pad|>` produced by the chat template with its image's full placeholder. */
export function expandImagePlaceholders(prompt, images) {
  const parts = prompt.split(MINICPMV.imagePad);
  if (parts.length - 1 !== images.length) throw new Error(`prompt has ${parts.length - 1} image slots for ${images.length} images`);
  let out = parts[0];
  for (let i = 0; i < images.length; i++) out += imagePlaceholder(images[i], i) + parts[i + 1];
  return out;
}

// ------------------------------------------------------------------------------------------ canvas M-RoPE
/**
 * Port of MiniCPMV4_7Model.get_rope_index for one unpadded sequence. `ids` are token ids, `targetSizes` the
 * patch grids [h, w] of every crop in order. Returns {t, h, w} Int32Arrays and the next free position.
 */
export function canvasPositions(ids, targetSizes, cfg = MINICPMV) {
  const S = ids.length;
  const T = new Int32Array(S), H = new Int32Array(S), W = new Int32Array(S);
  for (let i = 0; i < S; i++) { T[i] = i; H[i] = i; W[i] = i; }
  const markers = new Set([cfg.sliceStartId, cfg.sliceEndId, cfg.imageStartId, cfg.imageEndId, cfg.newlineId]);
  const isVisual = id => id === cfg.imageTokenId;
  // runs of visual tokens = crops
  const groups = [];
  let openSlices = null, previousEnd = 0;
  for (let i = 0; i < S;) {
    if (!isVisual(ids[i])) { i++; continue; }
    let j = i;
    while (j < S && isVisual(ids[j])) j++;
    const crop = [i, j];
    if (openSlices !== null && ids[crop[0] - 1] === cfg.sliceStartId) {
      openSlices.push(crop);
    } else {
      openSlices = [];
      const frame = { thumbStart: crop[0], thumbEnd: crop[1], slices: openSlices };
      let adjacent = groups.length > 0;
      for (let k = previousEnd; adjacent && k < crop[0] - 1; k++) if (!markers.has(ids[k])) adjacent = false;
      if (adjacent) groups[groups.length - 1].frames.push(frame);
      else groups.push({ start: crop[0] - 1, frames: [frame] });
    }
    previousEnd = crop[1];
    i = j;
  }
  const frameEnd = last => { let e = last; while (e < S && markers.has(ids[e])) e++; return e; };
  for (const g of groups) {
    for (const f of g.frames) f.end = frameEnd(f.slices.length ? f.slices[f.slices.length - 1][1] : f.thumbEnd);
    g.end = g.frames[g.frames.length - 1].end;
  }
  const set3 = (idx, t, h, w) => { T[idx] = t; H[idx] = h; W[idx] = w; };
  const mf = cfg.mergeFactor;
  let sizeIter = 0;
  const nextSize = () => targetSizes[sizeIter++];
  let currentPos = 0, currentIdx = 0;
  for (const g of groups) {
    if (g.start > currentIdx) {
      for (let k = currentIdx; k < g.start; k++) { const p = currentPos + (k - currentIdx); set3(k, p, p, p); }
      currentPos += g.start - currentIdx;
    }
    let cursor = g.start;
    for (const f of g.frames) {
      const thumbSize = nextSize();
      const frameStart = f.thumbStart - 1;
      if (frameStart > cursor) {
        for (let k = cursor; k < frameStart; k++) { const p = currentPos + (k - cursor); set3(k, p, p, p); }
        currentPos += frameStart - cursor + 1;
      }
      const fEnd = Math.min(f.end, g.end);
      const origin = currentPos;
      const halo = Math.max(origin - 1, 0);
      let sliceH = 0, sliceW = 0, numCols = 0;
      let canvasH = Math.floor(thumbSize[0] / mf), canvasW = Math.floor(thumbSize[1] / mf);
      let firstSlice = null;
      if (f.slices.length) {
        numCols = f.slices.length;
        for (let k = 0; k < f.slices.length - 1; k++) {
          if (f.slices[k + 1][0] - f.slices[k][1] > 2) { numCols = k + 1; break; }
        }
        let numRows = numCols > 0 ? Math.floor(f.slices.length / numCols) : 1;
        if (numRows * numCols !== f.slices.length) { numRows = 1; numCols = f.slices.length; }
        firstSlice = nextSize();
        sliceH = Math.floor(firstSlice[0] / mf); sliceW = Math.floor(firstSlice[1] / mf);
        canvasH = numRows * sliceH; canvasW = numCols * sliceW;
      }
      for (let k = frameStart; k < fEnd; k++) set3(k, origin, origin, origin);
      H[frameStart] = halo; W[frameStart] = halo;
      if (f.thumbEnd < fEnd) { H[f.thumbEnd] = origin + canvasH; W[f.thumbEnd] = origin + canvasW; }
      // thumbnail stretched over the canvas
      {
        const gh = Math.floor(thumbSize[0] / mf), gw = Math.floor(thumbSize[1] / mf);
        const hc = linspaceF32(0, canvasH - 1, gh).map(roundHalfEven), wc = linspaceF32(0, canvasW - 1, gw).map(roundHalfEven);
        let k = f.thumbStart;
        for (let a = 0; a < gh; a++) for (let b = 0; b < gw; b++, k++) set3(k, origin, origin + hc[a], origin + wc[b]);
      }
      f.slices.forEach(([s0, s1], k) => {
        const hOff = Math.floor(k / numCols) * sliceH, wOff = (k % numCols) * sliceW;
        const size = k === 0 ? firstSlice : nextSize();
        const sh = Math.floor(size[0] / mf), sw = Math.floor(size[1] / mf);
        if (s0 - 1 >= frameStart) { H[s0 - 1] = origin + hOff; W[s0 - 1] = origin + wOff; }
        if (s1 < fEnd) { H[s1] = origin + hOff + sh - 1; W[s1] = origin + wOff + sw - 1; }
        let idx = s0;
        for (let a = 0; a < sh; a++) for (let b = 0; b < sw; b++, idx++) set3(idx, origin, origin + hOff + a, origin + wOff + b);
      });
      for (let k = 0; k < f.slices.length - 1; k++) {
        const gs = f.slices[k][1], ge = f.slices[k + 1][0];
        if (ge - gs <= 2) continue;
        const boundaryH = (Math.floor(k / numCols) + 1) * sliceH - 1;
        const rightEdge = numCols * sliceW;
        for (let n = gs + 1; n < ge - 1; n++) { H[n] = origin + boundaryH; W[n] = origin + rightEdge; }
      }
      currentPos = origin + Math.max(canvasH, canvasW) + 1;
      cursor = fEnd;
    }
    if (cursor < g.end) {
      for (let k = cursor; k < g.end; k++) { const p = currentPos + (k - cursor); set3(k, p, p, p); }
      currentPos += g.end - cursor;
    }
    currentIdx = g.end;
  }
  for (let k = currentIdx; k < S; k++) { const p = currentPos + (k - currentIdx); set3(k, p, p, p); }
  let max = 0;
  for (let i = 0; i < S; i++) max = Math.max(max, T[i], H[i], W[i]);
  return { t: T, h: H, w: W, nextPosition: max + 1 };
}

// ------------------------------------------------------------------------------------------ ORT session runtime
const f16Zeros = n => new Uint16Array(n);

export class MiniCPMVRuntime {
  /**
   * @param ort       onnxruntime-web module (WebGPU build)
   * @param tokenizer object with encode(text) -> number[] (special tokens recognised, no BOS)
   * @param urls      { embed, vision, decoder } graph URLs (external data `<graph>_data` sits next to each)
   */
  constructor(ort, tokenizer, sessions, cfg = MINICPMV) {
    this.ort = ort;
    this.tokenizer = tokenizer;
    this.embed = sessions.embed;
    this.vision = sessions.vision;
    this.decoder = sessions.decoder;
    this.cfg = cfg;
    this.layerKinds = Array.from({ length: cfg.layers }, (_, i) => ((i + 1) % cfg.fullAttentionInterval === 0 ? 'attn' : 'gdn'));
  }

  static async create(ort, tokenizer, urls, { executionProviders = ['webgpu'], fetchData = null } = {}) {
    const make = async url => {
      const name = url.split('/').pop();
      const options = { executionProviders, graphOptimizationLevel: 'all' };
      const data = fetchData ? await fetchData(`${url}_data`) : null;
      options.externalData = [{ path: `${name}_data`, data: data || `${url}_data` }];
      if (executionProviders.includes('webgpu')) options.preferredOutputLocation = 'gpu-buffer';
      return ort.InferenceSession.create(url, options);
    };
    // ORT Web's WebGPU EP creates one session at a time
    const embed = await make(urls.embed);
    const vision = await make(urls.vision);
    const decoder = await make(urls.decoder);
    return new MiniCPMVRuntime(ort, tokenizer, { embed, vision, decoder });
  }

  tensor(type, data, dims) { return new this.ort.Tensor(type, data, dims); }

  async embedIds(ids) {
    const out = await this.embed.run({ input_ids: this.tensor('int64', BigInt64Array.from(ids.map(BigInt)), [1, ids.length]) });
    const t = out.inputs_embeds;
    const data = t.location === 'gpu-buffer' ? await t.getData(true) : t.data;
    return new Float32Array(data.buffer ? data : Float32Array.from(data));
  }

  /** Vision features for one preprocessed image, in placeholder order (thumbnail, slices). */
  async encodeImage(image) {
    const groups = new Map();
    image.crops.forEach((c, i) => {
      const key = `${c.height}x${c.width}`;
      if (!groups.has(key)) groups.set(key, []);
      groups.get(key).push(i);
    });
    const perCrop = new Array(image.crops.length);
    for (const idx of groups.values()) {
      const c0 = image.crops[idx[0]];
      const plane = 3 * c0.height * c0.width;
      const batch = new Float32Array(plane * idx.length);
      idx.forEach((ci, b) => batch.set(image.crops[ci].pixels, b * plane));
      const out = await this.vision.run({ pixel_values: this.tensor('float32', batch, [idx.length, 3, c0.height, c0.width]) });
      const t = out.image_features;
      const data = t.location === 'gpu-buffer' ? await t.getData(true) : t.data;
      const per = (c0.h / this.cfg.mergeFactor) * (c0.w / this.cfg.mergeFactor) * this.cfg.hidden;
      idx.forEach((ci, b) => { perCrop[ci] = data.subarray(b * per, (b + 1) * per); });
    }
    return perCrop;
  }

  /** Build prompt inputs: ids, embeddings (image rows filled), canvas positions. */
  async buildInputs(promptText, images = []) {
    const text = images.length ? expandImagePlaceholders(promptText, images) : promptText;
    const ids = this.tokenizer.encode(text);
    const embeds = await this.embedIds(ids);
    if (images.length) {
      const feats = [];
      for (const image of images) feats.push(...await this.encodeImage(image));
      let row = 0, fi = 0, off = 0;
      const H = this.cfg.hidden;
      for (let i = 0; i < ids.length; i++) {
        if (ids[i] !== this.cfg.imageTokenId) continue;
        while (fi < feats.length && off >= feats[fi].length) { fi++; off = 0; }
        if (fi >= feats.length) throw new Error('more image placeholders than vision features');
        embeds.set(feats[fi].subarray(off, off + H), i * H);
        off += H; row++;
      }
      const total = feats.reduce((s, f) => s + f.length / H, 0);
      if (row !== total) throw new Error(`image placeholders ${row} != vision features ${total}`);
    }
    const sizes = images.flatMap(im => im.crops.map(c => [c.h, c.w]));
    const pos = canvasPositions(ids, sizes, this.cfg);
    return { ids, embeds, pos, sizes };
  }

  emptyCache() {
    const feeds = {};
    const [convC, convK] = this.cfg.gdnConv, [hs, dk, dv] = this.cfg.gdnState;
    this.layerKinds.forEach((kind, i) => {
      if (kind === 'gdn') {
        feeds[`past_conv.${i}`] = this.tensor('float16', f16Zeros(convC * convK), [1, convC, convK]);
        feeds[`past_recurrent.${i}`] = this.tensor('float16', f16Zeros(hs * dk * dv), [1, hs, dk, dv]);
      } else {
        feeds[`past_key_values.${i}.key`] = this.tensor('float16', f16Zeros(0), [1, this.cfg.kvHeads, 0, this.cfg.headDim]);
        feeds[`past_key_values.${i}.value`] = this.tensor('float16', f16Zeros(0), [1, this.cfg.kvHeads, 0, this.cfg.headDim]);
      }
    });
    return { feeds, length: 0 };
  }

  /**
   * Run the decoder over `embeds` rows [start, end) with positions from `pos`, continuing `cache`.
   * Returns last-position logits (Float32Array), decision_hidden (Float32Array) and the new cache.
   */
  async forward(embeds, pos, start, end, cache, { keepCache = true } = {}) {
    const n = end - start, H = this.cfg.hidden;
    const posData = new BigInt64Array(3 * n);
    for (let i = 0; i < n; i++) {
      posData[i] = BigInt(pos.t[start + i]);
      posData[n + i] = BigInt(pos.h[start + i]);
      posData[2 * n + i] = BigInt(pos.w[start + i]);
    }
    const total = cache.length + n;
    const feeds = {
      ...cache.feeds,
      inputs_embeds: this.tensor('float32', embeds.slice(start * H, end * H), [1, n, H]),
      attention_mask: this.tensor('int64', new BigInt64Array(total).fill(1n), [1, total]),
      position_ids: this.tensor('int64', posData, [3, 1, n]),
    };
    const out = await this.decoder.run(feeds);
    const read = async t => {
      if (!t) return null;
      const d = t.location === 'gpu-buffer' ? await t.getData() : t.data;
      return d instanceof Uint16Array ? f16ToF32(d) : Float32Array.from(d);
    };
    const logits = await read(out.logits);
    const hidden = await read(out.decision_hidden);
    const next = { feeds: {}, length: total };
    for (const [name, t] of Object.entries(out)) {
      if (name.startsWith('present')) {
        const past = name.replace('present_conv', 'past_conv').replace('present_recurrent', 'past_recurrent').replace('present.', 'past_key_values.');
        if (keepCache) next.feeds[past] = t; else t.dispose?.();
      } else if (name !== 'logits' && name !== 'decision_hidden') t.dispose?.();
    }
    out.logits?.dispose?.();
    out.decision_hidden?.dispose?.();
    return { logits, hidden, cache: next };
  }

  /**
   * Greedy generation. `promptText` is the rendered chat prompt (each image as one `<|image_pad|>`); returns token ids.
   * New tokens continue the canvas positions: position = max(prompt positions) + 1 + step on all three axes.
   */
  async generate(promptText, images = [], { maxNewTokens = 128, stopIds = this.cfg.eosTokenIds, onToken = null } = {}) {
    const inputs = await this.buildInputs(promptText, images);
    let { logits, cache } = await this.forward(inputs.embeds, inputs.pos, 0, inputs.ids.length, this.emptyCache());
    let next = inputs.pos.nextPosition;
    const out = [];
    try {
      for (let step = 0; step < maxNewTokens; step++) {
        let best = 0;
        for (let i = 1; i < logits.length; i++) if (logits[i] > logits[best]) best = i;
        if (stopIds.includes(best)) break;
        out.push(best);
        onToken?.(best);
        const emb = await this.embedIds([best]);
        const p = { t: Int32Array.of(next), h: Int32Array.of(next), w: Int32Array.of(next) };
        next += 1;
        const r = await this.forward(emb, p, 0, 1, cache);
        MiniCPMVRuntime.disposeCache(cache);
        ({ logits, cache } = r);
      }
    } finally {
      MiniCPMVRuntime.disposeCache(cache);
    }
    return { ids: out, promptTokens: inputs.ids.length };
  }

  static disposeCache(cache) {
    for (const t of Object.values(cache?.feeds || {})) t.dispose?.();
  }
}

/** IEEE half -> float. */
export function f16ToF32(u16) {
  const out = new Float32Array(u16.length);
  for (let i = 0; i < u16.length; i++) {
    const h = u16[i], s = (h & 0x8000) ? -1 : 1, e = (h >> 10) & 0x1f, m = h & 0x3ff;
    out[i] = e === 0 ? s * m * 2 ** -24 : (e === 31 ? (m ? NaN : s * Infinity) : s * (1 + m / 1024) * 2 ** (e - 15));
  }
  return out;
}