// REAL inference evidence in Node: transformers.js (same version as the Space) + onnxruntime-node (CPU). // Same pinned model files (q4f16) the browser downloads. Same app code (src/sam.js + src/lib.js). // Usage: node scripts/evidence-node.mjs [outDir] import { Sam3TrackerModel, AutoProcessor, RawImage, env } from '@huggingface/transformers'; import sharp from 'sharp'; import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; import { execFileSync } from 'node:child_process'; import { MODEL_ID, MODEL_REVISION, EXAMPLES, VIDEO_EXAMPLE, TRANSFORMERS_VERSION } from '../src/config.js'; import { encodeImage, decodePrompt, trackFrames } from '../src/sam.js'; import { describeMask, boxIoU, fractionInside, overlayMask, cutout, maskIoU, sampleTimes } from '../src/lib.js'; import { hardware } from './hw.mjs'; const root = new URL('../', import.meta.url).pathname; const out = process.argv[2] || '/workspace/evidence/rotoscopia-lab'; mkdirSync(out + '/images', { recursive: true }); mkdirSync(out + '/video', { recursive: true }); const ortNode = JSON.parse(readFileSync(root + 'node_modules/onnxruntime-node/package.json', 'utf8')).version; const dtype = 'q4f16'; const png = (rgba, w, h, f) => sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width: w, height: h, channels: 4 } }).png().toFile(f); const maskPng = (m, w, h, f) => { const g = new Uint8Array(m.length); for (let i = 0; i < m.length; i++) g[i] = m[i] ? 255 : 0; return sharp(Buffer.from(g), { raw: { width: w, height: h, channels: 1 } }).png().toFile(f); }; const drawPrompt = (rgba, w, h, points, box) => { // mark prompts so the PNG documents the input const o = new Uint8ClampedArray(rgba); const put = (x, y, c) => { if (x < 0 || y < 0 || x >= w || y >= h) return; const j = (y * w + x) * 4; o[j] = c[0]; o[j + 1] = c[1]; o[j + 2] = c[2]; }; for (const p of points || []) { const c = p.label ? [0, 220, 0] : [230, 0, 0]; for (let dy = -7; dy <= 7; dy++) for (let dx = -7; dx <= 7; dx++) if (dx * dx + dy * dy <= 49) put(Math.round(p.x) + dx, Math.round(p.y) + dy, c); } if (box) { const [x0, y0, x1, y1] = box.map(Math.round); for (let t = 0; t < 3; t++) { for (let x = x0; x <= x1; x++) { put(x, y0 + t, [255, 200, 0]); put(x, y1 - t, [255, 200, 0]); } for (let y = y0; y <= y1; y++) { put(x0 + t, y, [255, 200, 0]); put(x1 - t, y, [255, 200, 0]); } } } return o; }; const report = { createdAt: new Date().toISOString(), kind: 'node-live-inference', runtime: { transformersjs: TRANSFORMERS_VERSION, onnxruntimeNode: ortNode, device: 'cpu', dtype }, model: { id: MODEL_ID, revision: MODEL_REVISION }, hardware: hardware(), images: [], video: null }; let t = performance.now(); const model = await Sam3TrackerModel.from_pretrained(MODEL_ID, { revision: MODEL_REVISION, dtype, device: 'cpu' }); const processor = await AutoProcessor.from_pretrained(MODEL_ID, { revision: MODEL_REVISION }); report.loadMs = Math.round(performance.now() - t); console.log('model loaded', report.loadMs, 'ms'); for (const ex of EXAMPLES) { const image = await RawImage.read(root + 'public/examples/' + ex.file); t = performance.now(); const enc = await encodeImage(model, processor, image); const encodeMs = performance.now() - t; t = performance.now(); const r = await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decodeMs = performance.now() - t; // second decode = interactive re-click cost with cached embeddings t = performance.now(); await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decode2Ms = performance.now() - t; const d = describeMask(r.mask, r.width, r.height); const rgba = image.rgba().data; const row = { id: ex.id, file: ex.file, size: [image.width, image.height], prompt: { points: ex.points, box: ex.box }, expectBox: ex.expectBox, maskArea: d.area, maskAreaPct: d.areaPct, maskBBox: d.bbox, bboxIoUvsExpected: +boxIoU(d.bbox, ex.expectBox).toFixed(4), fractionInsideExpected: +fractionInside(r.mask, r.width, r.height, ex.expectBox).toFixed(4), positivePointsOnMask: (ex.points || []).filter((p) => p.label).map((p) => !!r.mask[Math.round(p.y) * r.width + Math.round(p.x)]), negativePointsOffMask: (ex.points || []).filter((p) => !p.label).map((p) => !r.mask[Math.round(p.y) * r.width + Math.round(p.x)]), iouScores: r.scores.map((s) => +s.toFixed(4)), chosenMask: r.chosen, objectScoreLogit: +r.objectScoreLogit.toFixed(3), encodeMs: Math.round(encodeMs), decodeMs: Math.round(decodeMs), redecodeMs: Math.round(decode2Ms) }; report.images.push(row); console.log(JSON.stringify(row)); await png(overlayMask(drawPrompt(rgba, r.width, r.height, ex.points, ex.box), r.mask, r.width, r.height), r.width, r.height, `${out}/images/${ex.id}-overlay.png`); await maskPng(r.mask, r.width, r.height, `${out}/images/${ex.id}-mask.png`); await png(cutout(rgba, r.mask), r.width, r.height, `${out}/images/${ex.id}-cutout.png`); } // ---- Video: decode the bundled mp4 at the same timestamps the browser samples, track, compare to geometry GT. const V = VIDEO_EXAMPLE, vmeta = JSON.parse(readFileSync(root + 'public/examples/video-pan.json', 'utf8')); const mp4 = root + 'public/examples/' + V.file; const duration = +execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', mp4]).toString().trim(); const times = sampleTimes(duration, V.fps); const [FW, FH] = vmeta.out; const getFrame = async (i) => { const buf = execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-ss', String(times[i]), '-i', mp4, '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'rgba', '-'], { maxBuffer: 64 << 20 }); return new RawImage(new Uint8ClampedArray(buf), FW, FH, 4).rgb(); }; // Ground truth: segment the mug ONCE on the full still photo with the equivalent box, then warp it into each frame. const still = await RawImage.read(root + 'public/examples/' + vmeta.source); const stillEnc = await encodeImage(model, processor, still); const ref = await decodePrompt(model, processor, still, stillEnc, [], V.refBoxOriginal); const frameIndexAt = (tt) => Math.min(vmeta.frames.length - 1, Math.round(tt * vmeta.fps)); const gtFor = (tt) => { const f = vmeta.frames[frameIndexAt(tt)], m = new Uint8Array(FW * FH); for (let y = 0; y < FH; y++) for (let x = 0; x < FW; x++) { const sx = Math.floor(f.left + ((x + 0.5) * f.width) / FW), sy = Math.floor(f.top + ((y + 0.5) * f.height) / FH); m[y * FW + x] = ref.mask[sy * ref.width + sx]; } return m; }; const vrows = []; const results = await trackFrames(model, processor, times.length, getFrame, { points: V.points, box: V.box }, { onFrame: async (res, image) => { const gt = gtFor(times[res.index]); const row = { i: res.index, t: times[res.index], area: res.area, lost: res.lost, maskIoUvsGeometryGT: +maskIoU(res.mask, gt).toFixed(4), objectScoreLogit: +res.objectScoreLogit.toFixed(2), encodeMs: Math.round(res.encodeMs), decodeMs: Math.round(res.decodeMs) }; vrows.push(row); console.log('frame', JSON.stringify(row)); if (res.index % 5 === 0 || res.index === times.length - 1) await png(overlayMask(drawPrompt(image.rgba().data, FW, FH, res.prompt.points, res.prompt.box), res.mask, FW, FH), FW, FH, `${out}/video/frame-${String(res.index).padStart(2, '0')}.png`); }, }); const ious = vrows.map((r) => r.maskIoUvsGeometryGT); report.video = { file: V.file, duration, sampledFps: V.fps, frames: times.length, initPrompt: { points: V.points, box: V.box }, refBoxOriginal: V.refBoxOriginal, refMaskArea: describeMask(ref.mask, ref.width, ref.height), meanIoU: +(ious.reduce((a, b) => a + b, 0) / ious.length).toFixed(4), minIoU: Math.min(...ious), lostFrames: vrows.filter((r) => r.lost).length, meanFrameMs: Math.round(vrows.reduce((a, r) => a + r.encodeMs + r.decodeMs, 0) / vrows.length), perFrame: vrows }; writeFileSync(`${out}/node-live-report.json`, JSON.stringify(report, null, 2)); console.log('video', JSON.stringify({ ...report.video, perFrame: undefined })); console.log('written', out);