Spaces:
Running
Running
File size: 8,021 Bytes
87b5db3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | // REAL inference evidence in Node: transformers.js (same version as the Space) + onnxruntime-node (CPU).
// Same pinned model files (q4f16) the browser downloads. Same app code (src/sam.js + src/lib.js).
// Usage: node scripts/evidence-node.mjs [outDir]
import { Sam3TrackerModel, AutoProcessor, RawImage, env } from '@huggingface/transformers';
import sharp from 'sharp';
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { MODEL_ID, MODEL_REVISION, EXAMPLES, VIDEO_EXAMPLE, TRANSFORMERS_VERSION } from '../src/config.js';
import { encodeImage, decodePrompt, trackFrames } from '../src/sam.js';
import { describeMask, boxIoU, fractionInside, overlayMask, cutout, maskIoU, sampleTimes } from '../src/lib.js';
import { hardware } from './hw.mjs';
const root = new URL('../', import.meta.url).pathname;
const out = process.argv[2] || '/workspace/evidence/rotoscopia-lab';
mkdirSync(out + '/images', { recursive: true }); mkdirSync(out + '/video', { recursive: true });
const ortNode = JSON.parse(readFileSync(root + 'node_modules/onnxruntime-node/package.json', 'utf8')).version;
const dtype = 'q4f16';
const png = (rgba, w, h, f) => sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width: w, height: h, channels: 4 } }).png().toFile(f);
const maskPng = (m, w, h, f) => { const g = new Uint8Array(m.length); for (let i = 0; i < m.length; i++) g[i] = m[i] ? 255 : 0; return sharp(Buffer.from(g), { raw: { width: w, height: h, channels: 1 } }).png().toFile(f); };
const drawPrompt = (rgba, w, h, points, box) => { // mark prompts so the PNG documents the input
const o = new Uint8ClampedArray(rgba);
const put = (x, y, c) => { if (x < 0 || y < 0 || x >= w || y >= h) return; const j = (y * w + x) * 4; o[j] = c[0]; o[j + 1] = c[1]; o[j + 2] = c[2]; };
for (const p of points || []) { const c = p.label ? [0, 220, 0] : [230, 0, 0]; for (let dy = -7; dy <= 7; dy++) for (let dx = -7; dx <= 7; dx++) if (dx * dx + dy * dy <= 49) put(Math.round(p.x) + dx, Math.round(p.y) + dy, c); }
if (box) { const [x0, y0, x1, y1] = box.map(Math.round); for (let t = 0; t < 3; t++) { for (let x = x0; x <= x1; x++) { put(x, y0 + t, [255, 200, 0]); put(x, y1 - t, [255, 200, 0]); } for (let y = y0; y <= y1; y++) { put(x0 + t, y, [255, 200, 0]); put(x1 - t, y, [255, 200, 0]); } } }
return o;
};
const report = { createdAt: new Date().toISOString(), kind: 'node-live-inference', runtime: { transformersjs: TRANSFORMERS_VERSION, onnxruntimeNode: ortNode, device: 'cpu', dtype }, model: { id: MODEL_ID, revision: MODEL_REVISION }, hardware: hardware(), images: [], video: null };
let t = performance.now();
const model = await Sam3TrackerModel.from_pretrained(MODEL_ID, { revision: MODEL_REVISION, dtype, device: 'cpu' });
const processor = await AutoProcessor.from_pretrained(MODEL_ID, { revision: MODEL_REVISION });
report.loadMs = Math.round(performance.now() - t);
console.log('model loaded', report.loadMs, 'ms');
for (const ex of EXAMPLES) {
const image = await RawImage.read(root + 'public/examples/' + ex.file);
t = performance.now(); const enc = await encodeImage(model, processor, image); const encodeMs = performance.now() - t;
t = performance.now(); const r = await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decodeMs = performance.now() - t;
// second decode = interactive re-click cost with cached embeddings
t = performance.now(); await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decode2Ms = performance.now() - t;
const d = describeMask(r.mask, r.width, r.height);
const rgba = image.rgba().data;
const row = { id: ex.id, file: ex.file, size: [image.width, image.height], prompt: { points: ex.points, box: ex.box }, expectBox: ex.expectBox,
maskArea: d.area, maskAreaPct: d.areaPct, maskBBox: d.bbox, bboxIoUvsExpected: +boxIoU(d.bbox, ex.expectBox).toFixed(4),
fractionInsideExpected: +fractionInside(r.mask, r.width, r.height, ex.expectBox).toFixed(4),
positivePointsOnMask: (ex.points || []).filter((p) => p.label).map((p) => !!r.mask[Math.round(p.y) * r.width + Math.round(p.x)]),
negativePointsOffMask: (ex.points || []).filter((p) => !p.label).map((p) => !r.mask[Math.round(p.y) * r.width + Math.round(p.x)]),
iouScores: r.scores.map((s) => +s.toFixed(4)), chosenMask: r.chosen, objectScoreLogit: +r.objectScoreLogit.toFixed(3),
encodeMs: Math.round(encodeMs), decodeMs: Math.round(decodeMs), redecodeMs: Math.round(decode2Ms) };
report.images.push(row);
console.log(JSON.stringify(row));
await png(overlayMask(drawPrompt(rgba, r.width, r.height, ex.points, ex.box), r.mask, r.width, r.height), r.width, r.height, `${out}/images/${ex.id}-overlay.png`);
await maskPng(r.mask, r.width, r.height, `${out}/images/${ex.id}-mask.png`);
await png(cutout(rgba, r.mask), r.width, r.height, `${out}/images/${ex.id}-cutout.png`);
}
// ---- Video: decode the bundled mp4 at the same timestamps the browser samples, track, compare to geometry GT.
const V = VIDEO_EXAMPLE, vmeta = JSON.parse(readFileSync(root + 'public/examples/video-pan.json', 'utf8'));
const mp4 = root + 'public/examples/' + V.file;
const duration = +execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', mp4]).toString().trim();
const times = sampleTimes(duration, V.fps);
const [FW, FH] = vmeta.out;
const getFrame = async (i) => {
const buf = execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-ss', String(times[i]), '-i', mp4, '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'rgba', '-'], { maxBuffer: 64 << 20 });
return new RawImage(new Uint8ClampedArray(buf), FW, FH, 4).rgb();
};
// Ground truth: segment the mug ONCE on the full still photo with the equivalent box, then warp it into each frame.
const still = await RawImage.read(root + 'public/examples/' + vmeta.source);
const stillEnc = await encodeImage(model, processor, still);
const ref = await decodePrompt(model, processor, still, stillEnc, [], V.refBoxOriginal);
const frameIndexAt = (tt) => Math.min(vmeta.frames.length - 1, Math.round(tt * vmeta.fps));
const gtFor = (tt) => {
const f = vmeta.frames[frameIndexAt(tt)], m = new Uint8Array(FW * FH);
for (let y = 0; y < FH; y++) for (let x = 0; x < FW; x++) {
const sx = Math.floor(f.left + ((x + 0.5) * f.width) / FW), sy = Math.floor(f.top + ((y + 0.5) * f.height) / FH);
m[y * FW + x] = ref.mask[sy * ref.width + sx];
}
return m;
};
const vrows = [];
const results = await trackFrames(model, processor, times.length, getFrame, { points: V.points, box: V.box }, {
onFrame: async (res, image) => {
const gt = gtFor(times[res.index]);
const row = { i: res.index, t: times[res.index], area: res.area, lost: res.lost, maskIoUvsGeometryGT: +maskIoU(res.mask, gt).toFixed(4), objectScoreLogit: +res.objectScoreLogit.toFixed(2), encodeMs: Math.round(res.encodeMs), decodeMs: Math.round(res.decodeMs) };
vrows.push(row); console.log('frame', JSON.stringify(row));
if (res.index % 5 === 0 || res.index === times.length - 1) await png(overlayMask(drawPrompt(image.rgba().data, FW, FH, res.prompt.points, res.prompt.box), res.mask, FW, FH), FW, FH, `${out}/video/frame-${String(res.index).padStart(2, '0')}.png`);
},
});
const ious = vrows.map((r) => r.maskIoUvsGeometryGT);
report.video = { file: V.file, duration, sampledFps: V.fps, frames: times.length, initPrompt: { points: V.points, box: V.box }, refBoxOriginal: V.refBoxOriginal, refMaskArea: describeMask(ref.mask, ref.width, ref.height),
meanIoU: +(ious.reduce((a, b) => a + b, 0) / ious.length).toFixed(4), minIoU: Math.min(...ious), lostFrames: vrows.filter((r) => r.lost).length,
meanFrameMs: Math.round(vrows.reduce((a, r) => a + r.encodeMs + r.decodeMs, 0) / vrows.length), perFrame: vrows };
writeFileSync(`${out}/node-live-report.json`, JSON.stringify(report, null, 2));
console.log('video', JSON.stringify({ ...report.video, perFrame: undefined }));
console.log('written', out);
|