Spaces:
Running
Running
Download source/scripts/evidence-node.mjs from Segizu/rotoscopia-lab: direct link, hf CLI and curl.
- Browser
- Download file 8.02 kB
-
https://huggingface.co/spaces/Segizu/rotoscopia-lab/resolve/main/source/scripts/evidence-node.mjs
- Command line
-
hf download hf://spaces/Segizu/rotoscopia-lab/source/scripts/evidence-node.mjs
-
curl -L -o evidence-node.mjs https://huggingface.co/spaces/Segizu/rotoscopia-lab/resolve/main/source/scripts/evidence-node.mjs
8.02 kB
| // REAL inference evidence in Node: transformers.js (same version as the Space) + onnxruntime-node (CPU). | |
| // Same pinned model files (q4f16) the browser downloads. Same app code (src/sam.js + src/lib.js). | |
| // Usage: node scripts/evidence-node.mjs [outDir] | |
| import { Sam3TrackerModel, AutoProcessor, RawImage, env } from '@huggingface/transformers'; | |
| import sharp from 'sharp'; | |
| import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; | |
| import { execFileSync } from 'node:child_process'; | |
| import { MODEL_ID, MODEL_REVISION, EXAMPLES, VIDEO_EXAMPLE, TRANSFORMERS_VERSION } from '../src/config.js'; | |
| import { encodeImage, decodePrompt, trackFrames } from '../src/sam.js'; | |
| import { describeMask, boxIoU, fractionInside, overlayMask, cutout, maskIoU, sampleTimes } from '../src/lib.js'; | |
| import { hardware } from './hw.mjs'; | |
| const root = new URL('../', import.meta.url).pathname; | |
| const out = process.argv[2] || '/workspace/evidence/rotoscopia-lab'; | |
| mkdirSync(out + '/images', { recursive: true }); mkdirSync(out + '/video', { recursive: true }); | |
| const ortNode = JSON.parse(readFileSync(root + 'node_modules/onnxruntime-node/package.json', 'utf8')).version; | |
| const dtype = 'q4f16'; | |
| const png = (rgba, w, h, f) => sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width: w, height: h, channels: 4 } }).png().toFile(f); | |
| const maskPng = (m, w, h, f) => { const g = new Uint8Array(m.length); for (let i = 0; i < m.length; i++) g[i] = m[i] ? 255 : 0; return sharp(Buffer.from(g), { raw: { width: w, height: h, channels: 1 } }).png().toFile(f); }; | |
| const drawPrompt = (rgba, w, h, points, box) => { // mark prompts so the PNG documents the input | |
| const o = new Uint8ClampedArray(rgba); | |
| const put = (x, y, c) => { if (x < 0 || y < 0 || x >= w || y >= h) return; const j = (y * w + x) * 4; o[j] = c[0]; o[j + 1] = c[1]; o[j + 2] = c[2]; }; | |
| for (const p of points || []) { const c = p.label ? [0, 220, 0] : [230, 0, 0]; for (let dy = -7; dy <= 7; dy++) for (let dx = -7; dx <= 7; dx++) if (dx * dx + dy * dy <= 49) put(Math.round(p.x) + dx, Math.round(p.y) + dy, c); } | |
| if (box) { const [x0, y0, x1, y1] = box.map(Math.round); for (let t = 0; t < 3; t++) { for (let x = x0; x <= x1; x++) { put(x, y0 + t, [255, 200, 0]); put(x, y1 - t, [255, 200, 0]); } for (let y = y0; y <= y1; y++) { put(x0 + t, y, [255, 200, 0]); put(x1 - t, y, [255, 200, 0]); } } } | |
| return o; | |
| }; | |
| const report = { createdAt: new Date().toISOString(), kind: 'node-live-inference', runtime: { transformersjs: TRANSFORMERS_VERSION, onnxruntimeNode: ortNode, device: 'cpu', dtype }, model: { id: MODEL_ID, revision: MODEL_REVISION }, hardware: hardware(), images: [], video: null }; | |
| let t = performance.now(); | |
| const model = await Sam3TrackerModel.from_pretrained(MODEL_ID, { revision: MODEL_REVISION, dtype, device: 'cpu' }); | |
| const processor = await AutoProcessor.from_pretrained(MODEL_ID, { revision: MODEL_REVISION }); | |
| report.loadMs = Math.round(performance.now() - t); | |
| console.log('model loaded', report.loadMs, 'ms'); | |
| for (const ex of EXAMPLES) { | |
| const image = await RawImage.read(root + 'public/examples/' + ex.file); | |
| t = performance.now(); const enc = await encodeImage(model, processor, image); const encodeMs = performance.now() - t; | |
| t = performance.now(); const r = await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decodeMs = performance.now() - t; | |
| // second decode = interactive re-click cost with cached embeddings | |
| t = performance.now(); await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decode2Ms = performance.now() - t; | |
| const d = describeMask(r.mask, r.width, r.height); | |
| const rgba = image.rgba().data; | |
| const row = { id: ex.id, file: ex.file, size: [image.width, image.height], prompt: { points: ex.points, box: ex.box }, expectBox: ex.expectBox, | |
| maskArea: d.area, maskAreaPct: d.areaPct, maskBBox: d.bbox, bboxIoUvsExpected: +boxIoU(d.bbox, ex.expectBox).toFixed(4), | |
| fractionInsideExpected: +fractionInside(r.mask, r.width, r.height, ex.expectBox).toFixed(4), | |
| positivePointsOnMask: (ex.points || []).filter((p) => p.label).map((p) => !!r.mask[Math.round(p.y) * r.width + Math.round(p.x)]), | |
| negativePointsOffMask: (ex.points || []).filter((p) => !p.label).map((p) => !r.mask[Math.round(p.y) * r.width + Math.round(p.x)]), | |
| iouScores: r.scores.map((s) => +s.toFixed(4)), chosenMask: r.chosen, objectScoreLogit: +r.objectScoreLogit.toFixed(3), | |
| encodeMs: Math.round(encodeMs), decodeMs: Math.round(decodeMs), redecodeMs: Math.round(decode2Ms) }; | |
| report.images.push(row); | |
| console.log(JSON.stringify(row)); | |
| await png(overlayMask(drawPrompt(rgba, r.width, r.height, ex.points, ex.box), r.mask, r.width, r.height), r.width, r.height, `${out}/images/${ex.id}-overlay.png`); | |
| await maskPng(r.mask, r.width, r.height, `${out}/images/${ex.id}-mask.png`); | |
| await png(cutout(rgba, r.mask), r.width, r.height, `${out}/images/${ex.id}-cutout.png`); | |
| } | |
| // ---- Video: decode the bundled mp4 at the same timestamps the browser samples, track, compare to geometry GT. | |
| const V = VIDEO_EXAMPLE, vmeta = JSON.parse(readFileSync(root + 'public/examples/video-pan.json', 'utf8')); | |
| const mp4 = root + 'public/examples/' + V.file; | |
| const duration = +execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', mp4]).toString().trim(); | |
| const times = sampleTimes(duration, V.fps); | |
| const [FW, FH] = vmeta.out; | |
| const getFrame = async (i) => { | |
| const buf = execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-ss', String(times[i]), '-i', mp4, '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'rgba', '-'], { maxBuffer: 64 << 20 }); | |
| return new RawImage(new Uint8ClampedArray(buf), FW, FH, 4).rgb(); | |
| }; | |
| // Ground truth: segment the mug ONCE on the full still photo with the equivalent box, then warp it into each frame. | |
| const still = await RawImage.read(root + 'public/examples/' + vmeta.source); | |
| const stillEnc = await encodeImage(model, processor, still); | |
| const ref = await decodePrompt(model, processor, still, stillEnc, [], V.refBoxOriginal); | |
| const frameIndexAt = (tt) => Math.min(vmeta.frames.length - 1, Math.round(tt * vmeta.fps)); | |
| const gtFor = (tt) => { | |
| const f = vmeta.frames[frameIndexAt(tt)], m = new Uint8Array(FW * FH); | |
| for (let y = 0; y < FH; y++) for (let x = 0; x < FW; x++) { | |
| const sx = Math.floor(f.left + ((x + 0.5) * f.width) / FW), sy = Math.floor(f.top + ((y + 0.5) * f.height) / FH); | |
| m[y * FW + x] = ref.mask[sy * ref.width + sx]; | |
| } | |
| return m; | |
| }; | |
| const vrows = []; | |
| const results = await trackFrames(model, processor, times.length, getFrame, { points: V.points, box: V.box }, { | |
| onFrame: async (res, image) => { | |
| const gt = gtFor(times[res.index]); | |
| const row = { i: res.index, t: times[res.index], area: res.area, lost: res.lost, maskIoUvsGeometryGT: +maskIoU(res.mask, gt).toFixed(4), objectScoreLogit: +res.objectScoreLogit.toFixed(2), encodeMs: Math.round(res.encodeMs), decodeMs: Math.round(res.decodeMs) }; | |
| vrows.push(row); console.log('frame', JSON.stringify(row)); | |
| if (res.index % 5 === 0 || res.index === times.length - 1) await png(overlayMask(drawPrompt(image.rgba().data, FW, FH, res.prompt.points, res.prompt.box), res.mask, FW, FH), FW, FH, `${out}/video/frame-${String(res.index).padStart(2, '0')}.png`); | |
| }, | |
| }); | |
| const ious = vrows.map((r) => r.maskIoUvsGeometryGT); | |
| report.video = { file: V.file, duration, sampledFps: V.fps, frames: times.length, initPrompt: { points: V.points, box: V.box }, refBoxOriginal: V.refBoxOriginal, refMaskArea: describeMask(ref.mask, ref.width, ref.height), | |
| meanIoU: +(ious.reduce((a, b) => a + b, 0) / ious.length).toFixed(4), minIoU: Math.min(...ious), lostFrames: vrows.filter((r) => r.lost).length, | |
| meanFrameMs: Math.round(vrows.reduce((a, r) => a + r.encodeMs + r.decodeMs, 0) / vrows.length), perFrame: vrows }; | |
| writeFileSync(`${out}/node-live-report.json`, JSON.stringify(report, null, 2)); | |
| console.log('video', JSON.stringify({ ...report.video, perFrame: undefined })); | |
| console.log('written', out); | |