File size: 8,021 Bytes
87b5db3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
// REAL inference evidence in Node: transformers.js (same version as the Space) + onnxruntime-node (CPU).
// Same pinned model files (q4f16) the browser downloads. Same app code (src/sam.js + src/lib.js).
// Usage: node scripts/evidence-node.mjs [outDir]
import { Sam3TrackerModel, AutoProcessor, RawImage, env } from '@huggingface/transformers';
import sharp from 'sharp';
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { MODEL_ID, MODEL_REVISION, EXAMPLES, VIDEO_EXAMPLE, TRANSFORMERS_VERSION } from '../src/config.js';
import { encodeImage, decodePrompt, trackFrames } from '../src/sam.js';
import { describeMask, boxIoU, fractionInside, overlayMask, cutout, maskIoU, sampleTimes } from '../src/lib.js';
import { hardware } from './hw.mjs';

const root = new URL('../', import.meta.url).pathname;
const out = process.argv[2] || '/workspace/evidence/rotoscopia-lab';
mkdirSync(out + '/images', { recursive: true }); mkdirSync(out + '/video', { recursive: true });
const ortNode = JSON.parse(readFileSync(root + 'node_modules/onnxruntime-node/package.json', 'utf8')).version;
const dtype = 'q4f16';
const png = (rgba, w, h, f) => sharp(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), { raw: { width: w, height: h, channels: 4 } }).png().toFile(f);
const maskPng = (m, w, h, f) => { const g = new Uint8Array(m.length); for (let i = 0; i < m.length; i++) g[i] = m[i] ? 255 : 0; return sharp(Buffer.from(g), { raw: { width: w, height: h, channels: 1 } }).png().toFile(f); };
const drawPrompt = (rgba, w, h, points, box) => { // mark prompts so the PNG documents the input
  const o = new Uint8ClampedArray(rgba);
  const put = (x, y, c) => { if (x < 0 || y < 0 || x >= w || y >= h) return; const j = (y * w + x) * 4; o[j] = c[0]; o[j + 1] = c[1]; o[j + 2] = c[2]; };
  for (const p of points || []) { const c = p.label ? [0, 220, 0] : [230, 0, 0]; for (let dy = -7; dy <= 7; dy++) for (let dx = -7; dx <= 7; dx++) if (dx * dx + dy * dy <= 49) put(Math.round(p.x) + dx, Math.round(p.y) + dy, c); }
  if (box) { const [x0, y0, x1, y1] = box.map(Math.round); for (let t = 0; t < 3; t++) { for (let x = x0; x <= x1; x++) { put(x, y0 + t, [255, 200, 0]); put(x, y1 - t, [255, 200, 0]); } for (let y = y0; y <= y1; y++) { put(x0 + t, y, [255, 200, 0]); put(x1 - t, y, [255, 200, 0]); } } }
  return o;
};

const report = { createdAt: new Date().toISOString(), kind: 'node-live-inference', runtime: { transformersjs: TRANSFORMERS_VERSION, onnxruntimeNode: ortNode, device: 'cpu', dtype }, model: { id: MODEL_ID, revision: MODEL_REVISION }, hardware: hardware(), images: [], video: null };
let t = performance.now();
const model = await Sam3TrackerModel.from_pretrained(MODEL_ID, { revision: MODEL_REVISION, dtype, device: 'cpu' });
const processor = await AutoProcessor.from_pretrained(MODEL_ID, { revision: MODEL_REVISION });
report.loadMs = Math.round(performance.now() - t);
console.log('model loaded', report.loadMs, 'ms');

for (const ex of EXAMPLES) {
  const image = await RawImage.read(root + 'public/examples/' + ex.file);
  t = performance.now(); const enc = await encodeImage(model, processor, image); const encodeMs = performance.now() - t;
  t = performance.now(); const r = await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decodeMs = performance.now() - t;
  // second decode = interactive re-click cost with cached embeddings
  t = performance.now(); await decodePrompt(model, processor, image, enc, ex.points, ex.box); const decode2Ms = performance.now() - t;
  const d = describeMask(r.mask, r.width, r.height);
  const rgba = image.rgba().data;
  const row = { id: ex.id, file: ex.file, size: [image.width, image.height], prompt: { points: ex.points, box: ex.box }, expectBox: ex.expectBox,
    maskArea: d.area, maskAreaPct: d.areaPct, maskBBox: d.bbox, bboxIoUvsExpected: +boxIoU(d.bbox, ex.expectBox).toFixed(4),
    fractionInsideExpected: +fractionInside(r.mask, r.width, r.height, ex.expectBox).toFixed(4),
    positivePointsOnMask: (ex.points || []).filter((p) => p.label).map((p) => !!r.mask[Math.round(p.y) * r.width + Math.round(p.x)]),
    negativePointsOffMask: (ex.points || []).filter((p) => !p.label).map((p) => !r.mask[Math.round(p.y) * r.width + Math.round(p.x)]),
    iouScores: r.scores.map((s) => +s.toFixed(4)), chosenMask: r.chosen, objectScoreLogit: +r.objectScoreLogit.toFixed(3),
    encodeMs: Math.round(encodeMs), decodeMs: Math.round(decodeMs), redecodeMs: Math.round(decode2Ms) };
  report.images.push(row);
  console.log(JSON.stringify(row));
  await png(overlayMask(drawPrompt(rgba, r.width, r.height, ex.points, ex.box), r.mask, r.width, r.height), r.width, r.height, `${out}/images/${ex.id}-overlay.png`);
  await maskPng(r.mask, r.width, r.height, `${out}/images/${ex.id}-mask.png`);
  await png(cutout(rgba, r.mask), r.width, r.height, `${out}/images/${ex.id}-cutout.png`);
}

// ---- Video: decode the bundled mp4 at the same timestamps the browser samples, track, compare to geometry GT.
const V = VIDEO_EXAMPLE, vmeta = JSON.parse(readFileSync(root + 'public/examples/video-pan.json', 'utf8'));
const mp4 = root + 'public/examples/' + V.file;
const duration = +execFileSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', mp4]).toString().trim();
const times = sampleTimes(duration, V.fps);
const [FW, FH] = vmeta.out;
const getFrame = async (i) => {
  const buf = execFileSync('ffmpeg', ['-nostdin', '-loglevel', 'error', '-ss', String(times[i]), '-i', mp4, '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'rgba', '-'], { maxBuffer: 64 << 20 });
  return new RawImage(new Uint8ClampedArray(buf), FW, FH, 4).rgb();
};
// Ground truth: segment the mug ONCE on the full still photo with the equivalent box, then warp it into each frame.
const still = await RawImage.read(root + 'public/examples/' + vmeta.source);
const stillEnc = await encodeImage(model, processor, still);
const ref = await decodePrompt(model, processor, still, stillEnc, [], V.refBoxOriginal);
const frameIndexAt = (tt) => Math.min(vmeta.frames.length - 1, Math.round(tt * vmeta.fps));
const gtFor = (tt) => {
  const f = vmeta.frames[frameIndexAt(tt)], m = new Uint8Array(FW * FH);
  for (let y = 0; y < FH; y++) for (let x = 0; x < FW; x++) {
    const sx = Math.floor(f.left + ((x + 0.5) * f.width) / FW), sy = Math.floor(f.top + ((y + 0.5) * f.height) / FH);
    m[y * FW + x] = ref.mask[sy * ref.width + sx];
  }
  return m;
};
const vrows = [];
const results = await trackFrames(model, processor, times.length, getFrame, { points: V.points, box: V.box }, {
  onFrame: async (res, image) => {
    const gt = gtFor(times[res.index]);
    const row = { i: res.index, t: times[res.index], area: res.area, lost: res.lost, maskIoUvsGeometryGT: +maskIoU(res.mask, gt).toFixed(4), objectScoreLogit: +res.objectScoreLogit.toFixed(2), encodeMs: Math.round(res.encodeMs), decodeMs: Math.round(res.decodeMs) };
    vrows.push(row); console.log('frame', JSON.stringify(row));
    if (res.index % 5 === 0 || res.index === times.length - 1) await png(overlayMask(drawPrompt(image.rgba().data, FW, FH, res.prompt.points, res.prompt.box), res.mask, FW, FH), FW, FH, `${out}/video/frame-${String(res.index).padStart(2, '0')}.png`);
  },
});
const ious = vrows.map((r) => r.maskIoUvsGeometryGT);
report.video = { file: V.file, duration, sampledFps: V.fps, frames: times.length, initPrompt: { points: V.points, box: V.box }, refBoxOriginal: V.refBoxOriginal, refMaskArea: describeMask(ref.mask, ref.width, ref.height),
  meanIoU: +(ious.reduce((a, b) => a + b, 0) / ious.length).toFixed(4), minIoU: Math.min(...ious), lostFrames: vrows.filter((r) => r.lost).length,
  meanFrameMs: Math.round(vrows.reduce((a, r) => a + r.encodeMs + r.decodeMs, 0) / vrows.length), perFrame: vrows };
writeFileSync(`${out}/node-live-report.json`, JSON.stringify(report, null, 2));
console.log('video', JSON.stringify({ ...report.video, perFrame: undefined }));
console.log('written', out);