Segizu's picture
Rotoscopia Lab: segmentación interactiva + seguimiento simple en vídeo (SAM 3 tracker, transformers.js 4.3.2)
87b5db3 verified
Raw History Blame Contribute Delete
3.1 kB
// Model-facing core shared by the browser worker and the Node evidence script.
// `model` = Sam3TrackerModel, `processor` = its AutoProcessor, `image` = RawImage (RGB).
import { buildPrompt, bestMaskIndex, sliceMask, nextFramePrompt, isLost, maskArea } from './lib.js';
/** Run the (expensive) vision encoder once per image/frame. */
export async function encodeImage(model, processor, image) {
const inputs = await processor(image);
const emb = await model.get_image_embeddings(inputs);
return { inputs, emb };
}
/** Decode one object from cached embeddings + prompt (points/box). Cheap: runs only prompt encoder + mask decoder. */
export async function decodePrompt(model, processor, image, enc, points, box) {
const prompt = buildPrompt(points, box);
// Re-run the processor only to get correctly rescaled prompt tensors (pixel_values recomputed is wasted but
// cheap relative to the encoder; embeddings are reused).
const pin = await processor(image, prompt);
const feeds = { ...enc.emb };
for (const k of ['input_points', 'input_labels', 'input_boxes']) if (pin[k]) feeds[k] = pin[k];
const out = await model(feeds);
const masks = await processor.post_process_masks(out.pred_masks, pin.original_sizes, pin.reshaped_input_sizes);
const m = masks[0]; // dims [1, K, H, W]
const [, K, H, W] = m.dims;
const scores = Array.from(out.iou_scores.data).slice(0, K);
const k = bestMaskIndex(scores);
const objectScoreLogit = out.object_score_logits ? Number(out.object_score_logits.data[0]) : undefined;
return { mask: sliceMask(m.data, k, H, W), width: W, height: H, scores, chosen: k, objectScoreLogit };
}
/**
* Simple video propagation by RE-PROMPTING (heuristic of this app, not SAM's memory tracker):
* frame 0 uses the user prompt; every next frame is encoded independently and prompted with the previous
* good mask's expanded bbox + an interior point. If the object looks lost, the last good prompt is reused.
* getFrame(i) -> Promise<RawImage>; onFrame(result) is awaited after each frame; shouldStop() aborts.
*/
export async function trackFrames(model, processor, nFrames, getFrame, initPrompt, { onFrame, shouldStop, margin = 0.1 } = {}) {
let prompt = { points: initPrompt.points || [], box: initPrompt.box || null };
let prevArea = 0;
const results = [];
for (let i = 0; i < nFrames; i++) {
if (shouldStop?.()) break;
const image = await getFrame(i);
const t0 = performance.now();
const enc = await encodeImage(model, processor, image);
const t1 = performance.now();
const r = await decodePrompt(model, processor, image, enc, prompt.points, prompt.box);
const t2 = performance.now();
const area = maskArea(r.mask);
const lost = i > 0 && isLost({ objectScoreLogit: r.objectScoreLogit, area, prevArea });
const res = { index: i, ...r, area, lost, prompt, encodeMs: t1 - t0, decodeMs: t2 - t1 };
results.push(res);
if (!lost) {
prevArea = area;
prompt = nextFramePrompt(r.mask, r.width, r.height, margin) || prompt;
}
await onFrame?.(res, image);
}
return results;
}