// Model-facing core shared by the browser worker and the Node evidence script. // `model` = Sam3TrackerModel, `processor` = its AutoProcessor, `image` = RawImage (RGB). import { buildPrompt, bestMaskIndex, sliceMask, nextFramePrompt, isLost, maskArea } from './lib.js'; /** Run the (expensive) vision encoder once per image/frame. */ export async function encodeImage(model, processor, image) { const inputs = await processor(image); const emb = await model.get_image_embeddings(inputs); return { inputs, emb }; } /** Decode one object from cached embeddings + prompt (points/box). Cheap: runs only prompt encoder + mask decoder. */ export async function decodePrompt(model, processor, image, enc, points, box) { const prompt = buildPrompt(points, box); // Re-run the processor only to get correctly rescaled prompt tensors (pixel_values recomputed is wasted but // cheap relative to the encoder; embeddings are reused). const pin = await processor(image, prompt); const feeds = { ...enc.emb }; for (const k of ['input_points', 'input_labels', 'input_boxes']) if (pin[k]) feeds[k] = pin[k]; const out = await model(feeds); const masks = await processor.post_process_masks(out.pred_masks, pin.original_sizes, pin.reshaped_input_sizes); const m = masks[0]; // dims [1, K, H, W] const [, K, H, W] = m.dims; const scores = Array.from(out.iou_scores.data).slice(0, K); const k = bestMaskIndex(scores); const objectScoreLogit = out.object_score_logits ? Number(out.object_score_logits.data[0]) : undefined; return { mask: sliceMask(m.data, k, H, W), width: W, height: H, scores, chosen: k, objectScoreLogit }; } /** * Simple video propagation by RE-PROMPTING (heuristic of this app, not SAM's memory tracker): * frame 0 uses the user prompt; every next frame is encoded independently and prompted with the previous * good mask's expanded bbox + an interior point. If the object looks lost, the last good prompt is reused. * getFrame(i) -> Promise; onFrame(result) is awaited after each frame; shouldStop() aborts. */ export async function trackFrames(model, processor, nFrames, getFrame, initPrompt, { onFrame, shouldStop, margin = 0.1 } = {}) { let prompt = { points: initPrompt.points || [], box: initPrompt.box || null }; let prevArea = 0; const results = []; for (let i = 0; i < nFrames; i++) { if (shouldStop?.()) break; const image = await getFrame(i); const t0 = performance.now(); const enc = await encodeImage(model, processor, image); const t1 = performance.now(); const r = await decodePrompt(model, processor, image, enc, prompt.points, prompt.box); const t2 = performance.now(); const area = maskArea(r.mask); const lost = i > 0 && isLost({ objectScoreLogit: r.objectScoreLogit, area, prevArea }); const res = { index: i, ...r, area, lost, prompt, encodeMs: t1 - t0, decodeMs: t2 - t1 }; results.push(res); if (!lost) { prevArea = area; prompt = nextFramePrompt(r.mask, r.width, r.height, margin) || prompt; } await onFrame?.(res, image); } return results; }