File size: 3,099 Bytes
87b5db3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
// Model-facing core shared by the browser worker and the Node evidence script.
// `model` = Sam3TrackerModel, `processor` = its AutoProcessor, `image` = RawImage (RGB).
import { buildPrompt, bestMaskIndex, sliceMask, nextFramePrompt, isLost, maskArea } from './lib.js';

/** Run the (expensive) vision encoder once per image/frame. */
export async function encodeImage(model, processor, image) {
  const inputs = await processor(image);
  const emb = await model.get_image_embeddings(inputs);
  return { inputs, emb };
}

/** Decode one object from cached embeddings + prompt (points/box). Cheap: runs only prompt encoder + mask decoder. */
export async function decodePrompt(model, processor, image, enc, points, box) {
  const prompt = buildPrompt(points, box);
  // Re-run the processor only to get correctly rescaled prompt tensors (pixel_values recomputed is wasted but
  // cheap relative to the encoder; embeddings are reused).
  const pin = await processor(image, prompt);
  const feeds = { ...enc.emb };
  for (const k of ['input_points', 'input_labels', 'input_boxes']) if (pin[k]) feeds[k] = pin[k];
  const out = await model(feeds);
  const masks = await processor.post_process_masks(out.pred_masks, pin.original_sizes, pin.reshaped_input_sizes);
  const m = masks[0]; // dims [1, K, H, W]
  const [, K, H, W] = m.dims;
  const scores = Array.from(out.iou_scores.data).slice(0, K);
  const k = bestMaskIndex(scores);
  const objectScoreLogit = out.object_score_logits ? Number(out.object_score_logits.data[0]) : undefined;
  return { mask: sliceMask(m.data, k, H, W), width: W, height: H, scores, chosen: k, objectScoreLogit };
}

/**
 * Simple video propagation by RE-PROMPTING (heuristic of this app, not SAM's memory tracker):
 * frame 0 uses the user prompt; every next frame is encoded independently and prompted with the previous
 * good mask's expanded bbox + an interior point. If the object looks lost, the last good prompt is reused.
 * getFrame(i) -> Promise<RawImage>; onFrame(result) is awaited after each frame; shouldStop() aborts.
 */
export async function trackFrames(model, processor, nFrames, getFrame, initPrompt, { onFrame, shouldStop, margin = 0.1 } = {}) {
  let prompt = { points: initPrompt.points || [], box: initPrompt.box || null };
  let prevArea = 0;
  const results = [];
  for (let i = 0; i < nFrames; i++) {
    if (shouldStop?.()) break;
    const image = await getFrame(i);
    const t0 = performance.now();
    const enc = await encodeImage(model, processor, image);
    const t1 = performance.now();
    const r = await decodePrompt(model, processor, image, enc, prompt.points, prompt.box);
    const t2 = performance.now();
    const area = maskArea(r.mask);
    const lost = i > 0 && isLost({ objectScoreLogit: r.objectScoreLogit, area, prevArea });
    const res = { index: i, ...r, area, lost, prompt, encodeMs: t1 - t0, decodeMs: t2 - t1 };
    results.push(res);
    if (!lost) {
      prevArea = area;
      prompt = nextFramePrompt(r.mask, r.width, r.height, margin) || prompt;
    }
    await onFrame?.(res, image);
  }
  return results;
}