File size: 3,696 Bytes
87b5db3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
// Model-free tests of src/sam.js with a fake model/processor (checks wiring, not model quality).
import { describe, it, expect } from 'vitest';
import { encodeImage, decodePrompt, trackFrames } from '../src/sam.js';

const H = 20, W = 30;
function fakes({ lostAt = -1 } = {}) {
  const calls = { enc: 0, dec: [], proc: [] };
  const processor = async (image, prompt) => {
    calls.proc.push(prompt);
    const t = (v) => ({ data: v, dims: [] });
    return { pixel_values: t('px'), original_sizes: [[H, W]], reshaped_input_sizes: [[H, W]], ...(prompt ? Object.fromEntries(Object.entries(prompt).map(([k, v]) => [k, t(v)])) : {}) };
  };
  let frame = 0;
  processor.post_process_masks = async (pred) => {
    // 3 candidates: k=0 empty, k=1 small square at the "object" position, k=2 full image
    const data = new Uint8Array(3 * H * W);
    const ox = pred.shift;
    for (let y = 5; y < 10; y++) for (let x = ox; x < ox + 5; x++) data[1 * H * W + y * W + x] = 1;
    data.fill(1, 2 * H * W);
    return [{ dims: [1, 3, H, W], data }];
  };
  const model = async (feeds) => {
    calls.dec.push(feeds);
    const i = frame++;
    return { pred_masks: { shift: 2 + i * 2 }, iou_scores: { data: new Float32Array([0.1, 0.9, 0.3]) }, object_score_logits: { data: new Float32Array([i === lostAt ? -3 : 5]) } };
  };
  model.get_image_embeddings = async () => { calls.enc++; return { 'image_embeddings.0': 'E0', 'image_embeddings.1': 'E1', 'image_embeddings.2': 'E2' }; };
  return { model, processor, calls };
}

describe('sam.js core', () => {
  it('decodePrompt reuses cached embeddings, forwards prompts and picks best candidate', async () => {
    const { model, processor, calls } = fakes();
    const enc = await encodeImage(model, processor, {});
    const r = await decodePrompt(model, processor, {}, enc, [{ x: 3, y: 4, label: 1 }], null);
    expect(calls.enc).toBe(1);
    expect(calls.dec[0]['image_embeddings.0']).toBe('E0');
    expect(calls.dec[0].input_points.data).toEqual([[[[3, 4]]]]);
    expect(calls.dec[0].pixel_values).toBeUndefined(); // encoder not re-run
    expect(r.chosen).toBe(1); expect(r.width).toBe(W); expect(r.height).toBe(H);
    expect(r.mask.reduce((a, b) => a + b, 0)).toBe(25);
    expect(r.objectScoreLogit).toBe(5);
  });
  it('trackFrames re-prompts each frame from the previous mask and follows the moving object', async () => {
    const { model, processor, calls } = fakes();
    const res = await trackFrames(model, processor, 4, async () => ({}), { points: [], box: [2, 5, 6, 9] });
    expect(res).toHaveLength(4);
    expect(calls.enc).toBe(4); // one encoder pass per frame
    expect(calls.dec[0].input_boxes.data).toEqual([[[2, 5, 6, 9]]]);
    // frame 1 prompt derived from frame-0 mask (x 2..6, y 5..9): interior point (4,7) + expanded box
    expect(res[1].prompt.points[0]).toEqual({ x: 4, y: 7, label: 1 });
    expect(res[1].prompt.box[0]).toBeCloseTo(1.6); expect(res[1].prompt.box[2]).toBeCloseTo(6.4);
    expect(res[3].prompt.points[0].x).toBe(8); // followed the shift of +2 px/frame
    expect(res.every((r) => !r.lost)).toBe(true);
  });
  it('a lost frame does not update the prompt; stop aborts', async () => {
    const { model, processor } = fakes({ lostAt: 1 });
    const res = await trackFrames(model, processor, 3, async () => ({}), { points: [{ x: 4, y: 7, label: 1 }], box: null });
    expect(res[1].lost).toBe(true);
    expect(res[2].prompt).toEqual(res[1].prompt); // reused last good prompt
    let n = 0;
    const res2 = await trackFrames(fakes().model, fakes().processor, 10, async () => ({}), { points: [{ x: 1, y: 1, label: 1 }] }, { shouldStop: () => n++ >= 2 });
    expect(res2).toHaveLength(2);
  });
});