rotoscopia-lab / source /src /worker.js
Segizu's picture
Rotoscopia Lab: segmentación interactiva + seguimiento simple en vídeo (SAM 3 tracker, transformers.js 4.3.2)
87b5db3 verified
Raw History Blame Contribute Delete
4.93 kB
// Web Worker: owns the model. One job at a time (main thread enforces it, worker double-checks).
import { Sam3TrackerModel, AutoProcessor, RawImage, env } from '@huggingface/transformers';
import { MODEL_ID, MODEL_REVISION, VARIANTS } from './config.js';
import { encodeImage, decodePrompt, trackFrames } from './sam.js';
env.allowLocalModels = false;
env.allowRemoteModels = true;
env.useBrowserCache = true; // weights cached by the browser Cache API after the first download
let model = null, processor = null, device = null;
let current = null; // { key, image, enc }
let busy = false, stopRequested = false;
const pendingFrames = new Map();
function configureWasm(wasmBase, wantWebGPU) {
// Self-hosted ONNX Runtime Web binaries (copied from the pinned npm package); no third-party CDN.
const suffix = wantWebGPU ? '.asyncify' : '';
env.backends.onnx.wasm.wasmPaths = { mjs: `${wasmBase}ort-wasm-simd-threaded${suffix}.mjs`, wasm: `${wasmBase}ort-wasm-simd-threaded${suffix}.wasm` };
const cores = self.navigator?.hardwareConcurrency || 2;
env.backends.onnx.wasm.numThreads = self.crossOriginIsolated ? Math.min(4, cores) : 1;
}
async function load({ wantWebGPU, wasmBase, variant, fallbackReason = null }) {
configureWasm(wasmBase, wantWebGPU);
const progress_callback = (p) => self.postMessage({ type: 'progress', data: { status: p.status, file: p.file, loaded: p.loaded, total: p.total } });
const dtype = VARIANTS[variant].dtype;
const dev = wantWebGPU ? 'webgpu' : 'wasm';
try {
processor = await AutoProcessor.from_pretrained(MODEL_ID, { revision: MODEL_REVISION });
model = await Sam3TrackerModel.from_pretrained(MODEL_ID, { revision: MODEL_REVISION, dtype, device: dev, progress_callback });
} catch (e) {
if (wantWebGPU) { self.postMessage({ type: 'webgpu-failed', reason: String(e?.message || e).slice(0, 200) }); return; }
throw e;
}
device = dev;
self.postMessage({ type: 'ready', device, variant, fallbackReason, threads: env.backends.onnx.wasm.numThreads, isolated: !!self.crossOriginIsolated });
}
const toImage = ({ width, height, buffer }) => new RawImage(new Uint8ClampedArray(buffer), width, height, 4).rgb();
async function exclusive(fn) {
if (!model) throw new Error('El modelo no está cargado.');
if (busy) throw new Error('Ya hay un trabajo en curso.');
busy = true;
try { return await fn(); } finally { busy = false; }
}
async function encode(msg) {
return exclusive(async () => {
const image = toImage(msg);
const t0 = performance.now();
const enc = await encodeImage(model, processor, image);
current = { key: msg.key, image, enc };
self.postMessage({ type: 'encoded', id: msg.id, key: msg.key, encodeMs: performance.now() - t0, device });
});
}
async function decode(msg) {
return exclusive(async () => {
if (!current || current.key !== msg.key) throw new Error('La imagen aún no está analizada.');
const t0 = performance.now();
const r = await decodePrompt(model, processor, current.image, current.enc, msg.points, msg.box);
const decodeMs = performance.now() - t0;
self.postMessage({ type: 'decoded', id: msg.id, key: msg.key, decodeMs, device, ...r }, [r.mask.buffer]);
});
}
async function track(msg) {
return exclusive(async () => {
stopRequested = false;
current = null; // embeddings of the still image are no longer valid
const getFrame = (i) => new Promise((resolve, reject) => {
pendingFrames.set(i, { resolve, reject });
self.postMessage({ type: 'need-frame', id: msg.id, index: i });
});
const results = await trackFrames(model, processor, msg.nFrames, getFrame, { points: msg.points, box: msg.box }, {
shouldStop: () => stopRequested,
onFrame: (res) => {
const mask = res.mask;
self.postMessage({ type: 'track-frame', id: msg.id, index: res.index, mask, width: res.width, height: res.height, scores: res.scores, area: res.area, lost: res.lost,
objectScoreLogit: res.objectScoreLogit, prompt: res.prompt, encodeMs: res.encodeMs, decodeMs: res.decodeMs, device }, [mask.buffer]);
},
});
self.postMessage({ type: 'track-done', id: msg.id, frames: results.length, stopped: stopRequested });
});
}
self.addEventListener('message', async (ev) => {
const msg = ev.data || {};
if (msg.type === 'frame') { const p = pendingFrames.get(msg.index); pendingFrames.delete(msg.index); if (p) msg.error ? p.reject(new Error(msg.error)) : p.resolve(toImage(msg)); return; }
if (msg.type === 'stop') { stopRequested = true; return; }
try {
if (msg.type === 'load') await load(msg);
else if (msg.type === 'encode') await encode(msg);
else if (msg.type === 'decode') await decode(msg);
else if (msg.type === 'track') await track(msg);
} catch (e) {
self.postMessage({ type: 'error', id: msg.id, stage: msg.type, message: String(e?.message || e).slice(0, 400) });
}
});