/* Visionary — browser UI. Inference runs locally via web/pipeline.js. */ import { Pipeline } from "./pipeline.js"; const $ = s => document.querySelector(s); const chat = $("#chat"), fileIn = $("#file"), drop = $("#drop"); const esc = s => String(s).replace(/[&<>"']/g, c => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); const tick = () => new Promise(r => setTimeout(r, 16)); const T0 = performance.now(); const log = s => console.log(`[Visionary] +${Math.round(performance.now() - T0)}ms ${s}`); const MAX_BYTES = 15 * 1024 * 1024; const CACHE = "visionary-v1"; let reg, classes, pipe = null, current = null, busy = false, ready = null; const loadedClassifiers = {}; /* ------------------------------------------------ cached download w/ progress */ async function fetchCached(url, onProgress) { let cache = null; try { cache = await caches.open(CACHE); const hit = await cache.match(url); if (hit) { onProgress(1, 1); return new Uint8Array(await hit.arrayBuffer()); } } catch (e) { cache = null; } const r = await fetch(url); if (!r.ok) throw new Error(`Could not download ${url} (${r.status})`); const total = +r.headers.get("content-length") || 0; const reader = r.body.getReader(), parts = []; let got = 0; for (;;) { const { done, value } = await reader.read(); if (done) break; parts.push(value); got += value.length; onProgress(got, total || got); } const buf = new Uint8Array(got); let o = 0; for (const p of parts) { buf.set(p, o); o += p.length; } try { if (cache) await cache.put(url, new Response(buf, { headers: { "content-type": "application/octet-stream" } })); } catch (e) { /* quota: fine */ } return buf; } async function downloadAll(items) { const totals = items.map(i => i.size || 1), done = items.map(() => 0); const T = totals.reduce((a, b) => a + b, 0); const show = label => { const pct = Math.min(100, (done.reduce((a, b) => a + b, 0) / T) * 100); $("#loaderBar").style.width = pct.toFixed(1) + "%"; $("#loaderText").textContent = `${label} · ${pct.toFixed(0)}% of ${(T / 1048576).toFixed(0)} MB (first visit only, then cached)`; }; log(`download start (${items.length} files)`); return Promise.all(items.map((it, k) => fetchCached(it.url, (g, t) => { done[k] = (g / t) * totals[k]; show(it.label); }) .then(b => { log(`downloaded ${it.label} (${(b.length / 1048576).toFixed(1)} MB)`); return b; }))); } async function loadOpenCV(bytes) { const url = URL.createObjectURL(new Blob([bytes], { type: "text/javascript" })); await new Promise((res, rej) => { const s = document.createElement("script"); s.src = url; s.onload = res; s.onerror = rej; document.head.appendChild(s); }); // Emscripten's Module object defines .then(), so `await cv` (or returning cv // from an async function) makes the browser resolve a self-referencing // thenable forever and freezes the tab. Poll for readiness instead, then // strip .then and hand the module back wrapped in an object. for (let i = 0; !(window.cv && window.cv.Mat); i++) { if (i > 1200) throw new Error("OpenCV.js failed to initialise"); await new Promise(r => setTimeout(r, 50)); } const cv = window.cv; if (typeof cv.then === "function") delete cv.then; return { cv }; } /* --------------------------------------------------------------- bootstrap */ async function init() { try { [reg, classes] = await Promise.all([fetch("models/models.json").then(r => r.json()), fetch("data/classes.json").then(r => r.json())]); let saved = null; try { saved = localStorage.getItem("visionary-model"); } catch (e) {} current = reg.models[saved] ? saved : reg.default; renderMenu(); const spec = reg.models[current]; const A = reg.assets; const [cvBytes, wasm, yolo, cls] = await downloadAll([ { url: "vendor/opencv.js", label: "OpenCV.js", size: A.opencv }, { url: "vendor/ort-wasm-simd-threaded.wasm", label: "ONNX Runtime", size: A.wasm }, { url: "models/yolov8n.onnx", label: "YOLOv8 detector", size: A.yolo }, { url: spec.file, label: spec.name, size: spec.size }, ]); const step = s => { log(s); $("#loaderText").textContent = s; }; step("Initialising models…"); await tick(); ort.env.wasm.wasmPaths = new URL("vendor/", location.href).href; try { ort.env.wasm.wasmBinary = wasm.buffer; } catch (e) {} ort.env.wasm.numThreads = self.crossOriginIsolated ? Math.min(4, navigator.hardwareConcurrency || 2) : 1; // Run inference in a Web Worker so the page (spinner, scrolling) stays responsive. ort.env.wasm.proxy = true; step("Starting OpenCV.js…"); await tick(); const { cv } = await loadOpenCV(cvBytes); step(`Starting ONNX Runtime (${ort.env.wasm.numThreads} threads, isolated=${self.crossOriginIsolated})…`); await tick(); pipe = new Pipeline(ort, cv, { classes, temperature: spec.temperature, imgSize: spec.imgSize }); window.visionaryPipeline = pipe; // exposed for debugging / benchmarking in devtools // In proxy (worker) mode ORT *transfers* the buffer to the worker, detaching // it — so always hand ORT a copy and keep the original for model switching. await pipe.load(yolo, cls.slice()); loadedClassifiers[current] = cls; step("Warming up…"); await tick(); // warm-up so the first real request is fast await pipe.analyze({ data: new Uint8ClampedArray(64 * 64 * 4).fill(128), width: 64, height: 64 }, { withStages: false }); $("#loader")?.classList.add("done"); log("ready"); } catch (e) { console.error(e); $("#loaderText").textContent = "Could not load the models: " + (e.message || e) + ". Please reload the page."; throw e; } } ready = init(); /* -------------------------------------------------------------- model menu */ function renderMenu() { const m = reg.models[current]; $("#modelName").textContent = m ? m.name : "Model"; $("#menu").innerHTML = Object.entries(reg.models).map(([k, x]) => ``).join(""); } $("#pickerBtn").onclick = e => { e.stopPropagation(); $("#menu").classList.toggle("open"); }; $("#menu").onclick = async e => { const b = e.target.closest("button[data-k]"); if (!b || busy) return; const k = b.dataset.k; $("#menu").classList.remove("open"); if (k === current) return; busy = true; $("#sendBtn").disabled = true; const prev = current; $("#modelName").textContent = "Loading " + reg.models[k].name + "…"; try { await ready; const spec = reg.models[k]; const bytes = loadedClassifiers[k] || await fetchCached(spec.file, (g, t) => { $("#modelName").textContent = `Loading ${Math.round(g / t * 100)}%`; }); loadedClassifiers[k] = bytes; await pipe.setClassifier(bytes.slice(), {}, { temperature: spec.temperature, imgSize: spec.imgSize }); current = k; try { localStorage.setItem("visionary-model", k); } catch (e) {} } catch (err) { current = prev; addError("Could not switch model: " + (err.message || err)); } finally { renderMenu(); busy = false; $("#sendBtn").disabled = false; } }; document.addEventListener("click", () => $("#menu").classList.remove("open")); /* ------------------------------------------------------------------ input */ const SAMPLES = ["samples/ferrari.jpg", "samples/bmw_dark.jpg", "samples/rangerover.jpg"]; let sampleIdx = 0; async function trySample() { if (busy) return; const url = SAMPLES[sampleIdx++ % SAMPLES.length]; try { const blob = await (await fetch(url)).blob(); send(new File([blob], url.split("/").pop(), { type: "image/jpeg" })); } catch (e) { addError("Could not load the sample image."); } } $("#sampleBtn").onclick = trySample; window.visionarySample = trySample; document.querySelectorAll("[data-pick]").forEach(el => (el.onclick = () => !busy && fileIn.click())); fileIn.onchange = () => { if (fileIn.files[0]) send(fileIn.files[0]); fileIn.value = ""; }; ["dragenter", "dragover"].forEach(ev => document.addEventListener(ev, e => { e.preventDefault(); drop.classList.add("drag"); })); ["dragleave", "drop"].forEach(ev => document.addEventListener(ev, e => { e.preventDefault(); drop.classList.remove("drag"); })); document.addEventListener("drop", e => { const f = e.dataTransfer.files[0]; if (f) send(f); }); document.addEventListener("paste", e => { const it = [...e.clipboardData.items].find(i => i.type.startsWith("image/")); if (it) send(it.getAsFile()); }); const scrollEnd = () => window.scrollTo({ top: document.body.scrollHeight, behavior: "smooth" }); async function decode(file) { let bmp; try { bmp = await createImageBitmap(file, { imageOrientation: "from-image" }); } catch (e) { throw new Error("That file isn't a readable image. Please upload a JPG, PNG or WEBP."); } if (Math.min(bmp.width, bmp.height) < 32) throw new Error("Image is too small (minimum 32×32 pixels)."); const s = Math.min(1, 1600 / Math.max(bmp.width, bmp.height)); const w = Math.round(bmp.width * s), h = Math.round(bmp.height * s); const c = document.createElement("canvas"); c.width = w; c.height = h; const g = c.getContext("2d", { willReadFrequently: true }); g.drawImage(bmp, 0, 0, w, h); bmp.close && bmp.close(); return g.getImageData(0, 0, w, h); } function imgURL(id) { const c = document.createElement("canvas"); c.width = id.width; c.height = id.height; c.getContext("2d").putImageData(new ImageData(id.data, id.width, id.height), 0, 0); return c.toDataURL("image/jpeg", 0.85); } async function send(file) { if (busy) return; $("#hero")?.classList.add("compact"); if (!file.type.startsWith("image/")) { $("#hero")?.remove(); addError("That file isn't an image. Please upload a JPG, PNG or WEBP."); return; } if (file.size > MAX_BYTES) { $("#hero")?.remove(); addError("Image is larger than 15 MB. Please upload a smaller image."); return; } busy = true; $("#sendBtn").disabled = true; const hero = $("#hero"); if (hero) { if (pipe) hero.remove(); else hero.querySelector(".samples")?.remove(); } chat.insertAdjacentHTML("beforeend", `
Loading…
`; $("#closeB").onclick = closeBench; try { const b = await (await fetch("data/eval_results.json")).json(); const row = r => `Official Stanford Cars test set: ${esc(b.n_test)} images never seen in training. Server latency is CPU, 2 threads.
| Approach | Top-1 % | Top-5 % | Latency ms | Size MB |
|---|
| Config | Top-1 % | Top-5 % | Δ Top-1 |
|---|
| Condition | ${b.robustness.methods.map(m => `${esc(m)} | `).join("")}
|---|