toolvision-detector / index.html
moindedroper's picture
Update index.html
6d8785b verified
Raw History Blame Contribute Delete
20.3 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
<title>ToolVision: hand-tool detector</title>
<meta name="description" content="Detects wrenches, screwdrivers, hammers and pliers in your photo. Runs entirely in your browser.">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>🔧</text></svg>">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Archivo:wdth,wght@62..125,400..800&display=swap" rel="stylesheet">
<style>
:root {
color-scheme: light;
--board: #E4E7EB; /* steel pegboard */
--hole: #C7CCD3;
--panel: #FFFFFF;
--ink: #1C1E22;
--muted: #596070;
--line: #D5D9DF;
--foam: #F2BE22; /* shadow-board foam */
--foam-deep: #C9930A; /* cut-out in the foam */
--on-foam: #1C1E22;
--focus: #2563EB;
--error: #B42318;
--font: "Archivo", "Helvetica Neue", Arial, system-ui, sans-serif;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
color-scheme: dark;
--board: #191B1F;
--hole: #262A30;
--panel: #23262B;
--ink: #ECEEF1;
--muted: #A3AAB5;
--line: #353A42;
--foam: #D9A916;
--foam-deep: #96720A;
--on-foam: #1C1E22;
--focus: #7AA7FF;
--error: #FF8A7A;
}
}
:root[data-theme="dark"] {
color-scheme: dark;
--board: #191B1F; --hole: #262A30; --panel: #23262B; --ink: #ECEEF1;
--muted: #A3AAB5; --line: #353A42; --foam: #D9A916; --foam-deep: #96720A;
--on-foam: #1C1E22; --focus: #7AA7FF; --error: #FF8A7A;
}
* { box-sizing: border-box; }
html { -webkit-text-size-adjust: 100%; }
body {
margin: 0;
font-family: var(--font);
font-size: 1rem;
line-height: 1.5;
color: var(--ink);
background-color: var(--board);
background-image: radial-gradient(circle, var(--hole) 3px, transparent 3.5px);
background-size: 28px 28px;
background-position: 14px 14px;
padding: env(safe-area-inset-top, 0px) 0 env(safe-area-inset-bottom, 0px);
}
.wrap { max-width: 1120px; margin: 0 auto; padding: 2.5rem 1.25rem 3rem; }
header { margin-bottom: 2rem; }
h1 {
font-size: clamp(2.75rem, 7vw, 5rem);
font-weight: 800;
font-stretch: 125%;
font-variation-settings: "wdth" 125;
line-height: 0.95;
letter-spacing: -0.02em;
margin: 0 0 0.75rem;
}
.lede { font-size: 1.1875rem; max-width: 36em; margin: 0; color: var(--ink); }
.lede span { color: var(--muted); }
.layout {
display: grid;
grid-template-columns: minmax(0, 1.65fr) minmax(0, 1fr);
gap: 1.25rem;
align-items: start;
}
@media (max-width: 820px) { .layout { grid-template-columns: minmax(0, 1fr); } }
/* The stage: an empty shadow-board slot until a photo arrives */
.stage {
position: relative;
background: var(--foam);
color: var(--on-foam);
border-radius: 6px;
overflow: hidden;
min-height: 420px;
display: grid;
place-items: center;
}
.stage.has-image { background: var(--panel); min-height: 0; }
.stage.dragging { outline: 4px dashed var(--on-foam); outline-offset: -14px; }
.slot { text-align: center; padding: 2rem 1.5rem; display: grid; justify-items: center; gap: 1rem; }
.slot svg { width: min(260px, 60vw); height: auto; }
.slot-title { font-size: 1.5rem; font-weight: 700; font-stretch: 110%; font-variation-settings: "wdth" 110; margin: 0; }
.slot-hint { margin: 0; font-size: 0.9375rem; }
.stage.has-image .slot { display: none; }
canvas#out { display: none; width: 100%; height: auto; }
.stage.has-image canvas#out { display: block; }
button {
font: inherit;
font-weight: 700;
font-size: 1rem;
border: 2px solid var(--on-foam);
background: var(--on-foam);
color: var(--foam);
padding: 0.7rem 1.3rem;
border-radius: 4px;
cursor: pointer;
}
button:hover { filter: brightness(1.15); }
button:focus-visible, .thumb:focus-visible, input[type="range"]:focus-visible {
outline: 3px solid var(--focus);
outline-offset: 3px;
}
button:disabled { opacity: 0.55; cursor: progress; }
.panel {
background: var(--panel);
border-radius: 6px;
padding: 1.25rem 1.25rem 1.5rem;
}
.status { margin: 0 0 1.25rem; font-weight: 600; min-height: 3em; }
.status.error { color: var(--error); }
.control label { display: flex; justify-content: space-between; font-weight: 600; margin-bottom: 0.35rem; }
.control output { font-variant-numeric: tabular-nums; color: var(--muted); }
input[type="range"] { width: 100%; accent-color: var(--foam-deep); }
h2 { font-size: 1.0625rem; font-weight: 700; margin: 1.5rem 0 0.5rem; }
.results { list-style: none; padding: 0; margin: 0; }
.results li {
display: grid;
grid-template-columns: 14px 1fr auto;
gap: 0.75rem;
align-items: center;
padding: 0.55rem 0;
border-bottom: 1px solid var(--line);
}
.results li:last-child { border-bottom: 0; }
.swatch { width: 14px; height: 14px; border-radius: 3px; }
.results .conf { font-variant-numeric: tabular-nums; color: var(--muted); }
.results .empty { display: block; color: var(--muted); border: 0; }
.panel .again { margin-top: 1.25rem; width: 100%; background: var(--ink); border-color: var(--ink); color: var(--panel); }
.classes { display: flex; flex-wrap: wrap; gap: 0.4rem 1rem; margin: 0; padding: 0; list-style: none; color: var(--muted); font-size: 0.9375rem; }
.classes li { display: flex; align-items: center; gap: 0.4rem; }
.examples { margin-top: 1.25rem; }
.examples h2 { margin-top: 0; }
.thumbs { display: flex; gap: 0.75rem; overflow-x: auto; padding-bottom: 0.25rem; }
.thumb {
flex: 0 0 auto;
padding: 0;
border: 0;
background: var(--panel);
border-radius: 6px;
overflow: hidden;
width: 132px;
color: var(--ink);
text-align: left;
}
.thumb img { display: block; width: 132px; height: 99px; object-fit: cover; }
.thumb span { display: block; padding: 0.35rem 0.5rem; font-size: 0.8125rem; font-weight: 600; }
.thumb:hover { filter: none; outline: 2px solid var(--foam-deep); }
footer {
margin-top: 2.5rem;
background: var(--panel);
border-radius: 6px;
padding: 1.25rem 1.25rem 1.5rem;
display: grid;
grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
gap: 0.5rem 2rem;
}
footer h2 { margin-top: 0; }
footer p { margin: 0 0 0.75rem; max-width: 38em; color: var(--muted); }
footer a { color: var(--ink); font-weight: 600; }
.visually-hidden { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; }
</style>
</head>
<body>
<div class="wrap">
<header>
<h1>ToolVision</h1>
<p class="lede">Finds wrenches, screwdrivers, hammers and pliers in a photo.
<span>The model runs in your browser, so your photo never leaves your device.</span></p>
</header>
<div class="layout">
<div>
<div class="stage" id="stage">
<div class="slot">
<!-- An empty tool slot, like a cut-out in a shadow board -->
<svg viewBox="0 0 240 120" aria-hidden="true">
<g transform="rotate(-12 120 60)" fill="var(--foam-deep)">
<rect x="44" y="49" width="152" height="22" rx="11"/>
<circle cx="198" cy="60" r="25"/>
<circle cx="42" cy="60" r="28"/>
<circle cx="198" cy="60" r="12" fill="var(--foam)"/>
<rect x="6" y="51" width="40" height="18" fill="var(--foam)"/>
<circle cx="45" cy="60" r="9" fill="var(--foam)"/>
</g>
</svg>
<p class="slot-title">Drop a photo here</p>
<button type="button" id="choose">Choose a photo</button>
<p class="slot-hint">Or paste one with Ctrl+V. On a phone you can take a picture.</p>
</div>
<canvas id="out" role="img" aria-label="Your photo with detected tools outlined"></canvas>
</div>
<section class="examples" id="examples" hidden>
<h2>Or try one of these</h2>
<div class="thumbs" id="thumbs"></div>
</section>
</div>
<aside class="panel">
<p class="status" id="status" aria-live="polite">Loading the model…</p>
<div class="control">
<label for="conf">Minimum confidence <output id="confOut" for="conf">25%</output></label>
<input type="range" id="conf" min="5" max="95" step="5" value="25">
</div>
<h2>Detections</h2>
<ul class="results" id="results"><li class="empty">Nothing yet.</li></ul>
<button type="button" class="again" id="again" hidden>Choose another photo</button>
<h2>What it looks for</h2>
<ul class="classes" id="classes"></ul>
</aside>
</div>
<footer>
<div>
<h2>About the model</h2>
<!-- EDIT: describe exactly which experiment this model comes from -->
<p>A YOLO26n detector exported to ONNX. It was trained as part of a study comparing
synthetic images rendered in Blender with real photos, and evaluated on real photos it never saw during training.</p>
<p><a id="reportLink" href="#">Project report and results</a> &nbsp; <a id="datasetLink" href="#">Synthetic dataset</a></p>
</div>
<div>
<h2>Known limitations</h2>
<!-- EDIT: replace with your real failure cases from Day 20 -->
<p>Works best with one or a few tools in clear view. Heavy clutter, unusual angles,
very small tools and tools that are partly hidden reduce accuracy.</p>
<p>Licensed AGPL-3.0, because the model was trained with Ultralytics YOLO.</p>
</div>
</footer>
</div>
<input type="file" id="file" accept="image/*" class="visually-hidden" tabindex="-1" aria-hidden="true">
<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.1/dist/ort.min.js"></script>
<script>
/* ======================= EDIT THESE ======================= */
const CONFIG = {
modelUrl: "best.onnx",
classes: ["wrench", "screwdriver", "hammer", "pliers"], // same order as data.yaml
colors: ["#2563EB", "#DC2626", "#059669", "#9333EA"],
inputSize: 640,
iou: 0.7, // Ultralytics' default NMS threshold for predict()
reportUrl: "https://github.com/YOUR_USER/toolvision",
datasetUrl: "https://huggingface.co/datasets/YOUR_USER/toolvision",
// Put 3–4 photos in an examples/ folder and list them here:
examples: [
// { src: "examples/workbench.jpg", label: "Workbench" },
// { src: "examples/pliers-floor.jpg", label: "Pliers on concrete" },
],
};
/* ========================================================== */
const $ = (id) => document.getElementById(id);
const stage = $("stage"), out = $("out"), statusEl = $("status");
const confInput = $("conf"), confOut = $("confOut"), resultsEl = $("results");
const fileInput = $("file");
let session = null, inputName = null;
let current = null; // { img, meta, output, ms }
let busy = false;
$("reportLink").href = CONFIG.reportUrl;
$("datasetLink").href = CONFIG.datasetUrl;
$("classes").innerHTML = CONFIG.classes.map((c, i) =>
`<li><span class="swatch" style="background:${CONFIG.colors[i]}"></span>${c}</li>`).join("");
function setStatus(text, isError = false) {
statusEl.textContent = text;
statusEl.classList.toggle("error", isError);
}
/* ---------- Model loading ---------- */
const modelReady = (async () => {
try {
if (typeof ort === "undefined") throw new Error("onnxruntime-web did not load");
ort.env.wasm.wasmPaths = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.1/dist/";
session = await ort.InferenceSession.create(CONFIG.modelUrl, {
executionProviders: ["wasm"],
graphOptimizationLevel: "all",
});
inputName = session.inputNames[0];
// Warm-up run so the first real photo is fast
const S = CONFIG.inputSize;
await session.run({ [inputName]: new ort.Tensor("float32", new Float32Array(3 * S * S), [1, 3, S, S]) });
if (!current) setStatus("Ready. Choose a photo to start.");
return true;
} catch (err) {
console.error(err);
setStatus(`Couldn't load the model. Check that ${CONFIG.modelUrl} sits next to index.html and that the page is served over http(s), not opened as a file.`, true);
return false;
}
})();
/* ---------- Pre-processing: letterbox like Ultralytics ---------- */
const prep = document.createElement("canvas");
prep.width = prep.height = CONFIG.inputSize;
const prepCtx = prep.getContext("2d", { willReadFrequently: true });
function preprocess(img) {
const S = CONFIG.inputSize;
const w = img.naturalWidth, h = img.naturalHeight;
const r = Math.min(S / w, S / h);
const nw = Math.round(w * r), nh = Math.round(h * r);
const left = Math.round((S - nw) / 2 - 0.1), top = Math.round((S - nh) / 2 - 0.1);
prepCtx.fillStyle = "rgb(114,114,114)";
prepCtx.fillRect(0, 0, S, S);
prepCtx.imageSmoothingEnabled = true;
prepCtx.imageSmoothingQuality = "high";
prepCtx.drawImage(img, left, top, nw, nh);
const px = prepCtx.getImageData(0, 0, S, S).data;
const area = S * S, data = new Float32Array(3 * area);
for (let i = 0; i < area; i++) {
data[i] = px[i * 4] / 255; // R
data[i + area] = px[i * 4 + 1] / 255; // G
data[i + 2 * area] = px[i * 4 + 2] / 255; // B
}
return { tensor: new ort.Tensor("float32", data, [1, 3, S, S]), r, left, top, w, h };
}
/* ---------- Post-processing: decode [1, 4+nc, N] + per-class NMS ---------- */
function decode(output, meta, conf) {
const [, ch, n] = output.dims;
const d = output.data, nc = ch - 4;
const boxes = [];
for (let i = 0; i < n; i++) {
let best = 0, cls = -1;
for (let c = 0; c < nc; c++) {
const s = d[(4 + c) * n + i];
if (s > best) { best = s; cls = c; }
}
if (best < conf) continue;
const cx = d[i], cy = d[n + i], bw = d[2 * n + i], bh = d[3 * n + i];
const clamp = (v, max) => Math.max(0, Math.min(max, v));
boxes.push({
x1: clamp((cx - bw / 2 - meta.left) / meta.r, meta.w),
y1: clamp((cy - bh / 2 - meta.top) / meta.r, meta.h),
x2: clamp((cx + bw / 2 - meta.left) / meta.r, meta.w),
y2: clamp((cy + bh / 2 - meta.top) / meta.r, meta.h),
score: best, cls,
});
}
return nms(boxes, CONFIG.iou);
}
function iou(a, b) {
const ix = Math.max(0, Math.min(a.x2, b.x2) - Math.max(a.x1, b.x1));
const iy = Math.max(0, Math.min(a.y2, b.y2) - Math.max(a.y1, b.y1));
const inter = ix * iy;
const union = (a.x2 - a.x1) * (a.y2 - a.y1) + (b.x2 - b.x1) * (b.y2 - b.y1) - inter;
return union > 0 ? inter / union : 0;
}
function nms(boxes, thr) {
boxes.sort((a, b) => b.score - a.score);
const keep = [];
for (const b of boxes) {
if (keep.every((k) => k.cls !== b.cls || iou(k, b) <= thr)) keep.push(b);
if (keep.length >= 300) break;
}
return keep;
}
/* ---------- Drawing ---------- */
function draw(img, dets) {
const w = img.naturalWidth, h = img.naturalHeight;
out.width = w; out.height = h;
const ctx = out.getContext("2d");
ctx.drawImage(img, 0, 0, w, h);
const lw = Math.max(2, Math.round(Math.min(w, h) / 220));
const fs = Math.max(14, Math.round(Math.min(w, h) / 32));
ctx.font = `700 ${fs}px Archivo, "Helvetica Neue", Arial, sans-serif`;
ctx.textBaseline = "top";
for (const d of dets) {
const color = CONFIG.colors[d.cls] || "#FFFFFF";
ctx.lineWidth = lw;
ctx.strokeStyle = color;
ctx.strokeRect(d.x1, d.y1, d.x2 - d.x1, d.y2 - d.y1);
const label = `${CONFIG.classes[d.cls] ?? "class " + d.cls} ${Math.round(d.score * 100)}%`;
const pad = Math.round(fs * 0.3);
const tw = ctx.measureText(label).width + pad * 2, th = fs + pad * 2;
const ly = d.y1 - th >= 0 ? d.y1 - th : d.y1; // above the box, or inside if no room
const lx = Math.min(d.x1 - lw / 2, w - tw);
ctx.fillStyle = color;
ctx.fillRect(lx, ly, tw, th);
ctx.fillStyle = "#FFFFFF";
ctx.fillText(label, lx + pad, ly + pad);
}
}
function renderResults(dets) {
if (!dets.length) {
resultsEl.innerHTML = `<li class="empty">No tools above ${confInput.value}% confidence.</li>`;
return;
}
resultsEl.innerHTML = dets.map((d) => `
<li><span class="swatch" style="background:${CONFIG.colors[d.cls]}"></span>
<span>${CONFIG.classes[d.cls] ?? "class " + d.cls}</span>
<span class="conf">${Math.round(d.score * 100)}%</span></li>`).join("");
}
function update() {
if (!current) return;
const conf = confInput.value / 100;
const dets = decode(current.output, current.meta, conf);
draw(current.img, dets);
renderResults(dets);
if (dets.length) {
setStatus(`Found ${dets.length} ${dets.length === 1 ? "tool" : "tools"} in ${current.ms} ms.`);
} else {
setStatus(`No tools found above ${confInput.value}% confidence. Try lowering the threshold.`);
}
}
/* ---------- Running a photo ---------- */
async function runOnImage(img) {
if (busy) return;
busy = true;
try {
stage.classList.add("has-image");
$("again").hidden = false;
out.width = img.naturalWidth; out.height = img.naturalHeight;
out.getContext("2d").drawImage(img, 0, 0);
if (!session) {
setStatus("Loading the model…");
if (!(await modelReady)) return;
}
setStatus("Detecting…");
await new Promise(requestAnimationFrame); // let the status paint first
const meta = preprocess(img);
const t0 = performance.now();
const res = await session.run({ [inputName]: meta.tensor });
const ms = Math.round(performance.now() - t0);
current = { img, meta, output: res[session.outputNames[0]], ms };
update();
} catch (err) {
console.error(err);
setStatus("Detection failed on this photo. Try a different JPG or PNG.", true);
} finally {
busy = false;
}
}
function loadImage(src) {
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = () => reject(new Error("Image could not be decoded"));
img.src = src;
});
}
async function handleFile(file) {
if (!file) return;
if (!file.type.startsWith("image/")) {
setStatus("That file isn't an image. Choose a JPG, PNG or WebP photo.", true);
return;
}
try {
const img = await loadImage(URL.createObjectURL(file));
await runOnImage(img);
} catch {
setStatus("This photo couldn't be opened. Try saving it as JPG or PNG first.", true);
}
}
/* ---------- Inputs: button, drag-and-drop, paste, examples ---------- */
$("choose").addEventListener("click", () => fileInput.click());
$("again").addEventListener("click", () => fileInput.click());
fileInput.addEventListener("change", () => { handleFile(fileInput.files[0]); fileInput.value = ""; });
["dragenter", "dragover"].forEach((ev) => stage.addEventListener(ev, (e) => {
e.preventDefault(); stage.classList.add("dragging");
}));
["dragleave", "drop"].forEach((ev) => stage.addEventListener(ev, (e) => {
e.preventDefault(); stage.classList.remove("dragging");
}));
stage.addEventListener("drop", (e) => handleFile(e.dataTransfer.files[0]));
window.addEventListener("paste", (e) => {
const item = [...(e.clipboardData?.items || [])].find((i) => i.type.startsWith("image/"));
if (item) handleFile(item.getAsFile());
});
confInput.addEventListener("input", () => {
confOut.textContent = `${confInput.value}%`;
update();
});
if (CONFIG.examples.length) {
$("examples").hidden = false;
$("thumbs").innerHTML = CONFIG.examples.map((ex, i) => `
<button type="button" class="thumb" data-i="${i}">
<img src="${ex.src}" alt="" loading="lazy"><span>${ex.label || "Example " + (i + 1)}</span>
</button>`).join("");
$("thumbs").addEventListener("click", async (e) => {
const btn = e.target.closest(".thumb");
if (!btn) return;
try {
const img = await loadImage(CONFIG.examples[btn.dataset.i].src);
await runOnImage(img);
} catch {
setStatus("That example image is missing. Check the path in CONFIG.examples.", true);
}
});
}
</script>
</body>
</html>