crescent-moon / camera.html
jfrick's picture
Update the demo from the repository
5b0983f verified
Raw History Blame Contribute Delete
21.9 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Live camera detection</title>
<style>
:root { --ink: #1f1f1e; --muted: #6b6a63; --line: #e4e3dd; --accent: #2a78d6; --surface: #fcfcfb;
--field: #ffffff; --idle: #b9b8b1; color-scheme: light; }
:root[data-theme=dark] { --ink: #f3f4f6; --muted: #9ca3af; --line: #374151; --accent: #3b8ae6;
--surface: #0b0f19; --field: #1f2937; --idle: #4b5563; color-scheme: dark; }
body { margin: 0; font: 14px/1.4 system-ui, sans-serif; color: var(--ink); background: var(--surface); }
.controls { display: flex; flex-wrap: wrap; gap: 12px; align-items: end; margin-bottom: 10px; }
label { display: flex; flex-direction: column; gap: 4px; color: var(--muted); font-size: 12px; }
[hidden] { display: none !important; }
label.inline { flex-direction: row; align-items: center; color: var(--ink); font-size: 14px; }
input[type=text], select { font: inherit; padding: 6px 8px; border: 1px solid var(--line); border-radius: 6px; min-width: 220px;
color: var(--ink); background: var(--field); }
button { font: inherit; padding: 7px 18px; border: 0; border-radius: 6px; background: var(--accent); color: white; cursor: pointer; }
button.stop { background: #e34948; }
button:disabled { background: var(--idle); cursor: default; }
progress { width: 240px; height: 14px; accent-color: var(--accent); }
#downloadText { color: var(--muted); align-self: center; }
.stage { position: relative; display: inline-block; background: #000; border-radius: 6px; overflow: hidden; line-height: 0; }
video { display: block; max-width: 100%; max-height: 72vh; width: auto; height: auto; min-width: 320px; min-height: 240px; }
canvas { position: absolute; inset: 0; width: 100%; height: 100%; pointer-events: none; }
/* Shown over the picture while there is nothing representative to show yet. */
#notice { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center;
padding: 16px; text-align: center; line-height: 1.4; font-size: 15px; color: white;
background: rgba(0, 0, 0, 0.55); pointer-events: none; }
#status { margin-top: 8px; color: var(--muted); min-height: 1.4em; }
details { margin-top: 8px; color: var(--muted); }
details pre { margin: 6px 0 0; font: 12px/1.4 ui-monospace, monospace; white-space: pre-wrap; color: var(--ink); }
</style>
</head>
<body>
<div class="controls">
<label>Categories (comma-separated COCO names; empty means all 80)
<input id="categories" type="text" value="">
</label>
<label>Source <select id="camera"><option value="">Default camera</option>
<option value="file">A video file…</option></select></label>
<label id="fileField" hidden>Video file <input id="file" type="file" accept="video/*"></label>
<label>Model <select id="model"></select></label>
<label class="inline"><input id="track" type="checkbox" checked>&nbsp;Track across frames</label>
<button id="toggle">Start</button>
</div>
<div class="controls" id="deviceStep" hidden>
<progress id="progress" value="0" max="1" hidden></progress>
<span id="downloadText"></span>
</div>
<div class="stage"><video id="video" muted playsinline autoplay></video><canvas id="overlay"></canvas>
<div id="notice" hidden></div></div>
<div id="status">The camera or a video file plays live; boxes from the last detected frame are drawn over it.</div>
<details id="diagnostics" hidden><summary>Timings on this device</summary><pre id="timings"></pre></details>
<script>
const $ = (id) => document.getElementById(id);
// The page follows the demo's light or dark theme: Gradio marks its body with the class
// `dark`; opened on its own, the page follows the system setting.
(() => {
const system = matchMedia("(prefers-color-scheme: dark)");
let host = null;
try { if (parent !== window) host = parent.document.body; } catch { /* another origin */ }
const apply = () => {
const dark = host ? host.classList.contains("dark") : system.matches;
document.documentElement.dataset.theme = dark ? "dark" : "light";
// Inside the demo the page takes the tab's own background.
document.body.style.background = host ? "transparent" : "";
};
apply();
system.addEventListener("change", apply);
if (host) new MutationObserver(apply).observe(host, { attributes: true, attributeFilter: ["class"] });
})();
const video = $("video"), overlay = $("overlay"), status = $("status"), toggle = $("toggle");
const grab = document.createElement("canvas");
const grabCrop = document.createElement("canvas");
let config = null, running = false, stream = null, boxes = [], scale = 1, session = null;
// Counts starts: a detection loop ends when a later start or a stop has replaced it.
let run = 0;
// True while a start is loading the detector in this browser, before the source opens.
let loading = false;
// The detector in this browser, once loaded: { model, patches, side, candidates, select }.
let local = null, loadingLocal = null;
// The Model list's entry for the detector in this browser, and whether it is chosen.
const DEVICE = "device";
const onDevice = () => $("model").value === DEVICE;
const PALETTE = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948"];
fetch("camera/config", { cache: "no-store" }).then((r) => r.json()).then((c) => {
config = c;
for (const name of c.models) $("model").add(new Option(name, name, false, name === c.default));
// The browser's own detector is one more model, offered when the server has an export
// for it; it needs WebGPU.
if (!c.device) return;
const entry = new Option("On this device: slots, rank 64, 4-bit (WebGPU)", DEVICE);
if (!navigator.gpu) {
entry.disabled = true;
entry.text = "On this device: this browser has no WebGPU";
}
$("model").add(entry);
// Where the browser can run the detector itself, that is the first choice.
if (navigator.gpu) { $("model").value = DEVICE; showDeviceStep(); }
});
// One button starts the camera. For the detector in this browser it first loads the
// detector when it is not loaded yet: a download the first time (the browser keeps it),
// then the preparation on the GPU. The row under the controls shows that step.
const notice = (text) => { $("notice").hidden = !text; $("notice").textContent = text || ""; };
async function showDeviceStep() {
const on = onDevice();
$("deviceStep").hidden = !on; $("track").disabled = on;
if (running) return;
toggle.textContent = "Start";
if (!on || local || loadingLocal) return;
const kept = "caches" in window && (await (await caches.open("crescent-moon-weights")).keys()).length > 0;
if (running || !onDevice() || local) return;
toggle.textContent = kept ? "Load the detector and start" : "Download the detector (1.2 GB) and start";
$("downloadText").textContent = kept
? "The detector is already downloaded; it is prepared on the GPU when you start."
: "One download; the browser keeps it for next time.";
}
$("model").addEventListener("change", () => {
boxes = [];
if (onDevice() && running && !local) stop("Stopped; start again to load the detector on this device.");
if (!onDevice() && loading) stop("Stopped.");
showDeviceStep();
});
// Downloads the weights once (the browser keeps them) and builds the WebGPU pipelines.
// A second call while loading, or after it, returns the same detector.
function loadLocal() {
loadingLocal ??= (async () => {
const megabytes = (bytes) => (bytes / 2 ** 20).toFixed(0);
const step = (text) => { $("downloadText").textContent = text; if (running) notice(text); };
$("progress").hidden = false; $("progress").value = 0;
step("Loading the detector");
const [runtime, selection] = await Promise.all(
["model.js", "select.js"].map((name) => import(new URL(config.device.runtime + name, location.href).href)));
const model = await runtime.loadModel(new URL(config.device.weights, location.href), (done, total) => {
$("progress").value = done / total;
step(`Loading the detector: ${megabytes(done)} of ${megabytes(total)} MB`);
}, (text) => { $("progress").hidden = true; step(text); });
model.device.lost.then((info) => {
local = null; loadingLocal = null; $("progress").hidden = true; $("downloadText").textContent = "";
// The choice of model stays: the next start loads the detector again.
const message = `The browser's GPU device was lost (${info.message || info.reason}); start again to reload the detector.`;
if (onDevice() && running) stop(message); else if (onDevice()) { showDeviceStep(); status.textContent = message; }
});
// `warm` turns true after the first frame, which compiles the kernels.
model.device.addEventListener("uncapturederror", (event) => {
status.textContent = `GPU error on this device: ${event.error.message}`;
});
local = { model, patches: runtime.patches, side: runtime.CROP, warm: false, ...selection };
$("progress").hidden = true;
const { outs, rows, queries } = model.tuning;
$("downloadText").textContent = `Detector loaded. Fastest kernels on this GPU: products ${outs} x ${rows}, attention ${queries}`;
showTimings();
return local;
})().catch((error) => {
loadingLocal = null; $("progress").hidden = true; $("downloadText").textContent = "";
throw error;
});
return loadingLocal;
}
// What the detector in this browser measured: the GPU, each kernel choice timed while
// loading, and the stages of the latest frames.
function showTimings(seconds) {
const model = local.model, info = model.device.adapterInfo || {};
const ms = (value) => `${(value * 1000).toFixed(1)} ms`;
const { outs, rows, queries, seconds: timed = {}, rounds, error } = model.tuning;
if (seconds) local.frames = [...(local.frames || []), seconds].slice(-5);
const lines = [
`GPU: ${[info.vendor, info.architecture, info.device, info.description].filter(Boolean).join(", ") || "not named"}`,
`Kernels: products ${outs} x ${rows}, attention ${queries}`
+ (error ? ` (default; timing failed: ${error})` : rounds ? ` (fastest of ${rounds} rounds)` : " (default, not timed)"),
...Object.entries(timed).map(([name, value]) => ` ${name}: ${ms(value)}`),
...(local.frames || []).map((frame) => "Frame: " + Object.entries(frame).map(([name, value]) => `${name} ${ms(value)}`).join(", ")),
];
$("timings").textContent = lines.join("\n");
$("diagnostics").hidden = false;
}
function colourOf(label) {
let hash = 0;
for (const letter of label) hash = (hash * 31 + letter.codePointAt(0)) >>> 0;
return PALETTE[hash % PALETTE.length];
}
// One frame through the detector in this browser: the whole frame resized to the
// model's square crop, one forward pass, selection; boxes in video pixels.
async function detectLocal(w, h, mine) {
const side = local.side, settings = local.model.manifest.selection;
grabCrop.width = grabCrop.height = side;
const pen = grabCrop.getContext("2d", { willReadFrequently: true });
pen.drawImage(video, 0, 0, side, side);
const started = performance.now();
const rows = local.patches(pen.getImageData(0, 0, side, side).data, 4);
const { slots, seconds } = await local.model.run(rows, { timed: true });
const asked = $("categories").value.split(",").map((p) => p.trim().toLowerCase()).filter(Boolean);
const found = local.select(local.candidates(slots, local.model.manifest.vocabulary, settings.box_power), w, h,
{ threshold: settings.threshold, duplicateIou: settings.duplicate_iou, labels: asked.length ? asked : null });
const total = (performance.now() - started) / 1000;
if (mine !== run) return;
boxes = found.map((b) => ({ ...b, colour: colourOf(b.label) })); scale = 1;
showTimings(seconds);
if (!local.warm) {
local.warm = true;
status.textContent = `On this device: the first frame took ${total.toFixed(1)} s, most of it preparation; the next ones show the speed.`;
return;
}
const stage = (name) => `${name} ${seconds[name].toFixed(3)} s`;
status.textContent = `On this device: ${boxes.length} boxes; ${total.toFixed(3)} s per frame (`
+ `${stage("vision")}, ${stage("projection")}, ${stage("decoder")}, ${stage("readout")})`;
}
// Camera names are only shown once the page may use a camera, so the list is filled
// again after each start and whenever a camera is plugged in or out.
async function listCameras() {
const devices = (await navigator.mediaDevices.enumerateDevices()).filter((d) => d.kind === "videoinput");
const picker = $("camera"), chosen = picker.value || currentCamera();
picker.replaceChildren(...devices.map((d, i) =>
new Option(d.label || `Camera ${i + 1}`, d.deviceId, false, d.deviceId === chosen)));
if (!devices.length) picker.add(new Option("No camera found", ""));
// A video file from this device is one more source; it plays in a loop.
picker.add(new Option(videoFile ? `Video file: ${videoFile.name}` : "A video file…", FILE, false, chosen === FILE));
}
// The chosen video file and the URL it plays from.
const FILE = "file";
let videoFile = null, videoUrl = null, picked = false;
$("file").addEventListener("change", async () => {
const [file] = $("file").files;
if (!file) return;
videoFile = file; picked = true;
await listCameras();
$("camera").value = FILE;
$("camera").dispatchEvent(new Event("change"));
});
function currentCamera() {
const track = stream && stream.getVideoTracks()[0];
return track ? track.getSettings().deviceId || "" : "";
}
async function openCamera() {
const id = $("camera").value;
if (id === FILE) {
if (!videoFile) throw new Error("choose a video file first");
if (stream) stream.getTracks().forEach((t) => t.stop());
stream = null; video.srcObject = null;
if (videoUrl) URL.revokeObjectURL(videoUrl);
videoUrl = URL.createObjectURL(videoFile);
video.src = videoUrl; video.loop = true;
await video.play();
return;
}
video.removeAttribute("src"); video.loop = false;
const video_ = { width: { ideal: 1280 } };
if (id) video_.deviceId = { exact: id };
const next = await navigator.mediaDevices.getUserMedia({ video: video_, audio: false });
if (stream) stream.getTracks().forEach((t) => t.stop());
stream = next;
video.srcObject = stream;
await video.play();
}
navigator.mediaDevices.addEventListener("devicechange", listCameras);
listCameras();
$("camera").addEventListener("change", async () => {
// Choosing the file entry opens the file picker; the change that follows a pick does not.
if ($("camera").value === FILE && !picked) {
$("fileField").hidden = false;
$("file").click();
if (!videoFile) return;
}
picked = false;
$("fileField").hidden = $("camera").value !== FILE;
if (!running) return;
try {
await openCamera();
// The old camera's boxes and tracks do not belong to the new picture.
boxes = []; session = crypto.randomUUID();
} catch (error) {
status.textContent = `Could not open that camera: ${error.message}`;
}
});
function draw() {
const w = video.videoWidth, h = video.videoHeight;
if (w && (overlay.width !== w || overlay.height !== h)) { overlay.width = w; overlay.height = h; }
const pen = overlay.getContext("2d");
pen.clearRect(0, 0, overlay.width, overlay.height);
// Sizes in video pixels, chosen so they read the same at any displayed size.
const shown = overlay.getBoundingClientRect().width || w;
const unit = w / shown;
const line = Math.max(1, Math.round(2 * unit));
pen.font = `${Math.round(12 * unit)}px system-ui, sans-serif`;
pen.textBaseline = "top";
for (const b of boxes) {
const [x0, y0, x1, y1] = b.box.map((v) => v * scale);
pen.strokeStyle = b.colour; pen.lineWidth = line;
pen.strokeRect(x0, y0, x1 - x0, y1 - y0);
const text = b.score === null ? b.label : `${b.label} ${b.score.toFixed(2)}`;
const width = pen.measureText(text).width, height = Math.round(15 * unit);
pen.fillStyle = b.colour; pen.fillRect(x0, y0, width + 6 * unit, height);
pen.fillStyle = "white"; pen.fillText(text, x0 + 3 * unit, y0 + 1.5 * unit);
}
if (running) requestAnimationFrame(draw);
}
// One call of the Space's `frame` API: Gradio answers a POST with an event id and
// streams the result of that event.
// On huggingface.co the Space's GPU time is counted against the visitor: the page that
// embeds the Space hands out the headers that identify them, as it does to Gradio.
let gpuHeaders = null;
function visitorHeaders() {
gpuHeaders ??= new Promise((resolve) => {
if (top === window) { resolve({}); return; }
// The answer comes back on the channel's other port, as for Gradio's own client.
const channel = new MessageChannel();
const done = (headers) => { channel.port1.close(); resolve(headers && typeof headers === "object" ? headers : {}); };
channel.port1.onmessage = (event) => done(event.data);
try { top.postMessage("zerogpu-headers", "https://huggingface.co", [channel.port2]); } catch { done({}); }
setTimeout(() => done({}), 3000);
});
return gpuHeaders;
}
async function serverFrame(data) {
const posted = await fetch("gradio_api/call/frame", {
method: "POST", headers: { "Content-Type": "application/json", ...(await visitorHeaders()) },
body: JSON.stringify({ data }),
});
if (!posted.ok) throw new Error(`HTTP ${posted.status}`);
const { event_id: id } = await posted.json();
const events = await (await fetch(`gradio_api/call/frame/${id}`)).text();
const last = events.trim().split("\n\n").pop();
const kind = /^event: (.*)$/m.exec(last)?.[1], body = /^data: (.*)$/m.exec(last)?.[1];
if (kind !== "complete") {
// Gradio hides the reason; on ZeroGPU it is nearly always the visitor's GPU allowance.
throw new Error(JSON.parse(body || "null")?.error
|| "no GPU time for this visitor (sign in on huggingface.co, or use the model that runs on this device)");
}
return JSON.parse(body)[0];
}
async function detectLoop(mine) {
const started = performance.now();
let first = true;
while (mine === run) {
if (performance.now() - started > config.seconds * 1000) {
stop(`Stopped after ${config.seconds / 60} minutes; press Start to go on.`);
return;
}
const w = video.videoWidth, h = video.videoHeight;
if (!w) { await new Promise((r) => setTimeout(r, 100)); continue; }
if (onDevice()) {
if (!local.warm) {
notice("Preparing the first frame. It compiles the detector's GPU kernels, so it takes "
+ "much longer than the frames that follow.");
}
try {
await detectLocal(w, h, mine);
} catch (error) {
if (mine === run) stop(`The detector failed on this device: ${error.message}`);
return;
}
if (mine === run) notice("");
// Let the page paint and take input between frames.
await new Promise((r) => setTimeout(r, 0));
continue;
}
const s = Math.min(1, config.side / Math.max(w, h));
grab.width = Math.round(w * s); grab.height = Math.round(h * s);
grab.getContext("2d").drawImage(video, 0, 0, grab.width, grab.height);
const jpeg = grab.toDataURL("image/jpeg", 0.85).split(",")[1];
const t0 = performance.now();
let result;
try {
result = await serverFrame([jpeg, $("model").value, $("categories").value, $("track").checked, session]);
} catch (error) {
if (mine === run) stop(`The Space's GPU did not answer: ${error.message}`);
return;
}
if (result.stop) { if (mine === run) stop(result.stop); return; }
if (mine !== run) return;
notice("");
boxes = result.boxes; scale = w / grab.width;
const trip = (performance.now() - t0) / 1000;
status.textContent = `${$("model").value}: ${boxes.length} boxes; detector ${result.seconds.toFixed(3)} s, `
+ `server ${result.server_seconds.toFixed(3)} s, with the network ${trip.toFixed(3)} s`
+ (first ? " (first frame includes the GPU start)" : "");
first = false;
}
}
async function start() {
const mine = ++run;
running = true; boxes = [];
toggle.textContent = "Stop"; toggle.classList.add("stop");
// The detector in this browser is loaded before the camera opens; loaded once, it
// stays for later starts.
if (onDevice() && !local) {
status.textContent = "Loading the detector on this device; the picture starts when it is ready.";
loading = true;
try {
await loadLocal();
} catch (error) {
loading = false;
if (mine === run) stop(`Could not run on this device: ${error.message}`);
return;
}
loading = false;
if (mine !== run) return;
}
try {
await openCamera();
} catch (error) {
if (mine === run) stop(`Could not open the source: ${error.message}`);
return;
}
if (mine !== run) { stop(); return; }
await listCameras();
session = crypto.randomUUID();
if (onDevice()) {
status.textContent = "Running on this device.";
} else {
notice("Waiting for the Space's GPU. The first boxes can take a while; that wait is "
+ "the start-up, not the detector's speed.");
status.textContent = "Starting the GPU: the first boxes can take about a minute; the picture keeps playing.";
}
requestAnimationFrame(draw);
detectLoop(mine);
}
function stop(message) {
run++; running = false; loading = false; boxes = [];
overlay.getContext("2d").clearRect(0, 0, overlay.width, overlay.height);
if (stream) stream.getTracks().forEach((t) => t.stop());
stream = null; video.srcObject = null;
video.pause(); video.removeAttribute("src"); video.load();
toggle.classList.remove("stop"); notice("");
showDeviceStep();
if (message) status.textContent = message;
}
toggle.addEventListener("click", () => (running ? stop("Stopped.") : start()));
</script>
</body>
</html>