File size: 7,919 Bytes
2184fd3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | /* Next Echo Space: pick or record a short example sound, search a longer recording, listen to the hits. */
const { $, h, fmt, css } = NLUI;
let ctx, demos, recAudio = null, recInfo = null, qAudio = null, qLabel = "", last = null, actx = null;
function parseWav(buf) {
const dv = new DataView(buf); let p = 12, sr = 16000, bits = 16, ch = 1;
while (p < dv.byteLength) {
const id = String.fromCharCode(dv.getUint8(p), dv.getUint8(p + 1), dv.getUint8(p + 2), dv.getUint8(p + 3)), sz = dv.getUint32(p + 4, true);
if (id === "fmt ") { ch = dv.getUint16(p + 10, true); sr = dv.getUint32(p + 12, true); bits = dv.getUint16(p + 22, true); }
if (id === "data") {
if (bits !== 16 || ch !== 1 || sr !== 16000) return null;
const n = sz / 2, out = new Float64Array(n); for (let i = 0; i < n; i++) out[i] = dv.getInt16(p + 8 + 2 * i, true) / 32768; return out;
}
p += 8 + sz + (sz % 2);
}
return null;
}
async function decodeAny(buf) {
const direct = parseWav(buf.slice(0));
if (direct) return direct;
actx = actx || new (window.AudioContext || window.webkitAudioContext)();
const ab = await actx.decodeAudioData(buf);
const off = new OfflineAudioContext(1, Math.ceil(ab.duration * 16000), 16000);
const src = off.createBufferSource(); src.buffer = ab; src.connect(off.destination); src.start();
return Float64Array.from((await off.startRendering()).getChannelData(0));
}
async function fetchAudio(url) { return decodeAny(await (await fetch(url)).arrayBuffer()); }
function play(x, from, to) {
actx = actx || new (window.AudioContext || window.webkitAudioContext)();
const a = Math.max(0, Math.floor(from * 16000)), b = Math.min(x.length, Math.floor(to * 16000));
const buf = actx.createBuffer(1, Math.max(1, b - a), 16000); buf.getChannelData(0).set(Float32Array.from(x.subarray(a, b)));
const s = actx.createBufferSource(); s.buffer = buf; s.connect(actx.destination); s.start();
}
async function search() {
if (!recAudio || !qAudio) return null;
$("status").textContent = "Searching…"; await new Promise(r => setTimeout(r, 10));
const t0 = performance.now();
const c = Core.similarityCurves(ctx, qAudio, recAudio);
const [nx, bl] = Core.finalScores(c.learned, c.base, ctx.score);
const dl = Core.detect(c.times, nx, ctx.thresholds.learned), db = Core.detect(c.times, bl, ctx.thresholds.baseline);
last = { c, dl, db, nx, bl };
$("status").textContent = `${c.times.length} windows scored in ${fmt((performance.now() - t0) / 1000, 2)} s · query: ${qLabel}`;
draw(); hits();
return last;
}
function draw() {
if (!last) return;
const cv = $("tl"), W = cv.clientWidth || 700, H = 260, dpr = window.devicePixelRatio || 1;
cv.width = W * dpr; cv.height = H * dpr; cv.style.height = H + "px";
const g = cv.getContext("2d"); g.scale(dpr, dpr);
const dur = recAudio.length / 16000, X = t => 40 + (W - 50) * t / dur;
g.fillStyle = css("--panel"); g.fillRect(0, 0, W, H);
// waveform
g.strokeStyle = css("--line"); g.beginPath();
const step = Math.max(1, Math.floor(recAudio.length / (W * 2)));
for (let i = 0; i < recAudio.length; i += step) { const x = X(i / 16000); let m = 0; for (let j = i; j < Math.min(i + step, recAudio.length); j++) m = Math.max(m, Math.abs(recAudio[j])); g.moveTo(x, 40 - 30 * m); g.lineTo(x, 40 + 30 * m); }
g.stroke();
if (recInfo && recInfo.events) for (const e of recInfo.events) { g.fillStyle = "rgba(176,125,18,.25)"; g.fillRect(X(e[0]), 8, X(e[1]) - X(e[0]), 64); }
const lane = (y0, sims, th, dets, col, label) => {
const lo = Math.min(...sims, th), hi = Math.max(...sims, th) + 1e-9;
const Y = v => y0 + 70 - 70 * (v - lo) / (hi - lo);
for (const d of dets) { g.fillStyle = col + "33"; g.fillRect(X(d[0]), y0, X(d[1]) - X(d[0]), 70); }
g.strokeStyle = col; g.lineWidth = 1.6; g.beginPath(); last.c.times.forEach((t, i) => (i ? g.lineTo : g.moveTo).call(g, X(t + 0.5), Y(sims[i]))); g.stroke();
g.setLineDash([4, 4]); g.strokeStyle = css("--muted"); g.beginPath(); g.moveTo(40, Y(th)); g.lineTo(W - 10, Y(th)); g.stroke(); g.setLineDash([]);
g.fillStyle = css("--ink"); g.font = "12px system-ui"; g.fillText(label, 44, y0 + 12);
};
lane(84, last.nx, ctx.thresholds.learned, last.dl, "#0b6e4f", `Next Echo: ${last.dl.length} hits`);
lane(170, last.bl, ctx.thresholds.baseline, last.db, "#2f5fb3", `log-mel similarity (baseline): ${last.db.length} hits`);
g.fillStyle = css("--muted"); g.font = "11px system-ui";
for (let t = 0; t <= dur; t += 5) g.fillText(t + "s", X(t) - 6, H - 2);
}
function hits() {
const box = $("hits"); box.innerHTML = "";
for (const [name, dets] of [["Next Echo", last.dl], ["Log-mel", last.db]]) {
const row = h("div", { class: "chips", style: "margin:4px 0" }, h("b", { class: "small", style: "width:70px" }, name));
if (!dets.length) row.append(h("span", { class: "small muted" }, "no matches above threshold"));
for (const d of dets) row.append(h("button", { class: "chip", onclick: () => play(recAudio, d[0], d[1]) }, `▶ ${fmt(d[0], 1)}–${fmt(d[1], 1)} s (${fmt(d[2], 2)})`));
box.append(row);
}
if (recInfo && recInfo.events) $("truth").textContent = `Ground truth in this demo recording: ${recInfo.events.length} target event(s) of class “${recInfo.target}” (shaded). Condition: ${recInfo.condition}.`;
else $("truth").textContent = "Your recording has no ground truth.";
}
async function setRec(i) {
recInfo = demos.recordings[i]; recAudio = await fetchAudio(recInfo.file);
const q = demos.queries.findIndex(x => x.label === recInfo.target);
if (q >= 0 && !$("keepq").checked) { $("qsel").value = q; await setQuery(q); }
return search();
}
async function setQuery(i) { const q = demos.queries[i]; qAudio = await fetchAudio(q.file); qLabel = q.label; }
async function record() {
const btn = $("mic");
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const rec = new MediaRecorder(stream), chunks = [];
rec.ondataavailable = e => chunks.push(e.data);
rec.onstop = async () => { stream.getTracks().forEach(t => t.stop()); qAudio = await decodeAny(await new Blob(chunks).arrayBuffer()); qLabel = "your microphone recording"; btn.textContent = "🎙 Record 2 s example"; search(); };
rec.start(); btn.textContent = "Recording…"; setTimeout(() => rec.stop(), 2000);
} catch (e) { $("status").textContent = "Microphone not available: " + e.message; }
}
window.__liveTest = async function () {
const i = demos.recordings.findIndex(r => r.condition === "noisy");
$("rsel").value = i; const r = await setRec(i);
return { ok: r && r.c.times.length > 50 && Array.isArray(r.dl), recording: demos.recordings[i].file, target: demos.recordings[i].target,
learned_hits: r.dl.length, baseline_hits: r.db.length, true_events: demos.recordings[i].events.length };
};
NLUI.boot(Core, async c => {
ctx = c;
demos = await (await fetch("demos.json")).json();
demos.recordings.forEach((r, i) => $("rsel").append(h("option", { value: i }, `${r.target} · ${r.condition}${r.unseen_class ? " · class unseen in training" : ""}`)));
demos.queries.forEach((q, i) => $("qsel").append(h("option", { value: i }, q.label)));
$("rsel").addEventListener("change", e => setRec(Number(e.target.value)));
$("qsel").addEventListener("change", async e => { await setQuery(Number(e.target.value)); search(); });
$("mic").addEventListener("click", record);
$("qfile").addEventListener("change", async e => { qAudio = await decodeAny(await e.target.files[0].arrayBuffer()); qLabel = e.target.files[0].name; search(); });
$("rfile").addEventListener("change", async e => { recAudio = await decodeAny(await e.target.files[0].arrayBuffer()); recInfo = null; search(); });
$("playq").addEventListener("click", () => qAudio && play(qAudio, 0, qAudio.length / 16000));
window.addEventListener("resize", draw);
await setRec(0);
});
|