Next-Echo / app.js
Lamapi's picture
Publish Next-Echo space
2184fd3 verified
Raw History Blame Contribute Delete
7.92 kB
/* Next Echo Space: pick or record a short example sound, search a longer recording, listen to the hits. */
const { $, h, fmt, css } = NLUI;
let ctx, demos, recAudio = null, recInfo = null, qAudio = null, qLabel = "", last = null, actx = null;
function parseWav(buf) {
const dv = new DataView(buf); let p = 12, sr = 16000, bits = 16, ch = 1;
while (p < dv.byteLength) {
const id = String.fromCharCode(dv.getUint8(p), dv.getUint8(p + 1), dv.getUint8(p + 2), dv.getUint8(p + 3)), sz = dv.getUint32(p + 4, true);
if (id === "fmt ") { ch = dv.getUint16(p + 10, true); sr = dv.getUint32(p + 12, true); bits = dv.getUint16(p + 22, true); }
if (id === "data") {
if (bits !== 16 || ch !== 1 || sr !== 16000) return null;
const n = sz / 2, out = new Float64Array(n); for (let i = 0; i < n; i++) out[i] = dv.getInt16(p + 8 + 2 * i, true) / 32768; return out;
}
p += 8 + sz + (sz % 2);
}
return null;
}
async function decodeAny(buf) {
const direct = parseWav(buf.slice(0));
if (direct) return direct;
actx = actx || new (window.AudioContext || window.webkitAudioContext)();
const ab = await actx.decodeAudioData(buf);
const off = new OfflineAudioContext(1, Math.ceil(ab.duration * 16000), 16000);
const src = off.createBufferSource(); src.buffer = ab; src.connect(off.destination); src.start();
return Float64Array.from((await off.startRendering()).getChannelData(0));
}
async function fetchAudio(url) { return decodeAny(await (await fetch(url)).arrayBuffer()); }
function play(x, from, to) {
actx = actx || new (window.AudioContext || window.webkitAudioContext)();
const a = Math.max(0, Math.floor(from * 16000)), b = Math.min(x.length, Math.floor(to * 16000));
const buf = actx.createBuffer(1, Math.max(1, b - a), 16000); buf.getChannelData(0).set(Float32Array.from(x.subarray(a, b)));
const s = actx.createBufferSource(); s.buffer = buf; s.connect(actx.destination); s.start();
}
async function search() {
if (!recAudio || !qAudio) return null;
$("status").textContent = "Searching…"; await new Promise(r => setTimeout(r, 10));
const t0 = performance.now();
const c = Core.similarityCurves(ctx, qAudio, recAudio);
const [nx, bl] = Core.finalScores(c.learned, c.base, ctx.score);
const dl = Core.detect(c.times, nx, ctx.thresholds.learned), db = Core.detect(c.times, bl, ctx.thresholds.baseline);
last = { c, dl, db, nx, bl };
$("status").textContent = `${c.times.length} windows scored in ${fmt((performance.now() - t0) / 1000, 2)} s · query: ${qLabel}`;
draw(); hits();
return last;
}
function draw() {
if (!last) return;
const cv = $("tl"), W = cv.clientWidth || 700, H = 260, dpr = window.devicePixelRatio || 1;
cv.width = W * dpr; cv.height = H * dpr; cv.style.height = H + "px";
const g = cv.getContext("2d"); g.scale(dpr, dpr);
const dur = recAudio.length / 16000, X = t => 40 + (W - 50) * t / dur;
g.fillStyle = css("--panel"); g.fillRect(0, 0, W, H);
// waveform
g.strokeStyle = css("--line"); g.beginPath();
const step = Math.max(1, Math.floor(recAudio.length / (W * 2)));
for (let i = 0; i < recAudio.length; i += step) { const x = X(i / 16000); let m = 0; for (let j = i; j < Math.min(i + step, recAudio.length); j++) m = Math.max(m, Math.abs(recAudio[j])); g.moveTo(x, 40 - 30 * m); g.lineTo(x, 40 + 30 * m); }
g.stroke();
if (recInfo && recInfo.events) for (const e of recInfo.events) { g.fillStyle = "rgba(176,125,18,.25)"; g.fillRect(X(e[0]), 8, X(e[1]) - X(e[0]), 64); }
const lane = (y0, sims, th, dets, col, label) => {
const lo = Math.min(...sims, th), hi = Math.max(...sims, th) + 1e-9;
const Y = v => y0 + 70 - 70 * (v - lo) / (hi - lo);
for (const d of dets) { g.fillStyle = col + "33"; g.fillRect(X(d[0]), y0, X(d[1]) - X(d[0]), 70); }
g.strokeStyle = col; g.lineWidth = 1.6; g.beginPath(); last.c.times.forEach((t, i) => (i ? g.lineTo : g.moveTo).call(g, X(t + 0.5), Y(sims[i]))); g.stroke();
g.setLineDash([4, 4]); g.strokeStyle = css("--muted"); g.beginPath(); g.moveTo(40, Y(th)); g.lineTo(W - 10, Y(th)); g.stroke(); g.setLineDash([]);
g.fillStyle = css("--ink"); g.font = "12px system-ui"; g.fillText(label, 44, y0 + 12);
};
lane(84, last.nx, ctx.thresholds.learned, last.dl, "#0b6e4f", `Next Echo: ${last.dl.length} hits`);
lane(170, last.bl, ctx.thresholds.baseline, last.db, "#2f5fb3", `log-mel similarity (baseline): ${last.db.length} hits`);
g.fillStyle = css("--muted"); g.font = "11px system-ui";
for (let t = 0; t <= dur; t += 5) g.fillText(t + "s", X(t) - 6, H - 2);
}
function hits() {
const box = $("hits"); box.innerHTML = "";
for (const [name, dets] of [["Next Echo", last.dl], ["Log-mel", last.db]]) {
const row = h("div", { class: "chips", style: "margin:4px 0" }, h("b", { class: "small", style: "width:70px" }, name));
if (!dets.length) row.append(h("span", { class: "small muted" }, "no matches above threshold"));
for (const d of dets) row.append(h("button", { class: "chip", onclick: () => play(recAudio, d[0], d[1]) }, `▶ ${fmt(d[0], 1)}–${fmt(d[1], 1)} s (${fmt(d[2], 2)})`));
box.append(row);
}
if (recInfo && recInfo.events) $("truth").textContent = `Ground truth in this demo recording: ${recInfo.events.length} target event(s) of class “${recInfo.target}” (shaded). Condition: ${recInfo.condition}.`;
else $("truth").textContent = "Your recording has no ground truth.";
}
async function setRec(i) {
recInfo = demos.recordings[i]; recAudio = await fetchAudio(recInfo.file);
const q = demos.queries.findIndex(x => x.label === recInfo.target);
if (q >= 0 && !$("keepq").checked) { $("qsel").value = q; await setQuery(q); }
return search();
}
async function setQuery(i) { const q = demos.queries[i]; qAudio = await fetchAudio(q.file); qLabel = q.label; }
async function record() {
const btn = $("mic");
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const rec = new MediaRecorder(stream), chunks = [];
rec.ondataavailable = e => chunks.push(e.data);
rec.onstop = async () => { stream.getTracks().forEach(t => t.stop()); qAudio = await decodeAny(await new Blob(chunks).arrayBuffer()); qLabel = "your microphone recording"; btn.textContent = "🎙 Record 2 s example"; search(); };
rec.start(); btn.textContent = "Recording…"; setTimeout(() => rec.stop(), 2000);
} catch (e) { $("status").textContent = "Microphone not available: " + e.message; }
}
window.__liveTest = async function () {
const i = demos.recordings.findIndex(r => r.condition === "noisy");
$("rsel").value = i; const r = await setRec(i);
return { ok: r && r.c.times.length > 50 && Array.isArray(r.dl), recording: demos.recordings[i].file, target: demos.recordings[i].target,
learned_hits: r.dl.length, baseline_hits: r.db.length, true_events: demos.recordings[i].events.length };
};
NLUI.boot(Core, async c => {
ctx = c;
demos = await (await fetch("demos.json")).json();
demos.recordings.forEach((r, i) => $("rsel").append(h("option", { value: i }, `${r.target} · ${r.condition}${r.unseen_class ? " · class unseen in training" : ""}`)));
demos.queries.forEach((q, i) => $("qsel").append(h("option", { value: i }, q.label)));
$("rsel").addEventListener("change", e => setRec(Number(e.target.value)));
$("qsel").addEventListener("change", async e => { await setQuery(Number(e.target.value)); search(); });
$("mic").addEventListener("click", record);
$("qfile").addEventListener("change", async e => { qAudio = await decodeAny(await e.target.files[0].arrayBuffer()); qLabel = e.target.files[0].name; search(); });
$("rfile").addEventListener("change", async e => { recAudio = await decodeAny(await e.target.files[0].arrayBuffer()); recInfo = null; search(); });
$("playq").addEventListener("click", () => qAudio && play(qAudio, 0, qAudio.length / 16000));
window.addEventListener("resize", draw);
await setRec(0);
});