"""Next Echo: find where a short example sound occurs inside a longer recording. Shared by the notebook, inference.py and (ported) web/core.js. 16 kHz mono audio. * logmel: 512-point STFT (periodic Hann, hop 256) -> 64 mel bands -> log * windows of 64 frames (~1.04 s) every 16 frames (~0.256 s) * learned: CNN embedding (nlrt) of each window, cosine to the query embedding * baseline: cosine between per-band mean/std statistics of log-mel windows * robust_z: per-recording normalisation (median / MAD) so thresholds adapt to each recording's background * final_scores: Next Echo score = alpha * z(learned) + (1 - alpha) * z(log-mel); baseline = log-mel (raw or z), chosen on validation * detect: threshold the score curve, merge consecutive windows into intervals """ import math import numpy as np from nlrt import forward SR, N_FFT, HOP, N_MELS, WIN, STEP = 16000, 512, 256, 64, 64, 16 WIN_SEC = (WIN - 1) * HOP / SR + N_FFT / SR STEP_SEC = STEP * HOP / SR def mel_filterbank(n_mels=N_MELS, n_fft=N_FFT, sr=SR, fmin=50.0, fmax=8000.0): hz2mel = lambda f: 2595.0 * math.log10(1.0 + f / 700.0) # noqa: E731 mel2hz = lambda m: 700.0 * (10 ** (m / 2595.0) - 1.0) # noqa: E731 mels = np.linspace(hz2mel(fmin), hz2mel(fmax), n_mels + 2) hz = np.array([mel2hz(m) for m in mels]) bins = np.arange(n_fft // 2 + 1) * sr / n_fft fb = np.zeros((n_mels, len(bins))) for i in range(n_mels): lo, c, hi = hz[i], hz[i + 1], hz[i + 2] fb[i] = np.maximum(0, np.minimum((bins - lo) / (c - lo), (hi - bins) / (hi - c))) return fb def logmel(x, fb): x = np.asarray(x, dtype=np.float64) if len(x) < N_FFT: x = np.concatenate([x, np.zeros(N_FFT - len(x))]) n = 1 + (len(x) - N_FFT) // HOP w = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(N_FFT) / N_FFT) idx = np.arange(N_FFT)[None, :] + HOP * np.arange(n)[:, None] spec = np.fft.rfft(x[idx] * w, axis=1) power = spec.real ** 2 + spec.imag ** 2 return np.log(fb @ power.T + 1e-6) # (n_mels, frames) def pad_frames(lm, n): if lm.shape[1] >= n: return lm return np.concatenate([lm, np.full((lm.shape[0], n - lm.shape[1]), lm.min())], axis=1) def window_starts(n_frames): return list(range(0, max(n_frames - WIN, 0) + 1, STEP)) def embed_windows(layers, lm, mu, sd, starts): X = np.stack([(lm[:, s:s + WIN] - mu) / sd for s in starts])[:, None] E = forward(layers, X) return E / np.maximum(np.linalg.norm(E, axis=1, keepdims=True), 1e-12) def stats_windows(lm, mu, sd, starts): out = [] for s in starts: z = (lm[:, s:s + WIN] - mu) / sd v = np.concatenate([z.mean(axis=1), z.std(axis=1)]) v = v - v.mean() out.append(v / max(float(np.linalg.norm(v)), 1e-12)) return np.stack(out) def query_starts(lm): """Windows of the query that carry the sound: mean log energy within 1.5 of the loudest window.""" lm = pad_frames(lm, WIN) starts = list(range(0, lm.shape[1] - WIN + 1, 8)) e = [float(lm[:, s:s + WIN].mean()) for s in starts] top = max(e) return lm, [s for s, v in zip(starts, e) if v >= top - 1.5] def query_vectors(model, q_audio): fb, mu, sd = model["fb"], model["mu"], model["sd"] lm, starts = query_starts(logmel(q_audio, fb)) qe = embed_windows(model["layers"], lm, mu, sd, starts).mean(axis=0) qs = stats_windows(lm, mu, sd, starts).mean(axis=0) return qe / max(float(np.linalg.norm(qe)), 1e-12), qs / max(float(np.linalg.norm(qs)), 1e-12) def similarity_curves(model, q_audio, rec_audio): qe, qs = query_vectors(model, q_audio) lm = pad_frames(logmel(rec_audio, model["fb"]), WIN) starts = window_starts(lm.shape[1]) learned = embed_windows(model["layers"], lm, model["mu"], model["sd"], starts) @ qe base = stats_windows(lm, model["mu"], model["sd"], starts) @ qs times = [s * HOP / SR for s in starts] return times, learned, base def robust_z(c): c = np.asarray(c, dtype=np.float64) med = float(np.median(c)) mad = float(np.median(np.abs(c - med))) + 1e-6 return (c - med) / (1.4826 * mad) def final_scores(learned, base, cfg): """cfg: {"alpha": float, "baseline_norm": "raw" | "z"} -> (next_echo_curve, baseline_curve).""" a = cfg["alpha"] nxt = a * robust_z(learned) + (1.0 - a) * robust_z(base) bl = robust_z(base) if cfg["baseline_norm"] == "z" else np.asarray(base, dtype=np.float64) return nxt, bl def detect(times, sims, threshold): out, cur = [], None for t, s in zip(times, sims): if s >= threshold: if cur is None: cur = [t, t + WIN_SEC, float(s)] else: cur[1] = t + WIN_SEC cur[2] = max(cur[2], float(s)) elif cur is not None: out.append(cur); cur = None if cur is not None: out.append(cur) return out def match_events(dets, gts, tol=0.5): used, tp = set(), 0 for g0, g1 in gts: for j, d in enumerate(dets): if j not in used and d[0] < g1 + tol and d[1] > g0 - tol: used.add(j); tp += 1 break return tp, len(dets) - len(used), len(gts) - tp def load_model(spec): t = spec["tensors"] return {"layers": spec["nets"]["embed"], "fb": t["mel_fb"], "mu": float(spec["meta"]["mu"]), "sd": float(spec["meta"]["sd"]), "thresholds": spec["meta"]["thresholds"], "score": spec["meta"]["score"]}