File size: 5,453 Bytes
e187f2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | """Next Echo: find where a short example sound occurs inside a longer recording.
Shared by the notebook, inference.py and (ported) web/core.js. 16 kHz mono audio.
* logmel: 512-point STFT (periodic Hann, hop 256) -> 64 mel bands -> log
* windows of 64 frames (~1.04 s) every 16 frames (~0.256 s)
* learned: CNN embedding (nlrt) of each window, cosine to the query embedding
* baseline: cosine between per-band mean/std statistics of log-mel windows
* robust_z: per-recording normalisation (median / MAD) so thresholds adapt to each recording's background
* final_scores: Next Echo score = alpha * z(learned) + (1 - alpha) * z(log-mel); baseline = log-mel (raw or z), chosen on validation
* detect: threshold the score curve, merge consecutive windows into intervals
"""
import math
import numpy as np
from nlrt import forward
SR, N_FFT, HOP, N_MELS, WIN, STEP = 16000, 512, 256, 64, 64, 16
WIN_SEC = (WIN - 1) * HOP / SR + N_FFT / SR
STEP_SEC = STEP * HOP / SR
def mel_filterbank(n_mels=N_MELS, n_fft=N_FFT, sr=SR, fmin=50.0, fmax=8000.0):
hz2mel = lambda f: 2595.0 * math.log10(1.0 + f / 700.0) # noqa: E731
mel2hz = lambda m: 700.0 * (10 ** (m / 2595.0) - 1.0) # noqa: E731
mels = np.linspace(hz2mel(fmin), hz2mel(fmax), n_mels + 2)
hz = np.array([mel2hz(m) for m in mels])
bins = np.arange(n_fft // 2 + 1) * sr / n_fft
fb = np.zeros((n_mels, len(bins)))
for i in range(n_mels):
lo, c, hi = hz[i], hz[i + 1], hz[i + 2]
fb[i] = np.maximum(0, np.minimum((bins - lo) / (c - lo), (hi - bins) / (hi - c)))
return fb
def logmel(x, fb):
x = np.asarray(x, dtype=np.float64)
if len(x) < N_FFT:
x = np.concatenate([x, np.zeros(N_FFT - len(x))])
n = 1 + (len(x) - N_FFT) // HOP
w = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(N_FFT) / N_FFT)
idx = np.arange(N_FFT)[None, :] + HOP * np.arange(n)[:, None]
spec = np.fft.rfft(x[idx] * w, axis=1)
power = spec.real ** 2 + spec.imag ** 2
return np.log(fb @ power.T + 1e-6) # (n_mels, frames)
def pad_frames(lm, n):
if lm.shape[1] >= n:
return lm
return np.concatenate([lm, np.full((lm.shape[0], n - lm.shape[1]), lm.min())], axis=1)
def window_starts(n_frames):
return list(range(0, max(n_frames - WIN, 0) + 1, STEP))
def embed_windows(layers, lm, mu, sd, starts):
X = np.stack([(lm[:, s:s + WIN] - mu) / sd for s in starts])[:, None]
E = forward(layers, X)
return E / np.maximum(np.linalg.norm(E, axis=1, keepdims=True), 1e-12)
def stats_windows(lm, mu, sd, starts):
out = []
for s in starts:
z = (lm[:, s:s + WIN] - mu) / sd
v = np.concatenate([z.mean(axis=1), z.std(axis=1)])
v = v - v.mean()
out.append(v / max(float(np.linalg.norm(v)), 1e-12))
return np.stack(out)
def query_starts(lm):
"""Windows of the query that carry the sound: mean log energy within 1.5 of the loudest window."""
lm = pad_frames(lm, WIN)
starts = list(range(0, lm.shape[1] - WIN + 1, 8))
e = [float(lm[:, s:s + WIN].mean()) for s in starts]
top = max(e)
return lm, [s for s, v in zip(starts, e) if v >= top - 1.5]
def query_vectors(model, q_audio):
fb, mu, sd = model["fb"], model["mu"], model["sd"]
lm, starts = query_starts(logmel(q_audio, fb))
qe = embed_windows(model["layers"], lm, mu, sd, starts).mean(axis=0)
qs = stats_windows(lm, mu, sd, starts).mean(axis=0)
return qe / max(float(np.linalg.norm(qe)), 1e-12), qs / max(float(np.linalg.norm(qs)), 1e-12)
def similarity_curves(model, q_audio, rec_audio):
qe, qs = query_vectors(model, q_audio)
lm = pad_frames(logmel(rec_audio, model["fb"]), WIN)
starts = window_starts(lm.shape[1])
learned = embed_windows(model["layers"], lm, model["mu"], model["sd"], starts) @ qe
base = stats_windows(lm, model["mu"], model["sd"], starts) @ qs
times = [s * HOP / SR for s in starts]
return times, learned, base
def robust_z(c):
c = np.asarray(c, dtype=np.float64)
med = float(np.median(c))
mad = float(np.median(np.abs(c - med))) + 1e-6
return (c - med) / (1.4826 * mad)
def final_scores(learned, base, cfg):
"""cfg: {"alpha": float, "baseline_norm": "raw" | "z"} -> (next_echo_curve, baseline_curve)."""
a = cfg["alpha"]
nxt = a * robust_z(learned) + (1.0 - a) * robust_z(base)
bl = robust_z(base) if cfg["baseline_norm"] == "z" else np.asarray(base, dtype=np.float64)
return nxt, bl
def detect(times, sims, threshold):
out, cur = [], None
for t, s in zip(times, sims):
if s >= threshold:
if cur is None:
cur = [t, t + WIN_SEC, float(s)]
else:
cur[1] = t + WIN_SEC
cur[2] = max(cur[2], float(s))
elif cur is not None:
out.append(cur); cur = None
if cur is not None:
out.append(cur)
return out
def match_events(dets, gts, tol=0.5):
used, tp = set(), 0
for g0, g1 in gts:
for j, d in enumerate(dets):
if j not in used and d[0] < g1 + tol and d[1] > g0 - tol:
used.add(j); tp += 1
break
return tp, len(dets) - len(used), len(gts) - tp
def load_model(spec):
t = spec["tensors"]
return {"layers": spec["nets"]["embed"], "fb": t["mel_fb"], "mu": float(spec["meta"]["mu"]), "sd": float(spec["meta"]["sd"]),
"thresholds": spec["meta"]["thresholds"], "score": spec["meta"]["score"]}
|