File size: 5,453 Bytes
e187f2f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
"""Next Echo: find where a short example sound occurs inside a longer recording.

Shared by the notebook, inference.py and (ported) web/core.js. 16 kHz mono audio.
* logmel: 512-point STFT (periodic Hann, hop 256) -> 64 mel bands -> log
* windows of 64 frames (~1.04 s) every 16 frames (~0.256 s)
* learned: CNN embedding (nlrt) of each window, cosine to the query embedding
* baseline: cosine between per-band mean/std statistics of log-mel windows
* robust_z: per-recording normalisation (median / MAD) so thresholds adapt to each recording's background
* final_scores: Next Echo score = alpha * z(learned) + (1 - alpha) * z(log-mel); baseline = log-mel (raw or z), chosen on validation
* detect: threshold the score curve, merge consecutive windows into intervals
"""
import math

import numpy as np

from nlrt import forward

SR, N_FFT, HOP, N_MELS, WIN, STEP = 16000, 512, 256, 64, 64, 16
WIN_SEC = (WIN - 1) * HOP / SR + N_FFT / SR
STEP_SEC = STEP * HOP / SR


def mel_filterbank(n_mels=N_MELS, n_fft=N_FFT, sr=SR, fmin=50.0, fmax=8000.0):
    hz2mel = lambda f: 2595.0 * math.log10(1.0 + f / 700.0)  # noqa: E731
    mel2hz = lambda m: 700.0 * (10 ** (m / 2595.0) - 1.0)  # noqa: E731
    mels = np.linspace(hz2mel(fmin), hz2mel(fmax), n_mels + 2)
    hz = np.array([mel2hz(m) for m in mels])
    bins = np.arange(n_fft // 2 + 1) * sr / n_fft
    fb = np.zeros((n_mels, len(bins)))
    for i in range(n_mels):
        lo, c, hi = hz[i], hz[i + 1], hz[i + 2]
        fb[i] = np.maximum(0, np.minimum((bins - lo) / (c - lo), (hi - bins) / (hi - c)))
    return fb


def logmel(x, fb):
    x = np.asarray(x, dtype=np.float64)
    if len(x) < N_FFT:
        x = np.concatenate([x, np.zeros(N_FFT - len(x))])
    n = 1 + (len(x) - N_FFT) // HOP
    w = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(N_FFT) / N_FFT)
    idx = np.arange(N_FFT)[None, :] + HOP * np.arange(n)[:, None]
    spec = np.fft.rfft(x[idx] * w, axis=1)
    power = spec.real ** 2 + spec.imag ** 2
    return np.log(fb @ power.T + 1e-6)          # (n_mels, frames)


def pad_frames(lm, n):
    if lm.shape[1] >= n:
        return lm
    return np.concatenate([lm, np.full((lm.shape[0], n - lm.shape[1]), lm.min())], axis=1)


def window_starts(n_frames):
    return list(range(0, max(n_frames - WIN, 0) + 1, STEP))


def embed_windows(layers, lm, mu, sd, starts):
    X = np.stack([(lm[:, s:s + WIN] - mu) / sd for s in starts])[:, None]
    E = forward(layers, X)
    return E / np.maximum(np.linalg.norm(E, axis=1, keepdims=True), 1e-12)


def stats_windows(lm, mu, sd, starts):
    out = []
    for s in starts:
        z = (lm[:, s:s + WIN] - mu) / sd
        v = np.concatenate([z.mean(axis=1), z.std(axis=1)])
        v = v - v.mean()
        out.append(v / max(float(np.linalg.norm(v)), 1e-12))
    return np.stack(out)


def query_starts(lm):
    """Windows of the query that carry the sound: mean log energy within 1.5 of the loudest window."""
    lm = pad_frames(lm, WIN)
    starts = list(range(0, lm.shape[1] - WIN + 1, 8))
    e = [float(lm[:, s:s + WIN].mean()) for s in starts]
    top = max(e)
    return lm, [s for s, v in zip(starts, e) if v >= top - 1.5]


def query_vectors(model, q_audio):
    fb, mu, sd = model["fb"], model["mu"], model["sd"]
    lm, starts = query_starts(logmel(q_audio, fb))
    qe = embed_windows(model["layers"], lm, mu, sd, starts).mean(axis=0)
    qs = stats_windows(lm, mu, sd, starts).mean(axis=0)
    return qe / max(float(np.linalg.norm(qe)), 1e-12), qs / max(float(np.linalg.norm(qs)), 1e-12)


def similarity_curves(model, q_audio, rec_audio):
    qe, qs = query_vectors(model, q_audio)
    lm = pad_frames(logmel(rec_audio, model["fb"]), WIN)
    starts = window_starts(lm.shape[1])
    learned = embed_windows(model["layers"], lm, model["mu"], model["sd"], starts) @ qe
    base = stats_windows(lm, model["mu"], model["sd"], starts) @ qs
    times = [s * HOP / SR for s in starts]
    return times, learned, base


def robust_z(c):
    c = np.asarray(c, dtype=np.float64)
    med = float(np.median(c))
    mad = float(np.median(np.abs(c - med))) + 1e-6
    return (c - med) / (1.4826 * mad)


def final_scores(learned, base, cfg):
    """cfg: {"alpha": float, "baseline_norm": "raw" | "z"} -> (next_echo_curve, baseline_curve)."""
    a = cfg["alpha"]
    nxt = a * robust_z(learned) + (1.0 - a) * robust_z(base)
    bl = robust_z(base) if cfg["baseline_norm"] == "z" else np.asarray(base, dtype=np.float64)
    return nxt, bl


def detect(times, sims, threshold):
    out, cur = [], None
    for t, s in zip(times, sims):
        if s >= threshold:
            if cur is None:
                cur = [t, t + WIN_SEC, float(s)]
            else:
                cur[1] = t + WIN_SEC
                cur[2] = max(cur[2], float(s))
        elif cur is not None:
            out.append(cur); cur = None
    if cur is not None:
        out.append(cur)
    return out


def match_events(dets, gts, tol=0.5):
    used, tp = set(), 0
    for g0, g1 in gts:
        for j, d in enumerate(dets):
            if j not in used and d[0] < g1 + tol and d[1] > g0 - tol:
                used.add(j); tp += 1
                break
    return tp, len(dets) - len(used), len(gts) - tp


def load_model(spec):
    t = spec["tensors"]
    return {"layers": spec["nets"]["embed"], "fb": t["mel_fb"], "mu": float(spec["meta"]["mu"]), "sd": float(spec["meta"]["sd"]),
            "thresholds": spec["meta"]["thresholds"], "score": spec["meta"]["score"]}