app.py

# Hugging Face Spaces app ported from sed_ensemble_v2.ipynb
# Active models: AST x3 and PaSST (AudioSet-527); optional CLAP and Silero VAD.
# This intentionally does not use the separate DCASE 2024 CRNN app/checkpoint.

# Required by Hugging Face ZeroGPU. Keep this before importing torch.
import spaces

import os
import sys
import glob
import math
import difflib
import traceback

import numpy as np
import torch
import librosa
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import gradio as gr

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
print(f"ZeroGPU-aware inference device: {DEVICE}")
MAX_SEC = int(os.getenv("MAX_SEC", "60"))
MAX_CELLS = int(os.getenv("MAX_CELLS", "1500"))

# Limit CPU thread oversubscription on small hosted workers.
try:
    torch.set_num_threads(max(1, min(4, os.cpu_count() or 2)))
except Exception:
    pass


# -------------------- Model loading --------------------
MODELS = {}              # name -> (predict_fn(batch[B, samples]) -> probs[B, 527], sample_rate, family)

from transformers import AutoFeatureExtractor, ASTForAudioClassification


def load_ast(model_id):
    fe_ = AutoFeatureExtractor.from_pretrained(model_id)
    m = ASTForAudioClassification.from_pretrained(model_id).to(DEVICE).eval()

    @torch.no_grad()
    def predict(batch):
        x = fe_(list(batch), sampling_rate=16000, return_tensors="pt")["input_values"].to(DEVICE)
        with torch.autocast("cuda", dtype=torch.float16, enabled=(DEVICE == "cuda")):
            lg = m(input_values=x).logits
        return torch.sigmoid(lg.float()).cpu().numpy()

    return predict, [m.config.id2label[i] for i in range(len(m.config.id2label))]


AST_MAIN = "MIT/ast-finetuned-audioset-10-10-0.4593"
AST_EXTRA = ["MIT/ast-finetuned-audioset-12-12-0.447", "MIT/ast-finetuned-audioset-14-14-0.443"]   # [] to disable
fn, LABELS = load_ast(AST_MAIN)
assert len(LABELS) == 527
MODELS["AST"] = (fn, 16000, "AST")
for mid in AST_EXTRA:
    try:
        fn, labs = load_ast(mid)
        assert labs == LABELS, "label order differs"
        MODELS[mid.split("/")[-1]] = (fn, 16000, "AST")
    except Exception as e:
        print(f"{mid} skipped:", repr(e)[:200])

# --- PaSST (hear21passt): AudioSet-527, 32 kHz, <= 10 s ---
try:
    from hear21passt.base import get_basic_model
    passt = get_basic_model(mode="logits").to(DEVICE).eval()

    @torch.no_grad()
    def passt_predict(batch):
        x = np.stack([np.pad(b, (0, 320000 - len(b))) for b in batch]).astype(np.float32)   # zero-pad to 10 s
        return torch.sigmoid(passt(torch.from_numpy(x).to(DEVICE))).cpu().numpy()

    MODELS["PaSST"] = (passt_predict, 32000, "PaSST")
except Exception as e:
    print("PaSST skipped:", repr(e)[:300])

# BEATs is intentionally not enabled here: the working notebook did not load a fine-tuned AudioSet BEATs checkpoint.

# Do NOT run model forward passes at module scope on ZeroGPU.
# ZeroGPU only attaches the real GPU inside a function decorated with @spaces.GPU.
# The first real request to detect() exercises the loaded models.
if not MODELS:
    raise RuntimeError("No AudioSet classifier could be loaded. Check model downloads and startup logs.")

# --- CLAP: open-vocabulary categories from free text (experimental, not used for voting) ---
CLAP = None
NEG_PROMPTS = ["Sound of silence", "Sound of background noise", "Sound of people talking", "Sound of music", "Sound of wind"]
try:
    from transformers import ClapModel, AutoProcessor
    clap_model = ClapModel.from_pretrained("laion/clap-htsat-unfused").to(DEVICE).eval()
    clap_proc = AutoProcessor.from_pretrained("laion/clap-htsat-unfused")

    @torch.no_grad()
    def clap_logits(batch48, prompts):
        kw = dict(text=prompts, return_tensors="pt", padding=True)
        try:
            inp = clap_proc(audio=list(batch48), sampling_rate=48000, **kw)  # recent transformers
        except TypeError:
            inp = clap_proc(audios=list(batch48), sampling_rate=48000, **kw) # compatibility fallback
        inp = {k: v.to(DEVICE) for k, v in inp.items()}
        return clap_model(**inp).logits_per_audio.float().cpu().numpy()      # scaled audio-text similarity (B, K)

    # Do not execute CLAP inference during module import; inference belongs inside detect().
    CLAP = clap_logits
except Exception as e:
    print("CLAP skipped:", repr(e)[:300])

# --- Silero VAD: exact speech boundaries (speech only) ---
VAD = None
try:
    from silero_vad import load_silero_vad, get_speech_timestamps
    vad_model = load_silero_vad()

    def vad_segments(wav16, thr=0.5):
        ts = get_speech_timestamps(torch.from_numpy(np.asarray(wav16, np.float32)), vad_model, sampling_rate=16000,
                                   threshold=thr, min_speech_duration_ms=100, min_silence_duration_ms=150,
                                   return_seconds=True)
        return [(float(t["start"]), float(t["end"])) for t in ts]

    VAD = vad_segments
except Exception as e:
    print("Silero VAD skipped:", repr(e)[:300])

if not MODELS:
    raise RuntimeError("No AudioSet classifier loaded. Check Space build logs and model-download access.")

print("Tagger models:", list(MODELS), "| families:", sorted({v[2] for v in MODELS.values()}),
      "| CLAP:", CLAP is not None, "| VAD:", VAD is not None)

# -------------------- Category mapping and event logic --------------------
GROUPS = {
    # --- humans ---
    "speech": ["Speech", "Conversation", "Narration, monologue", "Male speech, man speaking",
               "Female speech, woman speaking", "Child speech, kid speaking"],
    "singing": ["Singing", "Choir", "Yodeling", "Chant", "Humming", "Rapping"],
    "shout_scream": ["Shout", "Yell", "Screaming", "Bellow", "Whoop"],
    "crying": ["Crying, sobbing", "Baby cry, infant cry", "Whimper"],
    "laughter": ["Laughter", "Giggle", "Chuckle, chortle", "Belly laugh", "Baby laughter", "Snicker"],
    "cough_sneeze": ["Cough", "Throat clearing", "Sneeze", "Sniff"],
    "breathing_snore": ["Breathing", "Gasp", "Snoring", "Wheeze", "Pant", "Respiratory sounds", "Snort", "Sigh"],
    "whistling": ["Whistling"],
    "clapping_cheer": ["Clapping", "Applause", "Cheering"],
    "footsteps": ["Walk, footsteps", "Run", "Shuffle"],
    "crowd": ["Crowd", "Hubbub, speech noise, speech babble", "Chatter", "Children shouting", "Children playing"],
    # --- animals ---
    "dog": ["Dog", "Bark", "Bow-wow", "Yip", "Howl", "Growling"],
    "cat": ["Cat", "Meow", "Purr", "Caterwaul"],
    "cattle_moo": ["Cattle, bovinae", "Moo", "Cowbell"],
    "pig": ["Pig", "Oink"],
    "sheep_goat": ["Sheep", "Bleat", "Goat"],
    "horse": ["Horse", "Neigh, whinny", "Clip-clop"],
    "chicken_rooster": ["Chicken, rooster", "Cluck", "Crowing, cock-a-doodle-doo", "Fowl"],
    "duck_goose": ["Duck", "Quack", "Goose", "Honk", "Turkey", "Gobble"],
    "bird": ["Bird", "Bird vocalization, bird call, bird song", "Chirp, tweet", "Pigeon, dove", "Coo", "Crow", "Caw",
             "Owl", "Hoot", "Squawk", "Bird flight, flapping wings"],
    "insect": ["Insect", "Fly, housefly", "Mosquito", "Buzz", "Bee, wasp, etc.", "Cricket"],
    "frog": ["Frog", "Croak"],
    "wild_animal": ["Roar", "Roaring cats (lions, tigers)", "Wild animals"],
    # --- music ---
    "music": ["Music", "Musical instrument", "Background music"],
    "guitar_strings": ["Guitar", "Plucked string instrument", "Acoustic guitar", "Electric guitar", "Bass guitar", "Banjo",
                       "Violin, fiddle", "Bowed string instrument", "Cello", "Harp", "Sitar", "Mandolin", "Ukulele"],
    "keyboard_instr": ["Piano", "Keyboard (musical)", "Organ", "Synthesizer", "Electric piano", "Harpsichord"],
    "drums_percussion": ["Drum", "Drum kit", "Percussion", "Snare drum", "Bass drum", "Cymbal", "Hi-hat", "Tabla",
                         "Drum machine", "Tambourine", "Maraca"],
    "wind_brass": ["Flute", "Trumpet", "Saxophone", "Brass instrument", "Wind instrument, woodwind instrument",
                   "Harmonica", "Accordion", "Bagpipes", "Clarinet", "Trombone", "French horn", "Didgeridoo"],
    "bells_chimes": ["Bell", "Church bell", "Chime", "Wind chime", "Jingle bell", "Bicycle bell", "Tubular bells", "Ding",
                     "Ding-dong", "Tinkle"],
    # --- home / DESED ---
    "alarm_bell": ["Alarm", "Alarm clock", "Buzzer", "Smoke detector, smoke alarm", "Beep, bleep"],
    "phone_ring": ["Telephone", "Telephone bell ringing", "Ringtone", "Cellphone buzz, vibrating alert", "Dial tone"],
    "blender": ["Blender"],
    "dishes": ["Dishes, pots, and pans", "Cutlery, silverware", "Chink, clink", "Clink", "Glass", "Shatter", "Clang"],
    "shaver_toothbrush": ["Electric shaver, electric razor", "Toothbrush", "Electric toothbrush"],
    "frying": ["Frying (food)", "Sizzle"],
    "running_water": ["Water", "Pour", "Water tap, faucet", "Splash, splatter", "Trickle, dribble", "Gurgling",
                      "Sink (filling or washing)", "Drip", "Liquid"],
    "toilet_flush": ["Toilet flush"],
    "vacuum_cleaner": ["Vacuum cleaner"],
    "appliance": ["Washing machine", "Mechanical fan", "Hair dryer", "Microwave oven", "Air conditioning", "Hum"],
    "door_knock": ["Door", "Knock", "Slam", "Doorbell", "Sliding door", "Tap"],
    "typing_click": ["Typing", "Computer keyboard", "Typewriter", "Click", "Clicking"],
    "clock_tick": ["Tick-tock", "Tick", "Clock"],
    "rustle_paper": ["Rustle", "Crumpling, crinkling", "Tearing", "Zipper (clothing)"],
    # --- environment ---
    "rain": ["Rain", "Raindrop", "Rain on surface"],
    "wind": ["Wind", "Wind noise (microphone)", "Rustling leaves"],
    "thunder": ["Thunderstorm", "Thunder"],
    "water_nature": ["Stream", "Waterfall", "Ocean", "Waves, surf"],
    "fire": ["Fire", "Crackle", "Wildfire"],
    # --- vehicles ---
    "car": ["Car", "Car passing by", "Motor vehicle (road)", "Skidding", "Tire squeal"],
    "engine": ["Engine", "Idling", "Engine starting", "Light engine (high frequency)", "Medium engine (mid frequency)",
               "Heavy engine (low frequency)", "Accelerating, revving, vroom"],
    "truck_bus": ["Truck", "Bus", "Air brake", "Reversing beeps"],
    "motorcycle": ["Motorcycle", "Moped"],
    "train": ["Train", "Rail transport", "Railroad car, train wagon", "Train horn", "Train wheels squealing",
              "Subway, metro, underground"],
    "aircraft": ["Aircraft", "Fixed-wing aircraft, airplane", "Helicopter", "Jet engine", "Propeller, airscrew",
                 "Aircraft engine"],
    "boat": ["Boat, Water vehicle", "Motorboat, speedboat", "Ship"],
    "vehicle_horn": ["Vehicle horn, car horn, honking", "Car alarm", "Toot"],
    "emergency_siren": ["Siren", "Emergency vehicle", "Police car (siren)", "Ambulance (siren)",
                        "Fire engine, fire truck (siren)", "Civil defense siren"],
    # --- impacts / tools ---
    "gunshot_explosion": ["Gunshot, gunfire", "Explosion", "Fireworks", "Firecracker", "Burst, pop", "Boom",
                          "Machine gun", "Cap gun"],
    "tools": ["Tools", "Hammer", "Drill", "Power tool", "Chainsaw", "Sawing", "Jackhammer", "Sanding",
              "Filing (rasp)", "Electric drill", "Lawn mower"],
    "impact": ["Thump, thud", "Bang", "Slap, smack", "Smash, crash", "Crack"],
}
CLASSES = list(GROUPS)
TRANSIENT = {"dog", "cat", "cattle_moo", "pig", "sheep_goat", "horse", "chicken_rooster", "duck_goose", "bird", "frog",
             "shout_scream", "laughter", "cough_sneeze", "clapping_cheer", "footsteps", "alarm_bell", "phone_ring",
             "dishes", "door_knock", "typing_click", "bells_chimes", "vehicle_horn", "gunshot_explosion", "impact",
             "clock_tick", "toilet_flush"}
TH_DEFAULT = 0.30                       # score (mean over model families) needed to TRIGGER an event
TH = {"speech": 0.40, "music": 0.40, "crowd": 0.40, "engine": 0.40, "car": 0.35, "singing": 0.35, "wind": 0.35,
      "cough_sneeze": 0.15, "laughter": 0.18}
for _c in TRANSIENT:                    # short sounds only fill part of a window, so their scores are lower
    TH.setdefault(_c, 0.20)
LOW_FRAC = 0.6                          # hysteresis: edges extend while score >= 60% of the trigger threshold
GAP_S = {c: (0.25 if c in TRANSIENT else 1.0) for c in CLASSES}     # merge gaps shorter than this
MIN_S = {c: (0.25 if c in TRANSIENT else 1.0) for c in CLASSES}     # drop events shorter than this
TRIM = 0.6                              # transient events: keep cells >= 60% of the event's peak score
DISCOVER_TH, DISCOVER_MAX = 0.40, 6     # automatic discovery of labels outside the groups above
DISCOVER_SKIP = {"Silence", "Noise", "Environmental noise", "Static", "Mains hum", "Distortion", "White noise",
                 "Pink noise", "Throbbing", "Vibration", "Inside, small room", "Inside, large room or hall",
                 "Inside, public space", "Outside, urban or manmade", "Outside, rural or natural", "Sound effect",
                 "Reverberation", "Echo", "Field recording", "Animal", "Domestic animals, pets",
                 "Livestock, farm animals, working animals", "Human voice", "Human sounds", "Human group actions",
                 "Domestic sounds, home sounds", "Vehicle", "Mechanisms", "Tool", "Liquid"}
CLAP_TH = 0.5                           # CLAP: softmax mass on a prompt vs. background prompts

# -------------------- Windowing and timeline helpers --------------------
def build_groups(labels, groups):
    low = {l.lower(): i for i, l in enumerate(labels)}
    ids = {g: sorted({low[n.lower()] for n in names if n.lower() in low}) for g, names in groups.items()}
    missing = {g: [n for n in names if n.lower() not in low] for g, names in groups.items()}
    return ids, missing


def window_batches(wav, sr, win_s, hop_s, n_cells, bs=4):
    """Cell i covers [i*hop, (i+1)*hop); its window is centred on the cell (audio is zero-padded at both ends)."""
    win, hop = int(round(win_s * sr)), int(round(hop_s * sr))
    pad = (win - hop) // 2
    padded = np.zeros(n_cells * hop + (win - hop), np.float32)
    n = min(len(wav), len(padded) - pad)
    padded[pad:pad + n] = wav[:n]
    for i0 in range(0, n_cells, bs):
        yield np.stack([padded[i * hop:i * hop + win] for i in range(i0, min(n_cells, i0 + bs))])


def fuse(P, group_ids, classes, th, vote_rel=0.6):
    """P: {family: (cells, 527)} -> S (cells, C) mean group score, votes (cells, C) families in favour, per-family scores."""
    ms = {m: np.stack([p[:, group_ids[c]].max(1) if group_ids[c] else np.zeros(len(p)) for c in classes], 1)
          for m, p in P.items()}
    thr = np.array([th[c] for c in classes])
    return np.mean(list(ms.values()), 0), np.sum([v >= vote_rel * thr for v in ms.values()], 0), ms


def _runs(a):
    out, i = [], None
    for k, v in enumerate(np.append(a, False)):
        if v and i is None:
            i = k
        elif not v and i is not None:
            out.append([i, k - 1]); i = None
    return out


def extract_events(S, votes, names, th, min_votes, hop, dur, low_frac=LOW_FRAC):
    """Hysteresis per class: a run needs one TRIGGER cell (score >= th and enough models agreeing); its edges extend
    while the score stays >= low_frac*th. Classes are independent, so events of different classes overlap freely."""
    ev = []
    for c, name in enumerate(names):
        hi = th[name]
        act = S[:, c] >= low_frac * hi
        trig = (S[:, c] >= hi) & (votes[:, c] >= min_votes)
        gap = math.ceil(GAP_S.get(name, 0.5) / hop - 1e-9) - 1
        merged = []
        for r in _runs(act):
            if merged and r[0] - merged[-1][1] - 1 <= gap:
                merged[-1][1] = r[1]
            else:
                merged.append(r)
        for i, j in merged:
            if not trig[i:j + 1].any() or j - i + 1 < math.ceil(MIN_S.get(name, 0.5) / hop - 1e-9):
                continue
            if name in TRANSIENT:                            # window-level tags are smeared; keep the peak region
                seg = S[i:j + 1, c]; k = int(seg.argmax()); keep = seg >= TRIM * seg[k]
                l = r_ = k
                while l > 0 and keep[l - 1]: l -= 1
                while r_ < len(seg) - 1 and keep[r_ + 1]: r_ += 1
                i, j = i + l, i + r_
            ev.append((i * hop, min((j + 1) * hop, dur), name, float(S[i:j + 1, c].mean()), int(votes[i:j + 1, c].max())))
    return sorted(ev)


def refine_with_vad(events, vad, S_speech, votes_speech, hop, th_speech, dur, cover=0.3):
    """Replace coarse speech events by VAD segments (exact boundaries) that the taggers also support."""
    out = [e for e in events if e[2] != "speech"]
    for a, b in vad:
        i0 = int(a / hop); i1 = max(int(math.ceil(b / hop)), i0 + 1)
        seg = S_speech[i0:i1]
        if len(seg) and np.mean(seg >= 0.5 * th_speech) >= cover:
            out.append((float(a), float(min(b, dur)), "speech", float(seg.mean()), int(votes_speech[i0:i1].max())))
    return sorted(out)


def overlaps(events, min_classes=2):
    pts = sorted({e[0] for e in events} | {e[1] for e in events})
    segs = []
    for a, b in zip(pts[:-1], pts[1:]):
        act = sorted({e[2] for e in events if e[0] <= (a + b) / 2 < e[1]})
        if len(act) >= min_classes:
            if segs and segs[-1][2] == act and abs(segs[-1][1] - a) < 1e-9:
                segs[-1][1] = b
            else:
                segs.append([a, b, act])
    return segs


def piano_roll(events, dur, ov):
    rows = []
    for e in sorted(events):                                  # rows ordered by first onset
        if e[2] not in rows:
            rows.append(e[2])
    fig, ax = plt.subplots(figsize=(11, max(2.6, 0.45 * len(rows) + 1.3)))
    for a, b, _ in ov:
        ax.axvspan(a, b, color="orange", alpha=0.15, lw=0)
    for r, c in enumerate(rows):
        for s, e, _, conf, _v in [x for x in events if x[2] == c]:
            ax.barh(r, e - s, left=s, height=0.7, color=plt.cm.viridis(min(max(conf, 0.0), 1.0)))
    ax.set_yticks(range(len(rows))); ax.set_yticklabels(rows, fontsize=9); ax.invert_yaxis()
    ax.set_xlim(0, dur); ax.set_xlabel("time (s)   [orange = 2+ sounds at once, bar colour = confidence]")
    if not rows:
        ax.text(0.5, 0.5, "no events", ha="center", va="center", transform=ax.transAxes)
    fig.tight_layout()
    return fig


def softmax(x, axis=1):
    x = x - x.max(axis, keepdims=True); e = np.exp(x)
    return e / e.sum(axis, keepdims=True)


def suggest(labels, name):
    return difflib.get_close_matches(name, labels, n=2, cutoff=0.6)

# -------------------- Detection function and Gradio UI --------------------
try:
    import noisereduce as nr
except Exception:
    nr = None

GROUP_IDS, MISSING = build_groups(LABELS, GROUPS)
bad = [(g, n) for g in CLASSES for n in MISSING[g]]
print(f"{len(CLASSES)} categories | {sum(len(v) for v in GROUP_IDS.values())} AudioSet labels matched | "
      f"{len(bad)} names unknown to the model (ignored)")
for g, n in bad[:40]:
    print(f"   {g}: '{n}' -> did you mean {suggest(LABELS, n) or 'nothing close'}")
for g in CLASSES:
    if not GROUP_IDS[g]:
        print(f"WARNING: no AudioSet label matched for '{g}' -> it will never fire")
_used = {i for ids in GROUP_IDS.values() for i in ids}
DISCOVER_IDS = [i for i, l in enumerate(LABELS)
                if i not in _used and l not in DISCOVER_SKIP and "music" not in l.lower()]


@torch.no_grad()
def tag_windows(wavs, win_s, hop_s, dur):
    n_cells = int(math.ceil(dur / hop_s))
    per_model = {name: np.concatenate([fn(b) for b in window_batches(wavs[sr], sr, win_s, hop_s, n_cells)])
                 for name, (fn, sr, fam) in MODELS.items()}
    fam = {}
    for name, (_, _, f) in MODELS.items():
        fam.setdefault(f, []).append(per_model[name])
    return {f: np.mean(v, 0) for f, v in fam.items()}


@spaces.GPU(duration=60)
def detect(audio_path, sens=0.0, win_s=2.0, hop_s=0.25, min_votes=2, use_vad=True, denoise=False,
           discover=True, extra=""):
    if audio_path is None:
        return "Upload or record audio.", None
    wav48, _ = librosa.load(audio_path, sr=48000, mono=True, duration=MAX_SEC)
    if wav48.size == 0:
        return "The uploaded audio is empty or could not be decoded.", None
    if denoise and nr is not None:                                 # optional: can hurt tagging, so default off
        wav48 = nr.reduce_noise(y=wav48, sr=48000, stationary=True, prop_decrease=0.6).astype(np.float32)
    wav48 = wav48 / (np.abs(wav48).max() + 1e-6) * 0.9
    dur = len(wav48) / 48000
    hop_s = float(hop_s)
    if dur / hop_s > MAX_CELLS:
        hop_s = math.ceil(dur / MAX_CELLS / 0.05) * 0.05
    wavs = {48000: wav48}
    for sr in {s for _, s, _ in MODELS.values()} | {16000}:
        wavs[sr] = wav48 if sr == 48000 else librosa.resample(wav48, orig_sr=48000, target_sr=sr)

    P = tag_windows(wavs, float(win_s), hop_s, dur)                 # {family: (cells, 527)}
    nf, mv = len(P), int(min(min_votes, len(P)))
    th = {c: float(np.clip(TH.get(c, TH_DEFAULT) + sens, 0.05, 0.9)) for c in CLASSES}
    S, votes, ms = fuse(P, GROUP_IDS, CLASSES, th)
    names, S_all, V_all = list(CLASSES), S, votes

    if discover and DISCOVER_IDS:                                   # any other AudioSet label that is clearly present
        Sd = np.mean(list(P.values()), 0)[:, DISCOVER_IDS]
        top = np.argsort(Sd.max(0))[::-1][:DISCOVER_MAX]
        top = [k for k in top if Sd[:, k].max() >= DISCOVER_TH]
        if top:
            dn = [f"auto: {LABELS[DISCOVER_IDS[k]]}" for k in top]
            Vd = np.sum([p[:, [DISCOVER_IDS[k] for k in top]] >= 0.6 * DISCOVER_TH for p in P.values()], 0)
            th.update({n: DISCOVER_TH for n in dn})
            names, S_all, V_all = names + dn, np.hstack([S, Sd[:, top]]), np.hstack([votes, Vd])

    ev = extract_events(S_all, V_all, names, th, mv, hop_s, dur)
    note_vad = ""
    if use_vad and VAD is not None:
        k = CLASSES.index("speech")
        ev = refine_with_vad(ev, VAD(wavs[16000]), S[:, k], votes[:, k], hop_s, th["speech"], dur)
        note_vad = " | speech boundaries from Silero VAD"

    prompts = [p.strip() for p in (extra or "").split(",") if p.strip()]
    if prompts and CLAP is not None:                                # open-vocabulary categories (experimental)
        allp = [f"Sound of {p}" for p in prompts] + NEG_PROMPTS
        n_cells = int(math.ceil(dur / hop_s))
        Pc = softmax(np.concatenate([CLAP(b, allp) for b in window_batches(wavs[48000], 48000, float(win_s), hop_s,
                                                                         n_cells)]))[:, :len(prompts)]
        cn = [f"clap: {p}" for p in prompts]
        ev += extract_events(Pc, (Pc >= CLAP_TH).astype(int), cn, {n: CLAP_TH for n in cn}, 1, hop_s, dur)
    elif prompts:
        note_vad += " | CLAP not loaded, extra categories ignored"
    ev = sorted(ev)
    ov = overlaps(ev)

    hit = {e[2] for e in ev}                                        # classes that came close but were not reported
    near = []
    for c, name in enumerate(CLASSES):
        k = int(S[:, c].argmax())
        if name not in hit and S[k, c] >= 0.4 * th[name]:
            per = ", ".join(f"{m} {ms[m][k, c]:.2f}" for m in ms)
            near.append((S[k, c] / th[name], f"  {k * hop_s:6.2f}s  {name:<16} fused {S[k, c]:.2f} (needs {th[name]:.2f}, "
                         f"{int(votes[k, c])}/{nf} agree)  [{per}]"))
    near = [t for _, t in sorted(near, reverse=True)[:8]]

    out = [f"Model families: {', '.join(P)} ({len(MODELS)} networks) | window {win_s}s hop {hop_s:g}s | "
           f"{mv}/{nf} families must agree{note_vad}", ""]
    for m, p in P.items():
        top = np.argsort(p.max(0))[::-1][:6]
        out.append(f"{m} hears: " + ", ".join(f"{LABELS[i]} {p[:, i].max():.2f}" for i in top))
    out += ["", "Start  -  End      | Sound                      | Conf  | Agree", "-" * 66]
    out += [f"{s:6.2f}s - {e:6.2f}s | {n:<26} | {c*100:4.1f}% | {'CLAP' if n.startswith('clap:') else f'{v}/{nf}'}"
            for s, e, n, c, v in ev] or ["(no events)"]
    out += ["", "Overlapping sounds:"] + ([f"  {a:6.2f}s - {b:6.2f}s : {' + '.join(cl)}" for a, b, cl in ov] or ["  (none)"])
    if near:
        out += ["", "Close but NOT reported (lower 'Sensitivity offset' or 'Families that must agree'):"] + near
    return "\n".join(out), piano_roll(ev, dur, ov)


demo = gr.Interface(
    fn=detect,
    inputs=[gr.Audio(type="filepath", label="Audio"),
            gr.Slider(-0.2, 0.2, value=0.0, step=0.05, label="Sensitivity offset (lower = more events)"),
            gr.Dropdown([1.0, 2.0, 3.0], value=2.0, label="Window (s): 1 = tighter timing, 3 = better tags"),
            gr.Dropdown([0.1, 0.25, 0.5], value=0.25, label="Hop (s): smaller = finer but slower"),
            gr.Slider(1, 3, value=2, step=1, label="Model families that must agree"),
            gr.Checkbox(value=VAD is not None, label="Exact speech boundaries (Silero VAD)"),
            gr.Checkbox(value=False, label="Denoise first (experimental, can hurt)"),
            gr.Checkbox(value=True, label="Auto-discover other AudioSet sounds"),
            gr.Textbox(value="", label="Extra categories, comma separated (CLAP, experimental)",
                       placeholder="tractor, drone buzzing, cash register")],
    outputs=[gr.Textbox(label="Detected events", lines=30), gr.Plot(label="Timeline (overlaps shaded)")],
    title="Overlap-aware sound event detection: AST x3 + PaSST + CLAP + VAD",
)
demo.queue(max_size=16).launch(
    ssr_mode=False,
    show_error=True,
    debug=True
)