Tonic commited on
Commit
970ceac
·
verified ·
1 Parent(s): cbe2712

Upload job_train.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. job_train.py +425 -424
job_train.py CHANGED
@@ -1,424 +1,425 @@
1
- # /// script
2
- # requires-python = ">=3.11"
3
- # dependencies = [
4
- # "torch>=2.4",
5
- # "torchvision>=0.19",
6
- # "timm>=1.0",
7
- # "datasets>=3.0",
8
- # "pyarrow",
9
- # "onnx>=1.16",
10
- # "onnxruntime>=1.18",
11
- # "pillow",
12
- # "numpy",
13
- # "huggingface_hub>=0.30",
14
- # ]
15
- # ///
16
- """Challenge-5 acoustic wake model: train on the unified dataset, evaluate, export ONNX, publish to HF.
17
-
18
- Runs as a Hugging Face Job (uv). One arch x one seed per job.
19
-
20
- Usage: job_train.py --arch mobilenet_v3_small --seed 42 [--epochs 10] [--batch 64]
21
- """
22
- import argparse, io, json, math, os, random, time
23
- from pathlib import Path
24
-
25
- import numpy as np
26
-
27
- ARCHES = ("mobilenet_v3_small", "shufflenet_v2_x0_5", "squeezenet1_1",
28
- "efficientnet_lite0", "ghostnet_050", "cnn6_custom")
29
- SOUND_CLASSES = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor")
30
- CLASS_NAMES_UNIFIED = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor", "unknown")
31
- DATASET = "Tonic/unified-drone-acoustic-cuas"
32
- RESULTS_REPO = "Tonic/challenge5-acoustic-results"
33
-
34
-
35
- def build_backbone(arch: str, pretrained: bool):
36
- """Returns (backbone_module, feat_dim). Backbone maps [B,3,224,224] -> [B,feat_dim]."""
37
- import torch.nn as nn
38
- from torchvision import models
39
- if arch == "mobilenet_v3_small":
40
- m = models.mobilenet_v3_small(weights="IMAGENET1K_V1" if pretrained else None)
41
- feat = m.classifier[0].in_features
42
- m.classifier = nn.Identity()
43
- return m, feat
44
- if arch == "shufflenet_v2_x0_5":
45
- m = models.shufflenet_v2_x0_5(weights="IMAGENET1K_V1" if pretrained else None)
46
- feat = m.fc.in_features
47
- m.fc = nn.Identity()
48
- return m, feat
49
- if arch == "squeezenet1_1":
50
- m = models.squeezenet1_1(weights="IMAGENET1K_V1" if pretrained else None)
51
- feat = 512
52
- class Sq(nn.Module):
53
- def __init__(self, m):
54
- super().__init__()
55
- self.features = m.features
56
- def forward(self, x):
57
- return self.features(x).mean(dim=(2, 3))
58
- return Sq(m), feat
59
- if arch in ("efficientnet_lite0", "ghostnet_050"):
60
- import timm
61
- names = {"efficientnet_lite0": ["tf_efficientnet_lite0", "efficientnet_lite0"],
62
- "ghostnet_050": ["ghostnet_050", "ghostnet_100"]}[arch]
63
- last = None
64
- import torch
65
- for n in names:
66
- try:
67
- m = timm.create_model(n, pretrained=pretrained, num_classes=0)
68
- m.eval()
69
- with torch.no_grad():
70
- feat = int(m(torch.zeros(1, 3, 224, 224)).shape[1])
71
- return m, feat
72
- except Exception as e:
73
- last = e
74
- raise RuntimeError(f"timm model for {arch} failed: {last}")
75
- if arch == "cnn6_custom":
76
- import torch.nn as nn
77
- ch = (32, 64, 128, 256, 256, 512)
78
- blocks = []
79
- prev = 3
80
- for c in ch:
81
- blocks += [nn.Conv2d(prev, c, 3, padding=1, bias=False),
82
- nn.BatchNorm2d(c), nn.ReLU(inplace=True),
83
- nn.Conv2d(c, c, 3, padding=1, groups=c, bias=False), # depthwise
84
- nn.BatchNorm2d(c), nn.ReLU(inplace=True),
85
- nn.Conv2d(c, c, 1, bias=False), nn.BatchNorm2d(c), nn.ReLU(inplace=True),
86
- nn.MaxPool2d(2)]
87
- prev = c
88
- class CNN6(nn.Module):
89
- def __init__(self):
90
- super().__init__()
91
- self.net = nn.Sequential(*blocks)
92
- def forward(self, x):
93
- return self.net(x).mean(dim=(2, 3))
94
- return CNN6(), ch[-1]
95
- raise ValueError(arch)
96
-
97
-
98
- def main():
99
- import torch
100
- import torch.nn as nn
101
-
102
- ap = argparse.ArgumentParser()
103
- ap.add_argument("--arch", choices=ARCHES, required=True)
104
- ap.add_argument("--seed", type=int, default=42)
105
- ap.add_argument("--epochs", type=int, default=15)
106
- ap.add_argument("--batch", type=int, default=64)
107
- ap.add_argument("--lr", type=float, default=3e-4)
108
- ap.add_argument("--max-train-rows", type=int, default=160000)
109
- ap.add_argument("--val-rows", type=int, default=4000)
110
- ap.add_argument("--pretrained", type=int, default=1)
111
- ap.add_argument("--out", type=Path, default=Path("/tmp/c5_out"))
112
- args = ap.parse_args()
113
-
114
- random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
115
- if torch.cuda.is_available():
116
- torch.cuda.manual_seed_all(args.seed)
117
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
118
- print(f"[job] arch={args.arch} seed={args.seed} device={device} "
119
- f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
120
-
121
- # ---------- data ----------
122
- from datasets import load_dataset, concatenate_datasets
123
- t0 = time.time()
124
- ds = load_dataset(DATASET, "windows_5s")
125
- print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
126
- unames = ds["train"].features["sound_class"].names
127
- assert unames == list(CLASS_NAMES_UNIFIED), unames
128
-
129
- def to_xy(split, source=None, labeled=True, cap=None):
130
- d = ds[split]
131
- if source is not None:
132
- d = d.filter(lambda r: r["source"] == source, num_proc=4)
133
- if labeled:
134
- d = d.filter(lambda r: r["labeled"], num_proc=4)
135
- if cap and len(d) > cap:
136
- d = d.shuffle(seed=args.seed).select(range(cap))
137
- return d
138
-
139
- train_d = to_xy("train", cap=args.max_train_rows)
140
- # oversample the 48 challenge5 train clips so the challenge distribution is not
141
- # drowned out by the (easy) external corpora
142
- c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
143
- rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
144
- c5_up = concatenate_datasets([c5_train] * (rep + 1))
145
- train_d = concatenate_datasets([train_d, c5_up])
146
- print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
147
- val_d = to_xy("validation", cap=args.val_rows)
148
- c5_valid = to_xy("validation", source="challenge5", labeled=True)
149
- c5_test = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
150
- print(f"[data] train={len(train_d)} val={len(val_d)} c5_valid={len(c5_valid)} c5_test={len(c5_test)}", flush=True)
151
-
152
- from PIL import Image
153
- from torch.utils.data import DataLoader
154
-
155
- def augment(img, is_c5):
156
- # img: HxWx3 float 0..1; gain + freq/time masks (stronger for challenge rows)
157
- g = random.uniform(0.4, 2.0) if is_c5 else random.uniform(0.8, 1.25)
158
- img = img * g
159
- if is_c5 and random.random() < 0.7:
160
- h, w = img.shape[0], img.shape[1]
161
- fm = random.randint(0, max(1, h // 8)); f0 = random.randint(0, h - fm)
162
- img[f0:f0 + fm] = 0
163
- tm = random.randint(0, max(1, w // 8)); t0 = random.randint(0, w - tm)
164
- img[:, t0:t0 + tm] = 0
165
- return np.clip(img, 0, 1)
166
-
167
- def collate(batch):
168
- xs = torch.stack([torch.from_numpy(augment(np.asarray(r["spectrogram"], dtype=np.float32) / 255.0,
169
- r["source"] == "challenge5"))
170
- .permute(2, 0, 1) for r in batch])
171
- wake = torch.tensor([float(r["has_drone_audio"]) for r in batch])
172
- cls = torch.tensor([SOUND_CLASSES.index(CLASS_NAMES_UNIFIED[r["sound_class"]])
173
- if r["sound_class"] < 4 else 0 for r in batch], dtype=torch.long)
174
- ids = [r["sample_id"] for r in batch]
175
- return xs, wake, cls, ids
176
-
177
- train_loader = DataLoader(train_d, batch_size=args.batch, shuffle=True, num_workers=4,
178
- collate_fn=collate, drop_last=True)
179
- val_loader = DataLoader(val_d, batch_size=256, shuffle=False, num_workers=4, collate_fn=collate)
180
- c5v_loader = DataLoader(c5_valid, batch_size=16, shuffle=False, collate_fn=collate)
181
- c5t_loader = DataLoader(c5_test, batch_size=16, shuffle=False, collate_fn=collate)
182
-
183
- # ---------- model ----------
184
- class WakeCNN(nn.Module):
185
- def __init__(self, arch, pretrained):
186
- super().__init__()
187
- self.register_buffer("mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
188
- self.register_buffer("std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
189
- self.backbone, feat = build_backbone(arch, pretrained)
190
- self.head_wake = nn.Linear(feat, 1)
191
- self.head_class = nn.Linear(feat, 4)
192
-
193
- def forward(self, x):
194
- x = (x - self.mean) / self.std
195
- f = self.backbone(x)
196
- return self.head_wake(f).squeeze(-1), self.head_class(f)
197
-
198
- model = WakeCNN(args.arch, bool(args.pretrained)).to(device)
199
- n_params = sum(p.numel() for p in model.parameters())
200
- print(f"[model] {args.arch} params={n_params/1e6:.2f}M", flush=True)
201
-
202
- opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
203
- sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs)
204
- pos = float(np.mean([r["has_drone_audio"] for r in train_d.select(range(2000))]))
205
- pos_weight = torch.tensor([min(2.0, max(1.0, ((1 - pos) / max(pos, 1e-3)) ** 0.5))], device=device)
206
- print(f"[train] pos_rate~{pos:.3f} pos_weight={float(pos_weight):.3f}", flush=True)
207
- bce = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
208
- ce = nn.CrossEntropyLoss()
209
-
210
- def eval_loader(loader, model):
211
- model.eval()
212
- dev = next(model.parameters()).device
213
- W, Y, C, P, ids = [], [], [], [], []
214
- with torch.no_grad():
215
- for xs, wake, cls, sids in loader:
216
- wl, cl = model(xs.to(dev))
217
- W.append(torch.sigmoid(wl).cpu()); Y.append(wake); C.append(cls)
218
- P.append(cl.argmax(1).cpu()); ids += sids
219
- return (torch.cat(W).numpy(), torch.cat(Y).numpy(), torch.cat(C).numpy(),
220
- torch.cat(P).numpy(), ids)
221
-
222
- def sweep(W, Y, grid=None):
223
- grid = grid or [round(0.05 + 0.05 * i, 2) for i in range(18)]
224
- best = (-1, 0.5, None)
225
- for t in grid:
226
- p = W >= t
227
- tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
228
- f1 = 2 * tp / max(1, 2 * tp + fp + fn)
229
- u = f1 - 0.15 * float(p.mean())
230
- if u > best[0]:
231
- best = (u, t, (f1, float(p.mean()), tp, fp, fn))
232
- return best
233
-
234
- def sweep_with_c5(W, Y, Wv, Yv):
235
- """Pick threshold maximizing unified-val utility; among thresholds within
236
- 0.03 utility of the best, tie-break by challenge5-valid F1 (8 clips)."""
237
- grid = [round(0.05 + 0.05 * i, 2) for i in range(18)]
238
- scored = []
239
- for t in grid:
240
- p = W >= t
241
- tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
242
- f1 = 2 * tp / max(1, 2 * tp + fp + fn)
243
- u = f1 - 0.15 * float(p.mean())
244
- pv = Wv >= t
245
- tpv = int((pv & (Yv > 0.5)).sum()); fpv = int((pv & (Yv <= 0.5)).sum()); fnv = int((~pv & (Yv > 0.5)).sum())
246
- f1v = 2 * tpv / max(1, 2 * tpv + fpv + fnv)
247
- scored.append((u, t, f1, float(p.mean()), f1v, pv.mean()))
248
- best_u = max(x[0] for x in scored)
249
- cands = [x for x in scored if x[0] >= best_u - 0.03]
250
- cands.sort(key=lambda x: (x[4], -x[5])) # best c5 F1, then lowest duty
251
- u, t, f1, duty, f1v, dutyv = cands[0]
252
- return u, t, (f1, duty, f1v, dutyv)
253
-
254
- args.out.mkdir(parents=True, exist_ok=True)
255
- log_path = args.out / "train_log.txt"
256
- logf = open(log_path, "a")
257
-
258
- def log(msg):
259
- print(msg, flush=True)
260
- logf.write(msg + "\n"); logf.flush()
261
-
262
- best_u, best_t, best_state = -1, 0.5, None
263
- for ep in range(1, args.epochs + 1):
264
- model.train()
265
- t0 = time.time()
266
- n_seen = 0
267
- scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
268
- for xs, wake, cls, _ in train_loader:
269
- opt.zero_grad()
270
- with torch.amp.autocast("cuda", enabled=device.type == "cuda"):
271
- wl, cl = model(xs.to(device))
272
- loss = bce(wl, wake.to(device)) + 0.5 * ce(cl, cls.to(device))
273
- scaler.scale(loss).backward()
274
- scaler.step(opt)
275
- scaler.update()
276
- n_seen += len(xs)
277
- sched.step()
278
- W, Y, C, P, _ = eval_loader(val_loader, model)
279
- Wv, Yv, Cv, Pv, _ = eval_loader(c5v_loader, model)
280
- u, t, stats = sweep_with_c5(W, Y, Wv, Yv)
281
- cls_acc_v = float((Pv == Cv).mean())
282
- log(f"[ep {ep}/{args.epochs}] {time.time()-t0:.0f}s rows={n_seen} "
283
- f"unified_val: F1={stats[0]:.4f} duty={stats[1]:.3f} @thr={t:.2f} u={u:.4f} | "
284
- f"c5_valid: F1={stats[2]:.4f} duty={stats[3]:.3f} cls_acc={cls_acc_v:.3f}")
285
- if u > best_u:
286
- best_u, best_t = u, t
287
- best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
288
- torch.save(best_state, args.out / "best.pt")
289
-
290
- if best_state is not None:
291
- model.load_state_dict(best_state)
292
- model.eval().cpu()
293
-
294
- # final metrics
295
- W, Y, C, P, _ = eval_loader(val_loader, model)
296
- Wv, Yv, Cv, Pv, ids_v = eval_loader(c5v_loader, model)
297
- u, t, stats = sweep_with_c5(W, Y, Wv, Yv)
298
- pv = Wv >= best_t
299
- tp = int((pv & (Yv > 0.5)).sum()); fp = int((pv & (Yv <= 0.5)).sum()); fn = int((~pv & (Yv > 0.5)).sum())
300
- f1v = 2 * tp / max(1, 2 * tp + fp + fn)
301
- dutyv = float(pv.mean())
302
- cls_acc_v = float((Pv == Cv).mean())
303
- log(f"[final] thr={best_t:.2f} c5_valid wake_F1={f1v:.4f} duty={dutyv:.3f} cls_acc={cls_acc_v:.4f} "
304
- f"(tp={tp} fp={fp} fn={fn}) unified_val_u={best_u:.4f}")
305
-
306
- # ---------- ONNX export with threshold metadata ----------
307
- import onnx
308
- dummy = torch.rand(1, 3, 224, 224)
309
- onnx_path = args.out / "audio_wake_cnn.onnx"
310
- torch.onnx.export(model, (dummy,), str(onnx_path), opset_version=17,
311
- input_names=["spectrogram"], output_names=["wake_logit", "sound_class_logits"],
312
- dynamo=False)
313
- m = onnx.load(str(onnx_path))
314
- md = {e.key: e.value for e in m.metadata_props}
315
- md["wake_threshold"] = str(best_t)
316
- md["arch"] = args.arch
317
- onnx.helper.set_model_props(m, md)
318
- onnx.save(m, str(onnx_path))
319
- onnx_mb = onnx_path.stat().st_size / 1e6
320
- log(f"[onnx] saved {onnx_mb:.2f} MB, threshold={best_t:.2f}")
321
-
322
- # ---------- ORT latency ----------
323
- import onnxruntime as ort
324
- sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
325
- x = np.random.rand(1, 3, 224, 224).astype(np.float32)
326
- for _ in range(10):
327
- sess.run(None, {"spectrogram": x})
328
- ts = []
329
- for _ in range(100):
330
- t0 = time.time()
331
- sess.run(None, {"spectrogram": x})
332
- ts.append((time.time() - t0) * 1000)
333
- lat_ms = float(np.mean(ts))
334
- log(f"[ort] CPU latency mean={lat_ms:.2f} ms p50={float(np.percentile(ts,50)):.2f}")
335
-
336
- # ---------- predictions on challenge5 valid + test ----------
337
- def predict(loader):
338
- outs = []
339
- with torch.no_grad():
340
- for xs, wake, cls, sids in loader:
341
- wl, cl = model(xs)
342
- wp = torch.sigmoid(wl).numpy()
343
- cp = cl.argmax(1).numpy()
344
- for sid, w, c in zip(sids, wp, cp):
345
- clip = sid.split("__")[1].replace(".wav", "")
346
- outs.append({"id": clip, "wake_triggered": bool(w >= best_t),
347
- "sound_class": SOUND_CLASSES[int(c)]})
348
- return outs
349
-
350
- preds_valid = predict(c5v_loader)
351
- preds_test = predict(c5t_loader)
352
- (args.out / "preds_valid.jsonl").write_text(
353
- "\n".join(json.dumps(p) for p in preds_valid) + "\n")
354
- (args.out / "preds_test.jsonl").write_text(
355
- "\n".join(json.dumps(p) for p in preds_test) + "\n")
356
-
357
- # ---------- publish ----------
358
- metrics = {
359
- "arch": args.arch, "seed": args.seed, "epochs": args.epochs, "batch": args.batch,
360
- "lr": args.lr, "params": n_params, "onnx_mb": round(onnx_mb, 2),
361
- "wake_threshold": best_t,
362
- "c5_valid": {"wake_F1": round(f1v, 4), "optical_duty_cycle": round(dutyv, 4),
363
- "sound_class_accuracy": round(cls_acc_v, 4),
364
- "optical_inactivity_pct": round((1 - dutyv) * 100, 2), "tp": tp, "fp": fp, "fn": fn},
365
- "unified_val": {"utility": round(best_u, 4), "thr": best_t,
366
- "F1": round(stats[0], 4), "duty": round(stats[1], 4)},
367
- "ort_cpu_latency_ms": round(lat_ms, 2),
368
- "train_rows": len(train_d), "val_rows": len(val_d),
369
- "dataset": DATASET,
370
- }
371
- (args.out / "metrics.json").write_text(json.dumps(metrics, indent=2))
372
-
373
- repo = f"Tonic/challenge5-acoustic-{args.arch}-s{args.seed}"
374
- from huggingface_hub import HfApi
375
- api = HfApi()
376
- api.create_repo(repo, repo_type="model", exist_ok=True, private=False)
377
- try:
378
- api.update_repo_settings(repo_id=repo, repo_type="model", gated="manual")
379
- except Exception as e:
380
- log(f"[publish] gating note: {str(e)[:80]}")
381
- card = f"""---
382
- license: apache-2.0
383
- tags: [acoustic-wake, c-uas, challenge-5, onnx, drone-detection]
384
- ---
385
- # Challenge-5 acoustic wake model: {args.arch} (seed {args.seed})
386
-
387
- Two-head CNN over 224x224 log-mel windows: `wake_logit` + `sound_class_logits` (4 classes),
388
- ImageNet normalization baked in-graph, `wake_threshold` stored in ONNX metadata.
389
-
390
- ## Metrics (challenge5 valid, {len(c5_valid)} clips)
391
- ```json
392
- {json.dumps(metrics['c5_valid'], indent=2)}
393
- ```
394
- ORT CPU latency: {lat_ms:.2f} ms/window | params {n_params/1e6:.2f}M | ONNX {onnx_mb:.2f} MB
395
-
396
- Trained on {DATASET} `windows_5s` (train, {len(train_d)} rows) + challenge5 train clips.
397
- Predictions: `preds_valid.jsonl`, `preds_test.jsonl` (challenge submission format).
398
- """
399
- (args.out / "README.md").write_text(card)
400
- api.upload_folder(folder_path=str(args.out), path_in_repo="", repo_id=repo, repo_type="model",
401
- commit_message=f"{args.arch} seed {args.seed}: F1={f1v:.3f} acc={cls_acc_v:.3f} thr={best_t:.2f}")
402
- log(f"[publish] https://huggingface.co/{repo}")
403
-
404
- # append to shared leaderboard dataset repo
405
- try:
406
- api.create_repo(RESULTS_REPO, repo_type="dataset", exist_ok=True, private=False)
407
- row = json.dumps(metrics) + "\n"
408
- try:
409
- old = api.hf_hub_download(RESULTS_REPO, "results.jsonl", repo_type="dataset")
410
- data = open(old, "rb").read() + row.encode()
411
- except Exception:
412
- data = row.encode()
413
- api.upload_file(path_or_fileobj=data, path_in_repo="results.jsonl",
414
- repo_id=RESULTS_REPO, repo_type="dataset",
415
- commit_message=f"add {args.arch} s{args.seed}")
416
- log(f"[publish] leaderboard {RESULTS_REPO} updated")
417
- except Exception as e:
418
- log(f"[publish] leaderboard append failed: {str(e)[:120]}")
419
-
420
- log(f"[done] arch={args.arch} seed={args.seed} F1={f1v:.4f} acc={cls_acc_v:.4f} duty={dutyv:.3f} thr={best_t:.2f} lat={lat_ms:.2f}ms")
421
-
422
-
423
- if __name__ == "__main__":
424
- main()
 
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = [
4
+ # "torch>=2.4",
5
+ # "torchvision>=0.19",
6
+ # "timm>=1.0",
7
+ # "datasets>=3.0",
8
+ # "pyarrow",
9
+ # "onnx>=1.16",
10
+ # "onnxruntime>=1.18",
11
+ # "pillow",
12
+ # "numpy",
13
+ # "huggingface_hub>=0.30",
14
+ # ]
15
+ # ///
16
+ """Challenge-5 acoustic wake model: train on the unified dataset, evaluate, export ONNX, publish to HF.
17
+
18
+ Runs as a Hugging Face Job (uv). One arch x one seed per job.
19
+
20
+ Usage: job_train.py --arch mobilenet_v3_small --seed 42 [--epochs 10] [--batch 64]
21
+ """
22
+ import argparse, io, json, math, os, random, time
23
+ from pathlib import Path
24
+
25
+ import numpy as np
26
+
27
+ ARCHES = ("mobilenet_v3_small", "shufflenet_v2_x0_5", "squeezenet1_1",
28
+ "efficientnet_lite0", "ghostnet_050", "cnn6_custom")
29
+ SOUND_CLASSES = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor")
30
+ CLASS_NAMES_UNIFIED = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor", "unknown")
31
+ DATASET = "Tonic/unified-drone-acoustic-cuas"
32
+ RESULTS_REPO = "Tonic/challenge5-acoustic-results"
33
+
34
+
35
+ def build_backbone(arch: str, pretrained: bool):
36
+ """Returns (backbone_module, feat_dim). Backbone maps [B,3,224,224] -> [B,feat_dim]."""
37
+ import torch.nn as nn
38
+ from torchvision import models
39
+ if arch == "mobilenet_v3_small":
40
+ m = models.mobilenet_v3_small(weights="IMAGENET1K_V1" if pretrained else None)
41
+ feat = m.classifier[0].in_features
42
+ m.classifier = nn.Identity()
43
+ return m, feat
44
+ if arch == "shufflenet_v2_x0_5":
45
+ m = models.shufflenet_v2_x0_5(weights="IMAGENET1K_V1" if pretrained else None)
46
+ feat = m.fc.in_features
47
+ m.fc = nn.Identity()
48
+ return m, feat
49
+ if arch == "squeezenet1_1":
50
+ m = models.squeezenet1_1(weights="IMAGENET1K_V1" if pretrained else None)
51
+ feat = 512
52
+ class Sq(nn.Module):
53
+ def __init__(self, m):
54
+ super().__init__()
55
+ self.features = m.features
56
+ def forward(self, x):
57
+ return self.features(x).mean(dim=(2, 3))
58
+ return Sq(m), feat
59
+ if arch in ("efficientnet_lite0", "ghostnet_050"):
60
+ import timm
61
+ names = {"efficientnet_lite0": ["tf_efficientnet_lite0", "efficientnet_lite0"],
62
+ "ghostnet_050": ["ghostnet_050", "ghostnet_100"]}[arch]
63
+ last = None
64
+ import torch
65
+ for n in names:
66
+ try:
67
+ m = timm.create_model(n, pretrained=pretrained, num_classes=0)
68
+ m.eval()
69
+ with torch.no_grad():
70
+ feat = int(m(torch.zeros(1, 3, 224, 224)).shape[1])
71
+ return m, feat
72
+ except Exception as e:
73
+ last = e
74
+ raise RuntimeError(f"timm model for {arch} failed: {last}")
75
+ if arch == "cnn6_custom":
76
+ import torch.nn as nn
77
+ ch = (32, 64, 128, 256, 256, 512)
78
+ blocks = []
79
+ prev = 3
80
+ for c in ch:
81
+ blocks += [nn.Conv2d(prev, c, 3, padding=1, bias=False),
82
+ nn.BatchNorm2d(c), nn.ReLU(inplace=True),
83
+ nn.Conv2d(c, c, 3, padding=1, groups=c, bias=False), # depthwise
84
+ nn.BatchNorm2d(c), nn.ReLU(inplace=True),
85
+ nn.Conv2d(c, c, 1, bias=False), nn.BatchNorm2d(c), nn.ReLU(inplace=True),
86
+ nn.MaxPool2d(2)]
87
+ prev = c
88
+ class CNN6(nn.Module):
89
+ def __init__(self):
90
+ super().__init__()
91
+ self.net = nn.Sequential(*blocks)
92
+ def forward(self, x):
93
+ return self.net(x).mean(dim=(2, 3))
94
+ return CNN6(), ch[-1]
95
+ raise ValueError(arch)
96
+
97
+
98
+ def main():
99
+ import torch
100
+ import torch.nn as nn
101
+
102
+ ap = argparse.ArgumentParser()
103
+ ap.add_argument("--arch", choices=ARCHES, required=True)
104
+ ap.add_argument("--seed", type=int, default=42)
105
+ ap.add_argument("--epochs", type=int, default=15)
106
+ ap.add_argument("--batch", type=int, default=64)
107
+ ap.add_argument("--lr", type=float, default=3e-4)
108
+ ap.add_argument("--dataset", type=str, default=DATASET)
109
+ ap.add_argument("--max-train-rows", type=int, default=160000)
110
+ ap.add_argument("--val-rows", type=int, default=4000)
111
+ ap.add_argument("--pretrained", type=int, default=1)
112
+ ap.add_argument("--out", type=Path, default=Path("/tmp/c5_out"))
113
+ args = ap.parse_args()
114
+
115
+ random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
116
+ if torch.cuda.is_available():
117
+ torch.cuda.manual_seed_all(args.seed)
118
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
119
+ print(f"[job] arch={args.arch} seed={args.seed} device={device} "
120
+ f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
121
+
122
+ # ---------- data ----------
123
+ from datasets import load_dataset, concatenate_datasets
124
+ t0 = time.time()
125
+ ds = load_dataset(args.dataset, "windows_5s")
126
+ print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
127
+ unames = ds["train"].features["sound_class"].names
128
+ assert unames == list(CLASS_NAMES_UNIFIED), unames
129
+
130
+ def to_xy(split, source=None, labeled=True, cap=None):
131
+ d = ds[split]
132
+ if source is not None:
133
+ d = d.filter(lambda r: r["source"] == source, num_proc=4)
134
+ if labeled:
135
+ d = d.filter(lambda r: r["labeled"], num_proc=4)
136
+ if cap and len(d) > cap:
137
+ d = d.shuffle(seed=args.seed).select(range(cap))
138
+ return d
139
+
140
+ train_d = to_xy("train", cap=args.max_train_rows)
141
+ # oversample the 48 challenge5 train clips so the challenge distribution is not
142
+ # drowned out by the (easy) external corpora
143
+ c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
144
+ rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
145
+ c5_up = concatenate_datasets([c5_train] * (rep + 1))
146
+ train_d = concatenate_datasets([train_d, c5_up])
147
+ print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
148
+ val_d = to_xy("validation", cap=args.val_rows)
149
+ c5_valid = to_xy("validation", source="challenge5", labeled=True)
150
+ c5_test = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
151
+ print(f"[data] train={len(train_d)} val={len(val_d)} c5_valid={len(c5_valid)} c5_test={len(c5_test)}", flush=True)
152
+
153
+ from PIL import Image
154
+ from torch.utils.data import DataLoader
155
+
156
+ def augment(img, is_c5):
157
+ # img: HxWx3 float 0..1; gain + freq/time masks (stronger for challenge rows)
158
+ g = random.uniform(0.4, 2.0) if is_c5 else random.uniform(0.8, 1.25)
159
+ img = img * g
160
+ if is_c5 and random.random() < 0.7:
161
+ h, w = img.shape[0], img.shape[1]
162
+ fm = random.randint(0, max(1, h // 8)); f0 = random.randint(0, h - fm)
163
+ img[f0:f0 + fm] = 0
164
+ tm = random.randint(0, max(1, w // 8)); t0 = random.randint(0, w - tm)
165
+ img[:, t0:t0 + tm] = 0
166
+ return np.clip(img, 0, 1)
167
+
168
+ def collate(batch):
169
+ xs = torch.stack([torch.from_numpy(augment(np.asarray(r["spectrogram"], dtype=np.float32) / 255.0,
170
+ r["source"] == "challenge5"))
171
+ .permute(2, 0, 1) for r in batch])
172
+ wake = torch.tensor([float(r["has_drone_audio"]) for r in batch])
173
+ cls = torch.tensor([SOUND_CLASSES.index(CLASS_NAMES_UNIFIED[r["sound_class"]])
174
+ if r["sound_class"] < 4 else 0 for r in batch], dtype=torch.long)
175
+ ids = [r["sample_id"] for r in batch]
176
+ return xs, wake, cls, ids
177
+
178
+ train_loader = DataLoader(train_d, batch_size=args.batch, shuffle=True, num_workers=4,
179
+ collate_fn=collate, drop_last=True)
180
+ val_loader = DataLoader(val_d, batch_size=256, shuffle=False, num_workers=4, collate_fn=collate)
181
+ c5v_loader = DataLoader(c5_valid, batch_size=16, shuffle=False, collate_fn=collate)
182
+ c5t_loader = DataLoader(c5_test, batch_size=16, shuffle=False, collate_fn=collate)
183
+
184
+ # ---------- model ----------
185
+ class WakeCNN(nn.Module):
186
+ def __init__(self, arch, pretrained):
187
+ super().__init__()
188
+ self.register_buffer("mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
189
+ self.register_buffer("std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
190
+ self.backbone, feat = build_backbone(arch, pretrained)
191
+ self.head_wake = nn.Linear(feat, 1)
192
+ self.head_class = nn.Linear(feat, 4)
193
+
194
+ def forward(self, x):
195
+ x = (x - self.mean) / self.std
196
+ f = self.backbone(x)
197
+ return self.head_wake(f).squeeze(-1), self.head_class(f)
198
+
199
+ model = WakeCNN(args.arch, bool(args.pretrained)).to(device)
200
+ n_params = sum(p.numel() for p in model.parameters())
201
+ print(f"[model] {args.arch} params={n_params/1e6:.2f}M", flush=True)
202
+
203
+ opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
204
+ sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs)
205
+ pos = float(np.mean([r["has_drone_audio"] for r in train_d.select(range(2000))]))
206
+ pos_weight = torch.tensor([min(2.0, max(1.0, ((1 - pos) / max(pos, 1e-3)) ** 0.5))], device=device)
207
+ print(f"[train] pos_rate~{pos:.3f} pos_weight={float(pos_weight):.3f}", flush=True)
208
+ bce = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
209
+ ce = nn.CrossEntropyLoss()
210
+
211
+ def eval_loader(loader, model):
212
+ model.eval()
213
+ dev = next(model.parameters()).device
214
+ W, Y, C, P, ids = [], [], [], [], []
215
+ with torch.no_grad():
216
+ for xs, wake, cls, sids in loader:
217
+ wl, cl = model(xs.to(dev))
218
+ W.append(torch.sigmoid(wl).cpu()); Y.append(wake); C.append(cls)
219
+ P.append(cl.argmax(1).cpu()); ids += sids
220
+ return (torch.cat(W).numpy(), torch.cat(Y).numpy(), torch.cat(C).numpy(),
221
+ torch.cat(P).numpy(), ids)
222
+
223
+ def sweep(W, Y, grid=None):
224
+ grid = grid or [round(0.05 + 0.05 * i, 2) for i in range(18)]
225
+ best = (-1, 0.5, None)
226
+ for t in grid:
227
+ p = W >= t
228
+ tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
229
+ f1 = 2 * tp / max(1, 2 * tp + fp + fn)
230
+ u = f1 - 0.15 * float(p.mean())
231
+ if u > best[0]:
232
+ best = (u, t, (f1, float(p.mean()), tp, fp, fn))
233
+ return best
234
+
235
+ def sweep_with_c5(W, Y, Wv, Yv):
236
+ """Pick threshold maximizing unified-val utility; among thresholds within
237
+ 0.03 utility of the best, tie-break by challenge5-valid F1 (8 clips)."""
238
+ grid = [round(0.05 + 0.05 * i, 2) for i in range(18)]
239
+ scored = []
240
+ for t in grid:
241
+ p = W >= t
242
+ tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
243
+ f1 = 2 * tp / max(1, 2 * tp + fp + fn)
244
+ u = f1 - 0.15 * float(p.mean())
245
+ pv = Wv >= t
246
+ tpv = int((pv & (Yv > 0.5)).sum()); fpv = int((pv & (Yv <= 0.5)).sum()); fnv = int((~pv & (Yv > 0.5)).sum())
247
+ f1v = 2 * tpv / max(1, 2 * tpv + fpv + fnv)
248
+ scored.append((u, t, f1, float(p.mean()), f1v, pv.mean()))
249
+ best_u = max(x[0] for x in scored)
250
+ cands = [x for x in scored if x[0] >= best_u - 0.03]
251
+ cands.sort(key=lambda x: (x[4], -x[5])) # best c5 F1, then lowest duty
252
+ u, t, f1, duty, f1v, dutyv = cands[0]
253
+ return u, t, (f1, duty, f1v, dutyv)
254
+
255
+ args.out.mkdir(parents=True, exist_ok=True)
256
+ log_path = args.out / "train_log.txt"
257
+ logf = open(log_path, "a")
258
+
259
+ def log(msg):
260
+ print(msg, flush=True)
261
+ logf.write(msg + "\n"); logf.flush()
262
+
263
+ best_u, best_t, best_state = -1, 0.5, None
264
+ for ep in range(1, args.epochs + 1):
265
+ model.train()
266
+ t0 = time.time()
267
+ n_seen = 0
268
+ scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
269
+ for xs, wake, cls, _ in train_loader:
270
+ opt.zero_grad()
271
+ with torch.amp.autocast("cuda", enabled=device.type == "cuda"):
272
+ wl, cl = model(xs.to(device))
273
+ loss = bce(wl, wake.to(device)) + 0.5 * ce(cl, cls.to(device))
274
+ scaler.scale(loss).backward()
275
+ scaler.step(opt)
276
+ scaler.update()
277
+ n_seen += len(xs)
278
+ sched.step()
279
+ W, Y, C, P, _ = eval_loader(val_loader, model)
280
+ Wv, Yv, Cv, Pv, _ = eval_loader(c5v_loader, model)
281
+ u, t, stats = sweep_with_c5(W, Y, Wv, Yv)
282
+ cls_acc_v = float((Pv == Cv).mean())
283
+ log(f"[ep {ep}/{args.epochs}] {time.time()-t0:.0f}s rows={n_seen} "
284
+ f"unified_val: F1={stats[0]:.4f} duty={stats[1]:.3f} @thr={t:.2f} u={u:.4f} | "
285
+ f"c5_valid: F1={stats[2]:.4f} duty={stats[3]:.3f} cls_acc={cls_acc_v:.3f}")
286
+ if u > best_u:
287
+ best_u, best_t = u, t
288
+ best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
289
+ torch.save(best_state, args.out / "best.pt")
290
+
291
+ if best_state is not None:
292
+ model.load_state_dict(best_state)
293
+ model.eval().cpu()
294
+
295
+ # final metrics
296
+ W, Y, C, P, _ = eval_loader(val_loader, model)
297
+ Wv, Yv, Cv, Pv, ids_v = eval_loader(c5v_loader, model)
298
+ u, t, stats = sweep_with_c5(W, Y, Wv, Yv)
299
+ pv = Wv >= best_t
300
+ tp = int((pv & (Yv > 0.5)).sum()); fp = int((pv & (Yv <= 0.5)).sum()); fn = int((~pv & (Yv > 0.5)).sum())
301
+ f1v = 2 * tp / max(1, 2 * tp + fp + fn)
302
+ dutyv = float(pv.mean())
303
+ cls_acc_v = float((Pv == Cv).mean())
304
+ log(f"[final] thr={best_t:.2f} c5_valid wake_F1={f1v:.4f} duty={dutyv:.3f} cls_acc={cls_acc_v:.4f} "
305
+ f"(tp={tp} fp={fp} fn={fn}) unified_val_u={best_u:.4f}")
306
+
307
+ # ---------- ONNX export with threshold metadata ----------
308
+ import onnx
309
+ dummy = torch.rand(1, 3, 224, 224)
310
+ onnx_path = args.out / "audio_wake_cnn.onnx"
311
+ torch.onnx.export(model, (dummy,), str(onnx_path), opset_version=17,
312
+ input_names=["spectrogram"], output_names=["wake_logit", "sound_class_logits"],
313
+ dynamo=False)
314
+ m = onnx.load(str(onnx_path))
315
+ md = {e.key: e.value for e in m.metadata_props}
316
+ md["wake_threshold"] = str(best_t)
317
+ md["arch"] = args.arch
318
+ onnx.helper.set_model_props(m, md)
319
+ onnx.save(m, str(onnx_path))
320
+ onnx_mb = onnx_path.stat().st_size / 1e6
321
+ log(f"[onnx] saved {onnx_mb:.2f} MB, threshold={best_t:.2f}")
322
+
323
+ # ---------- ORT latency ----------
324
+ import onnxruntime as ort
325
+ sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
326
+ x = np.random.rand(1, 3, 224, 224).astype(np.float32)
327
+ for _ in range(10):
328
+ sess.run(None, {"spectrogram": x})
329
+ ts = []
330
+ for _ in range(100):
331
+ t0 = time.time()
332
+ sess.run(None, {"spectrogram": x})
333
+ ts.append((time.time() - t0) * 1000)
334
+ lat_ms = float(np.mean(ts))
335
+ log(f"[ort] CPU latency mean={lat_ms:.2f} ms p50={float(np.percentile(ts,50)):.2f}")
336
+
337
+ # ---------- predictions on challenge5 valid + test ----------
338
+ def predict(loader):
339
+ outs = []
340
+ with torch.no_grad():
341
+ for xs, wake, cls, sids in loader:
342
+ wl, cl = model(xs)
343
+ wp = torch.sigmoid(wl).numpy()
344
+ cp = cl.argmax(1).numpy()
345
+ for sid, w, c in zip(sids, wp, cp):
346
+ clip = sid.split("__")[1].replace(".wav", "")
347
+ outs.append({"id": clip, "wake_triggered": bool(w >= best_t),
348
+ "sound_class": SOUND_CLASSES[int(c)]})
349
+ return outs
350
+
351
+ preds_valid = predict(c5v_loader)
352
+ preds_test = predict(c5t_loader)
353
+ (args.out / "preds_valid.jsonl").write_text(
354
+ "\n".join(json.dumps(p) for p in preds_valid) + "\n")
355
+ (args.out / "preds_test.jsonl").write_text(
356
+ "\n".join(json.dumps(p) for p in preds_test) + "\n")
357
+
358
+ # ---------- publish ----------
359
+ metrics = {
360
+ "arch": args.arch, "seed": args.seed, "epochs": args.epochs, "batch": args.batch,
361
+ "lr": args.lr, "params": n_params, "onnx_mb": round(onnx_mb, 2),
362
+ "wake_threshold": best_t,
363
+ "c5_valid": {"wake_F1": round(f1v, 4), "optical_duty_cycle": round(dutyv, 4),
364
+ "sound_class_accuracy": round(cls_acc_v, 4),
365
+ "optical_inactivity_pct": round((1 - dutyv) * 100, 2), "tp": tp, "fp": fp, "fn": fn},
366
+ "unified_val": {"utility": round(best_u, 4), "thr": best_t,
367
+ "F1": round(stats[0], 4), "duty": round(stats[1], 4)},
368
+ "ort_cpu_latency_ms": round(lat_ms, 2),
369
+ "train_rows": len(train_d), "val_rows": len(val_d),
370
+ "dataset": args.dataset,
371
+ }
372
+ (args.out / "metrics.json").write_text(json.dumps(metrics, indent=2))
373
+
374
+ repo = f"Tonic/challenge5-acoustic-{args.arch}-s{args.seed}"
375
+ from huggingface_hub import HfApi
376
+ api = HfApi()
377
+ api.create_repo(repo, repo_type="model", exist_ok=True, private=False)
378
+ try:
379
+ api.update_repo_settings(repo_id=repo, repo_type="model", gated="manual")
380
+ except Exception as e:
381
+ log(f"[publish] gating note: {str(e)[:80]}")
382
+ card = f"""---
383
+ license: apache-2.0
384
+ tags: [acoustic-wake, c-uas, challenge-5, onnx, drone-detection]
385
+ ---
386
+ # Challenge-5 acoustic wake model: {args.arch} (seed {args.seed})
387
+
388
+ Two-head CNN over 224x224 log-mel windows: `wake_logit` + `sound_class_logits` (4 classes),
389
+ ImageNet normalization baked in-graph, `wake_threshold` stored in ONNX metadata.
390
+
391
+ ## Metrics (challenge5 valid, {len(c5_valid)} clips)
392
+ ```json
393
+ {json.dumps(metrics['c5_valid'], indent=2)}
394
+ ```
395
+ ORT CPU latency: {lat_ms:.2f} ms/window | params {n_params/1e6:.2f}M | ONNX {onnx_mb:.2f} MB
396
+
397
+ Trained on {args.dataset} `windows_5s` (train, {len(train_d)} rows) + challenge5 train clips.
398
+ Predictions: `preds_valid.jsonl`, `preds_test.jsonl` (challenge submission format).
399
+ """
400
+ (args.out / "README.md").write_text(card)
401
+ api.upload_folder(folder_path=str(args.out), path_in_repo="", repo_id=repo, repo_type="model",
402
+ commit_message=f"{args.arch} seed {args.seed}: F1={f1v:.3f} acc={cls_acc_v:.3f} thr={best_t:.2f}")
403
+ log(f"[publish] https://huggingface.co/{repo}")
404
+
405
+ # append to shared leaderboard dataset repo
406
+ try:
407
+ api.create_repo(RESULTS_REPO, repo_type="dataset", exist_ok=True, private=False)
408
+ row = json.dumps(metrics) + "\n"
409
+ try:
410
+ old = api.hf_hub_download(RESULTS_REPO, "results.jsonl", repo_type="dataset")
411
+ data = open(old, "rb").read() + row.encode()
412
+ except Exception:
413
+ data = row.encode()
414
+ api.upload_file(path_or_fileobj=data, path_in_repo="results.jsonl",
415
+ repo_id=RESULTS_REPO, repo_type="dataset",
416
+ commit_message=f"add {args.arch} s{args.seed}")
417
+ log(f"[publish] leaderboard {RESULTS_REPO} updated")
418
+ except Exception as e:
419
+ log(f"[publish] leaderboard append failed: {str(e)[:120]}")
420
+
421
+ log(f"[done] arch={args.arch} seed={args.seed} F1={f1v:.4f} acc={cls_acc_v:.4f} duty={dutyv:.3f} thr={best_t:.2f} lat={lat_ms:.2f}ms")
422
+
423
+
424
+ if __name__ == "__main__":
425
+ main()