Tonic commited on
Commit
254b347
·
verified ·
1 Parent(s): 2f9894b

training script

Browse files
Files changed (1) hide show
  1. job_train.py +378 -0
job_train.py ADDED
@@ -0,0 +1,378 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = [
4
+ # "torch>=2.4",
5
+ # "torchvision>=0.19",
6
+ # "timm>=1.0",
7
+ # "datasets>=3.0",
8
+ # "pyarrow",
9
+ # "onnx>=1.16",
10
+ # "onnxruntime>=1.18",
11
+ # "pillow",
12
+ # "numpy",
13
+ # "huggingface_hub>=0.30",
14
+ # ]
15
+ # ///
16
+ """Challenge-5 acoustic wake model: train on the unified dataset, evaluate, export ONNX, publish to HF.
17
+
18
+ Runs as a Hugging Face Job (uv). One arch x one seed per job.
19
+
20
+ Usage: job_train.py --arch mobilenet_v3_small --seed 42 [--epochs 10] [--batch 64]
21
+ """
22
+ import argparse, io, json, math, os, random, time
23
+ from pathlib import Path
24
+
25
+ import numpy as np
26
+
27
+ ARCHES = ("mobilenet_v3_small", "shufflenet_v2_x0_5", "squeezenet1_1",
28
+ "efficientnet_lite0", "ghostnet_050", "cnn6_custom")
29
+ SOUND_CLASSES = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor")
30
+ CLASS_NAMES_UNIFIED = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor", "unknown")
31
+ DATASET = "Tonic/unified-drone-acoustic-cuas"
32
+ RESULTS_REPO = "Tonic/challenge5-acoustic-results"
33
+
34
+
35
+ def build_backbone(arch: str, pretrained: bool):
36
+ """Returns (backbone_module, feat_dim). Backbone maps [B,3,224,224] -> [B,feat_dim]."""
37
+ import torch.nn as nn
38
+ from torchvision import models
39
+ if arch == "mobilenet_v3_small":
40
+ m = models.mobilenet_v3_small(weights="IMAGENET1K_V1" if pretrained else None)
41
+ feat = m.classifier[0].in_features
42
+ m.classifier = nn.Identity()
43
+ return m, feat
44
+ if arch == "shufflenet_v2_x0_5":
45
+ m = models.shufflenet_v2_x0_5(weights="IMAGENET1K_V1" if pretrained else None)
46
+ feat = m.fc.in_features
47
+ m.fc = nn.Identity()
48
+ return m, feat
49
+ if arch == "squeezenet1_1":
50
+ m = models.squeezenet1_1(weights="IMAGENET1K_V1" if pretrained else None)
51
+ feat = 512
52
+ class Sq(nn.Module):
53
+ def __init__(self, m):
54
+ super().__init__()
55
+ self.features = m.features
56
+ def forward(self, x):
57
+ return self.features(x).mean(dim=(2, 3))
58
+ return Sq(m), feat
59
+ if arch in ("efficientnet_lite0", "ghostnet_050"):
60
+ import timm
61
+ names = {"efficientnet_lite0": ["tf_efficientnet_lite0", "efficientnet_lite0"],
62
+ "ghostnet_050": ["ghostnet_050", "ghostnet_100"]}[arch]
63
+ last = None
64
+ for n in names:
65
+ try:
66
+ m = timm.create_model(n, pretrained=pretrained, num_classes=0)
67
+ return m, m.num_features
68
+ except Exception as e:
69
+ last = e
70
+ raise RuntimeError(f"timm model for {arch} failed: {last}")
71
+ if arch == "cnn6_custom":
72
+ import torch.nn as nn
73
+ ch = (32, 64, 128, 256, 256, 512)
74
+ blocks = []
75
+ for c in ch:
76
+ blocks += [nn.Conv2d(blocks[-1].out_channels if blocks else 3, c, 3, padding=1, bias=False),
77
+ nn.BatchNorm2d(c), nn.ReLU(inplace=True),
78
+ nn.Conv2d(c, c, 3, padding=1, groups=c, bias=False), # depthwise
79
+ nn.BatchNorm2d(c), nn.ReLU(inplace=True),
80
+ nn.Conv2d(c, c, 1, bias=False), nn.BatchNorm2d(c), nn.ReLU(inplace=True),
81
+ nn.MaxPool2d(2)]
82
+ class CNN6(nn.Module):
83
+ def __init__(self):
84
+ super().__init__()
85
+ self.net = nn.Sequential(*blocks)
86
+ def forward(self, x):
87
+ return self.net(x).mean(dim=(2, 3))
88
+ return CNN6(), ch[-1]
89
+ raise ValueError(arch)
90
+
91
+
92
+ def main():
93
+ import torch
94
+ import torch.nn as nn
95
+
96
+ ap = argparse.ArgumentParser()
97
+ ap.add_argument("--arch", choices=ARCHES, required=True)
98
+ ap.add_argument("--seed", type=int, default=42)
99
+ ap.add_argument("--epochs", type=int, default=10)
100
+ ap.add_argument("--batch", type=int, default=64)
101
+ ap.add_argument("--lr", type=float, default=3e-4)
102
+ ap.add_argument("--max-train-rows", type=int, default=120000)
103
+ ap.add_argument("--val-rows", type=int, default=4000)
104
+ ap.add_argument("--pretrained", type=int, default=1)
105
+ ap.add_argument("--out", type=Path, default=Path("/tmp/c5_out"))
106
+ args = ap.parse_args()
107
+
108
+ random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
109
+ if torch.cuda.is_available():
110
+ torch.cuda.manual_seed_all(args.seed)
111
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
112
+ print(f"[job] arch={args.arch} seed={args.seed} device={device} "
113
+ f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
114
+
115
+ # ---------- data ----------
116
+ from datasets import load_dataset
117
+ t0 = time.time()
118
+ ds = load_dataset(DATASET, "windows_5s")
119
+ print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
120
+ unames = ds["train"].features["sound_class"].names
121
+ assert unames == list(CLASS_NAMES_UNIFIED), unames
122
+
123
+ def to_xy(split, source=None, labeled=True, cap=None):
124
+ d = ds[split]
125
+ if source is not None:
126
+ d = d.filter(lambda r: r["source"] == source, num_proc=4)
127
+ if labeled:
128
+ d = d.filter(lambda r: r["labeled"], num_proc=4)
129
+ if cap and len(d) > cap:
130
+ d = d.shuffle(seed=args.seed).select(range(cap))
131
+ return d
132
+
133
+ train_d = to_xy("train", cap=args.max_train_rows)
134
+ val_d = to_xy("validation", cap=args.val_rows)
135
+ c5_valid = to_xy("validation", source="challenge5", labeled=True)
136
+ c5_test = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
137
+ print(f"[data] train={len(train_d)} val={len(val_d)} c5_valid={len(c5_valid)} c5_test={len(c5_test)}", flush=True)
138
+
139
+ from PIL import Image
140
+ from torch.utils.data import DataLoader
141
+
142
+ def collate(batch):
143
+ xs = torch.stack([torch.from_numpy(np.asarray(r["spectrogram"], dtype=np.float32) / 255.0)
144
+ .permute(2, 0, 1) for r in batch])
145
+ wake = torch.tensor([float(r["has_drone_audio"]) for r in batch])
146
+ cls = torch.tensor([SOUND_CLASSES.index(CLASS_NAMES_UNIFIED[r["sound_class"]])
147
+ if r["sound_class"] < 4 else 0 for r in batch], dtype=torch.long)
148
+ ids = [r["sample_id"] for r in batch]
149
+ return xs, wake, cls, ids
150
+
151
+ train_loader = DataLoader(train_d, batch_size=args.batch, shuffle=True, num_workers=4,
152
+ collate_fn=collate, drop_last=True)
153
+ val_loader = DataLoader(val_d, batch_size=256, shuffle=False, num_workers=4, collate_fn=collate)
154
+ c5v_loader = DataLoader(c5_valid, batch_size=16, shuffle=False, collate_fn=collate)
155
+ c5t_loader = DataLoader(c5_test, batch_size=16, shuffle=False, collate_fn=collate)
156
+
157
+ # ---------- model ----------
158
+ class WakeCNN(nn.Module):
159
+ def __init__(self, arch, pretrained):
160
+ super().__init__()
161
+ self.register_buffer("mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
162
+ self.register_buffer("std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
163
+ self.backbone, feat = build_backbone(arch, pretrained)
164
+ self.head_wake = nn.Linear(feat, 1)
165
+ self.head_class = nn.Linear(feat, 4)
166
+
167
+ def forward(self, x):
168
+ x = (x - self.mean) / self.std
169
+ f = self.backbone(x)
170
+ return self.head_wake(f).squeeze(-1), self.head_class(f)
171
+
172
+ model = WakeCNN(args.arch, bool(args.pretrained)).to(device)
173
+ n_params = sum(p.numel() for p in model.parameters())
174
+ print(f"[model] {args.arch} params={n_params/1e6:.2f}M", flush=True)
175
+
176
+ opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
177
+ sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs)
178
+ pos = float(np.mean([r["has_drone_audio"] for r in train_d.select(range(2000))]))
179
+ pos_weight = torch.tensor([min(2.0, max(1.0, ((1 - pos) / max(pos, 1e-3)) ** 0.5))], device=device)
180
+ print(f"[train] pos_rate~{pos:.3f} pos_weight={float(pos_weight):.3f}", flush=True)
181
+ bce = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
182
+ ce = nn.CrossEntropyLoss()
183
+
184
+ def eval_loader(loader, model):
185
+ model.eval()
186
+ W, Y, C, P, ids = [], [], [], [], []
187
+ with torch.no_grad():
188
+ for xs, wake, cls, sids in loader:
189
+ wl, cl = model(xs.to(device))
190
+ W.append(torch.sigmoid(wl).cpu()); Y.append(wake); C.append(cls)
191
+ P.append(cl.argmax(1).cpu()); ids += sids
192
+ return (torch.cat(W).numpy(), torch.cat(Y).numpy(), torch.cat(C).numpy(),
193
+ torch.cat(P).numpy(), ids)
194
+
195
+ def sweep(W, Y, grid=None):
196
+ grid = grid or [round(0.30 + 0.05 * i, 2) for i in range(11)]
197
+ best = (-1, 0.5, None)
198
+ for t in grid:
199
+ p = W >= t
200
+ tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
201
+ f1 = 2 * tp / max(1, 2 * tp + fp + fn)
202
+ u = f1 - 0.15 * float(p.mean())
203
+ if u > best[0]:
204
+ best = (u, t, (f1, float(p.mean()), tp, fp, fn))
205
+ return best
206
+
207
+ args.out.mkdir(parents=True, exist_ok=True)
208
+ log_path = args.out / "train_log.txt"
209
+ logf = open(log_path, "a")
210
+
211
+ def log(msg):
212
+ print(msg, flush=True)
213
+ logf.write(msg + "\n"); logf.flush()
214
+
215
+ best_u, best_t, best_state = -1, 0.5, None
216
+ for ep in range(1, args.epochs + 1):
217
+ model.train()
218
+ t0 = time.time()
219
+ n_seen = 0
220
+ scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
221
+ for xs, wake, cls, _ in train_loader:
222
+ opt.zero_grad()
223
+ with torch.amp.autocast("cuda", enabled=device.type == "cuda"):
224
+ wl, cl = model(xs.to(device))
225
+ loss = bce(wl, wake.to(device)) + 0.5 * ce(cl, cls.to(device))
226
+ scaler.scale(loss).backward()
227
+ scaler.step(opt)
228
+ scaler.update()
229
+ n_seen += len(xs)
230
+ sched.step()
231
+ W, Y, C, P, _ = eval_loader(val_loader, model)
232
+ u, t, stats = sweep(W, Y)
233
+ Wv, Yv, Cv, Pv, _ = eval_loader(c5v_loader, model)
234
+ uv, tv, statsv = sweep(Wv, Yv)
235
+ cls_acc_v = float((Pv == Cv).mean())
236
+ log(f"[ep {ep}/{args.epochs}] {time.time()-t0:.0f}s rows={n_seen} "
237
+ f"unified_val: F1={stats[0]:.4f} duty={stats[1]:.3f} @thr={t:.2f} u={u:.4f} | "
238
+ f"c5_valid: F1={statsv[0]:.4f} duty={statsv[1]:.3f} @thr={tv:.2f} cls_acc={cls_acc_v:.3f}")
239
+ if u > best_u:
240
+ best_u, best_t = u, t
241
+ best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
242
+ torch.save(best_state, args.out / "best.pt")
243
+
244
+ if best_state is not None:
245
+ model.load_state_dict(best_state)
246
+ model.eval().cpu()
247
+
248
+ # final metrics
249
+ W, Y, C, P, _ = eval_loader(val_loader, model)
250
+ u, t, stats = sweep(W, Y)
251
+ Wv, Yv, Cv, Pv, ids_v = eval_loader(c5v_loader, model)
252
+ pv = Wv >= best_t
253
+ tp = int((pv & (Yv > 0.5)).sum()); fp = int((pv & (Yv <= 0.5)).sum()); fn = int((~pv & (Yv > 0.5)).sum())
254
+ f1v = 2 * tp / max(1, 2 * tp + fp + fn)
255
+ dutyv = float(pv.mean())
256
+ cls_acc_v = float((Pv == Cv).mean())
257
+ log(f"[final] thr={best_t:.2f} c5_valid wake_F1={f1v:.4f} duty={dutyv:.3f} cls_acc={cls_acc_v:.4f} "
258
+ f"(tp={tp} fp={fp} fn={fn}) unified_val_u={best_u:.4f}")
259
+
260
+ # ---------- ONNX export with threshold metadata ----------
261
+ import onnx
262
+ dummy = torch.rand(1, 3, 224, 224)
263
+ onnx_path = args.out / "audio_wake_cnn.onnx"
264
+ torch.onnx.export(model, (dummy,), str(onnx_path), opset_version=17,
265
+ input_names=["spectrogram"], output_names=["wake_logit", "sound_class_logits"],
266
+ dynamo=False)
267
+ m = onnx.load(str(onnx_path))
268
+ md = {e.key: e.value for e in m.metadata_props}
269
+ md["wake_threshold"] = str(best_t)
270
+ md["arch"] = args.arch
271
+ onnx.helper.set_model_props(m, md)
272
+ onnx.save(m, str(onnx_path))
273
+ onnx_mb = onnx_path.stat().st_size / 1e6
274
+ log(f"[onnx] saved {onnx_mb:.2f} MB, threshold={best_t:.2f}")
275
+
276
+ # ---------- ORT latency ----------
277
+ import onnxruntime as ort
278
+ sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
279
+ x = np.random.rand(1, 3, 224, 224).astype(np.float32)
280
+ for _ in range(10):
281
+ sess.run(None, {"spectrogram": x})
282
+ ts = []
283
+ for _ in range(100):
284
+ t0 = time.time()
285
+ sess.run(None, {"spectrogram": x})
286
+ ts.append((time.time() - t0) * 1000)
287
+ lat_ms = float(np.mean(ts))
288
+ log(f"[ort] CPU latency mean={lat_ms:.2f} ms p50={float(np.percentile(ts,50)):.2f}")
289
+
290
+ # ---------- predictions on challenge5 valid + test ----------
291
+ def predict(loader):
292
+ outs = []
293
+ with torch.no_grad():
294
+ for xs, wake, cls, sids in loader:
295
+ wl, cl = model(xs)
296
+ wp = torch.sigmoid(wl).numpy()
297
+ cp = cl.argmax(1).numpy()
298
+ for sid, w, c in zip(sids, wp, cp):
299
+ clip = sid.split("__")[1].replace(".wav", "")
300
+ outs.append({"id": clip, "wake_triggered": bool(w >= best_t),
301
+ "sound_class": SOUND_CLASSES[int(c)]})
302
+ return outs
303
+
304
+ preds_valid = predict(c5v_loader)
305
+ preds_test = predict(c5t_loader)
306
+ (args.out / "preds_valid.jsonl").write_text(
307
+ "\n".join(json.dumps(p) for p in preds_valid) + "\n")
308
+ (args.out / "preds_test.jsonl").write_text(
309
+ "\n".join(json.dumps(p) for p in preds_test) + "\n")
310
+
311
+ # ---------- publish ----------
312
+ metrics = {
313
+ "arch": args.arch, "seed": args.seed, "epochs": args.epochs, "batch": args.batch,
314
+ "lr": args.lr, "params": n_params, "onnx_mb": round(onnx_mb, 2),
315
+ "wake_threshold": best_t,
316
+ "c5_valid": {"wake_F1": round(f1v, 4), "optical_duty_cycle": round(dutyv, 4),
317
+ "sound_class_accuracy": round(cls_acc_v, 4),
318
+ "optical_inactivity_pct": round((1 - dutyv) * 100, 2), "tp": tp, "fp": fp, "fn": fn},
319
+ "unified_val": {"utility": round(best_u, 4), "thr": best_t,
320
+ "F1": round(stats[0], 4), "duty": round(stats[1], 4)},
321
+ "ort_cpu_latency_ms": round(lat_ms, 2),
322
+ "train_rows": len(train_d), "val_rows": len(val_d),
323
+ "dataset": DATASET,
324
+ }
325
+ (args.out / "metrics.json").write_text(json.dumps(metrics, indent=2))
326
+
327
+ repo = f"Tonic/challenge5-acoustic-{args.arch}-s{args.seed}"
328
+ from huggingface_hub import HfApi
329
+ api = HfApi()
330
+ api.create_repo(repo, repo_type="model", exist_ok=True, private=False)
331
+ try:
332
+ api.update_repo_settings(repo_id=repo, repo_type="model", gated="manual")
333
+ except Exception as e:
334
+ log(f"[publish] gating note: {str(e)[:80]}")
335
+ card = f"""---
336
+ license: apache-2.0
337
+ tags: [acoustic-wake, c-uas, challenge-5, onnx, drone-detection]
338
+ ---
339
+ # Challenge-5 acoustic wake model: {args.arch} (seed {args.seed})
340
+
341
+ Two-head CNN over 224x224 log-mel windows: `wake_logit` + `sound_class_logits` (4 classes),
342
+ ImageNet normalization baked in-graph, `wake_threshold` stored in ONNX metadata.
343
+
344
+ ## Metrics (challenge5 valid, {len(c5_valid)} clips)
345
+ ```json
346
+ {json.dumps(metrics['c5_valid'], indent=2)}
347
+ ```
348
+ ORT CPU latency: {lat_ms:.2f} ms/window | params {n_params/1e6:.2f}M | ONNX {onnx_mb:.2f} MB
349
+
350
+ Trained on {DATASET} `windows_5s` (train, {len(train_d)} rows) + challenge5 train clips.
351
+ Predictions: `preds_valid.jsonl`, `preds_test.jsonl` (challenge submission format).
352
+ """
353
+ (args.out / "README.md").write_text(card)
354
+ api.upload_folder(folder_path=str(args.out), path_in_repo="", repo_id=repo, repo_type="model",
355
+ commit_message=f"{args.arch} seed {args.seed}: F1={f1v:.3f} acc={cls_acc_v:.3f} thr={best_t:.2f}")
356
+ log(f"[publish] https://huggingface.co/{repo}")
357
+
358
+ # append to shared leaderboard dataset repo
359
+ try:
360
+ api.create_repo(RESULTS_REPO, repo_type="dataset", exist_ok=True, private=False)
361
+ row = json.dumps(metrics) + "\n"
362
+ try:
363
+ old = api.hf_hub_download(RESULTS_REPO, "results.jsonl", repo_type="dataset")
364
+ data = open(old, "rb").read() + row.encode()
365
+ except Exception:
366
+ data = row.encode()
367
+ api.upload_file(path_or_fileobj=data, path_in_repo="results.jsonl",
368
+ repo_id=RESULTS_REPO, repo_type="dataset",
369
+ commit_message=f"add {args.arch} s{args.seed}")
370
+ log(f"[publish] leaderboard {RESULTS_REPO} updated")
371
+ except Exception as e:
372
+ log(f"[publish] leaderboard append failed: {str(e)[:120]}")
373
+
374
+ log(f"[done] arch={args.arch} seed={args.seed} F1={f1v:.4f} acc={cls_acc_v:.4f} duty={dutyv:.3f} thr={best_t:.2f} lat={lat_ms:.2f}ms")
375
+
376
+
377
+ if __name__ == "__main__":
378
+ main()