Download job_train.py from Tonic/challenge5-acoustic-code: direct link, hf CLI and curl.
- Browser
- Download file 19.7 kB
-
https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/job_train.py
- Command line
-
hf download hf://Tonic/challenge5-acoustic-code/job_train.py
-
curl -L -o job_train.py https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/job_train.py
19.7 kB
| # /// script | |
| # requires-python = ">=3.11" | |
| # dependencies = [ | |
| # "torch>=2.4", | |
| # "torchvision>=0.19", | |
| # "timm>=1.0", | |
| # "datasets>=3.0", | |
| # "pyarrow", | |
| # "onnx>=1.16", | |
| # "onnxruntime>=1.18", | |
| # "pillow", | |
| # "numpy", | |
| # "huggingface_hub>=0.30", | |
| # ] | |
| # /// | |
| """Challenge-5 acoustic wake model: train on the unified dataset, evaluate, export ONNX, publish to HF. | |
| Runs as a Hugging Face Job (uv). One arch x one seed per job. | |
| Usage: job_train.py --arch mobilenet_v3_small --seed 42 [--epochs 10] [--batch 64] | |
| """ | |
| import argparse, io, json, math, os, random, time | |
| from pathlib import Path | |
| import numpy as np | |
| ARCHES = ("mobilenet_v3_small", "shufflenet_v2_x0_5", "squeezenet1_1", | |
| "efficientnet_lite0", "ghostnet_050", "cnn6_custom") | |
| SOUND_CLASSES = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor") | |
| CLASS_NAMES_UNIFIED = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor", "unknown") | |
| DATASET = "Tonic/unified-drone-acoustic-cuas" | |
| RESULTS_REPO = "Tonic/challenge5-acoustic-results" | |
| def build_backbone(arch: str, pretrained: bool): | |
| """Returns (backbone_module, feat_dim). Backbone maps [B,3,224,224] -> [B,feat_dim].""" | |
| import torch.nn as nn | |
| from torchvision import models | |
| if arch == "mobilenet_v3_small": | |
| m = models.mobilenet_v3_small(weights="IMAGENET1K_V1" if pretrained else None) | |
| feat = m.classifier[0].in_features | |
| m.classifier = nn.Identity() | |
| return m, feat | |
| if arch == "shufflenet_v2_x0_5": | |
| m = models.shufflenet_v2_x0_5(weights="IMAGENET1K_V1" if pretrained else None) | |
| feat = m.fc.in_features | |
| m.fc = nn.Identity() | |
| return m, feat | |
| if arch == "squeezenet1_1": | |
| m = models.squeezenet1_1(weights="IMAGENET1K_V1" if pretrained else None) | |
| feat = 512 | |
| class Sq(nn.Module): | |
| def __init__(self, m): | |
| super().__init__() | |
| self.features = m.features | |
| def forward(self, x): | |
| return self.features(x).mean(dim=(2, 3)) | |
| return Sq(m), feat | |
| if arch in ("efficientnet_lite0", "ghostnet_050"): | |
| import timm | |
| names = {"efficientnet_lite0": ["tf_efficientnet_lite0", "efficientnet_lite0"], | |
| "ghostnet_050": ["ghostnet_050", "ghostnet_100"]}[arch] | |
| last = None | |
| import torch | |
| for n in names: | |
| try: | |
| m = timm.create_model(n, pretrained=pretrained, num_classes=0) | |
| m.eval() | |
| with torch.no_grad(): | |
| feat = int(m(torch.zeros(1, 3, 224, 224)).shape[1]) | |
| return m, feat | |
| except Exception as e: | |
| last = e | |
| raise RuntimeError(f"timm model for {arch} failed: {last}") | |
| if arch == "cnn6_custom": | |
| import torch.nn as nn | |
| ch = (32, 64, 128, 256, 256, 512) | |
| blocks = [] | |
| prev = 3 | |
| for c in ch: | |
| blocks += [nn.Conv2d(prev, c, 3, padding=1, bias=False), | |
| nn.BatchNorm2d(c), nn.ReLU(inplace=True), | |
| nn.Conv2d(c, c, 3, padding=1, groups=c, bias=False), # depthwise | |
| nn.BatchNorm2d(c), nn.ReLU(inplace=True), | |
| nn.Conv2d(c, c, 1, bias=False), nn.BatchNorm2d(c), nn.ReLU(inplace=True), | |
| nn.MaxPool2d(2)] | |
| prev = c | |
| class CNN6(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.net = nn.Sequential(*blocks) | |
| def forward(self, x): | |
| return self.net(x).mean(dim=(2, 3)) | |
| return CNN6(), ch[-1] | |
| raise ValueError(arch) | |
| def main(): | |
| import torch | |
| import torch.nn as nn | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--arch", choices=ARCHES, required=True) | |
| ap.add_argument("--seed", type=int, default=42) | |
| ap.add_argument("--epochs", type=int, default=15) | |
| ap.add_argument("--batch", type=int, default=64) | |
| ap.add_argument("--lr", type=float, default=3e-4) | |
| ap.add_argument("--dataset", type=str, default=DATASET) | |
| ap.add_argument("--max-train-rows", type=int, default=160000) | |
| ap.add_argument("--val-rows", type=int, default=4000) | |
| ap.add_argument("--pretrained", type=int, default=1) | |
| ap.add_argument("--out", type=Path, default=Path("/tmp/c5_out")) | |
| args = ap.parse_args() | |
| random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed) | |
| if torch.cuda.is_available(): | |
| torch.cuda.manual_seed_all(args.seed) | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| print(f"[job] arch={args.arch} seed={args.seed} device={device} " | |
| f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True) | |
| # ---------- data ---------- | |
| from datasets import load_dataset, concatenate_datasets | |
| t0 = time.time() | |
| ds = load_dataset(args.dataset, "windows_5s") | |
| print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True) | |
| unames = ds["train"].features["sound_class"].names | |
| assert unames == list(CLASS_NAMES_UNIFIED), unames | |
| def to_xy(split, source=None, labeled=True, cap=None): | |
| d = ds[split] | |
| if source is not None: | |
| d = d.filter(lambda r: r["source"] == source, num_proc=4) | |
| if labeled: | |
| d = d.filter(lambda r: r["labeled"], num_proc=4) | |
| if cap and len(d) > cap: | |
| d = d.shuffle(seed=args.seed).select(range(cap)) | |
| return d | |
| train_d = to_xy("train", cap=args.max_train_rows) | |
| # oversample the 48 challenge5 train clips so the challenge distribution is not | |
| # drowned out by the (easy) external corpora | |
| c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4) | |
| rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train)))) | |
| c5_up = concatenate_datasets([c5_train] * (rep + 1)) | |
| train_d = concatenate_datasets([train_d, c5_up]) | |
| print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True) | |
| val_d = to_xy("validation", cap=args.val_rows) | |
| c5_valid = to_xy("validation", source="challenge5", labeled=True) | |
| c5_test = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=4) | |
| print(f"[data] train={len(train_d)} val={len(val_d)} c5_valid={len(c5_valid)} c5_test={len(c5_test)}", flush=True) | |
| from PIL import Image | |
| from torch.utils.data import DataLoader | |
| def augment(img, is_c5): | |
| # img: HxWx3 float 0..1; gain + freq/time masks (stronger for challenge rows) | |
| g = random.uniform(0.4, 2.0) if is_c5 else random.uniform(0.8, 1.25) | |
| img = img * g | |
| if is_c5 and random.random() < 0.7: | |
| h, w = img.shape[0], img.shape[1] | |
| fm = random.randint(0, max(1, h // 8)); f0 = random.randint(0, h - fm) | |
| img[f0:f0 + fm] = 0 | |
| tm = random.randint(0, max(1, w // 8)); t0 = random.randint(0, w - tm) | |
| img[:, t0:t0 + tm] = 0 | |
| return np.clip(img, 0, 1) | |
| def collate(batch): | |
| xs = torch.stack([torch.from_numpy(augment(np.asarray(r["spectrogram"], dtype=np.float32) / 255.0, | |
| r["source"] == "challenge5")) | |
| .permute(2, 0, 1) for r in batch]) | |
| wake = torch.tensor([float(r["has_drone_audio"]) for r in batch]) | |
| cls = torch.tensor([SOUND_CLASSES.index(CLASS_NAMES_UNIFIED[r["sound_class"]]) | |
| if r["sound_class"] < 4 else 0 for r in batch], dtype=torch.long) | |
| ids = [r["sample_id"] for r in batch] | |
| return xs, wake, cls, ids | |
| train_loader = DataLoader(train_d, batch_size=args.batch, shuffle=True, num_workers=4, | |
| collate_fn=collate, drop_last=True) | |
| val_loader = DataLoader(val_d, batch_size=256, shuffle=False, num_workers=4, collate_fn=collate) | |
| c5v_loader = DataLoader(c5_valid, batch_size=16, shuffle=False, collate_fn=collate) | |
| c5t_loader = DataLoader(c5_test, batch_size=16, shuffle=False, collate_fn=collate) | |
| # ---------- model ---------- | |
| class WakeCNN(nn.Module): | |
| def __init__(self, arch, pretrained): | |
| super().__init__() | |
| self.register_buffer("mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) | |
| self.register_buffer("std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) | |
| self.backbone, feat = build_backbone(arch, pretrained) | |
| self.head_wake = nn.Linear(feat, 1) | |
| self.head_class = nn.Linear(feat, 4) | |
| def forward(self, x): | |
| x = (x - self.mean) / self.std | |
| f = self.backbone(x) | |
| return self.head_wake(f).squeeze(-1), self.head_class(f) | |
| model = WakeCNN(args.arch, bool(args.pretrained)).to(device) | |
| n_params = sum(p.numel() for p in model.parameters()) | |
| print(f"[model] {args.arch} params={n_params/1e6:.2f}M", flush=True) | |
| opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4) | |
| sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs) | |
| pos = float(np.mean([r["has_drone_audio"] for r in train_d.select(range(2000))])) | |
| pos_weight = torch.tensor([min(2.0, max(1.0, ((1 - pos) / max(pos, 1e-3)) ** 0.5))], device=device) | |
| print(f"[train] pos_rate~{pos:.3f} pos_weight={float(pos_weight):.3f}", flush=True) | |
| bce = nn.BCEWithLogitsLoss(pos_weight=pos_weight) | |
| ce = nn.CrossEntropyLoss() | |
| def eval_loader(loader, model): | |
| model.eval() | |
| dev = next(model.parameters()).device | |
| W, Y, C, P, ids = [], [], [], [], [] | |
| with torch.no_grad(): | |
| for xs, wake, cls, sids in loader: | |
| wl, cl = model(xs.to(dev)) | |
| W.append(torch.sigmoid(wl).cpu()); Y.append(wake); C.append(cls) | |
| P.append(cl.argmax(1).cpu()); ids += sids | |
| return (torch.cat(W).numpy(), torch.cat(Y).numpy(), torch.cat(C).numpy(), | |
| torch.cat(P).numpy(), ids) | |
| def sweep(W, Y, grid=None): | |
| grid = grid or [round(0.05 + 0.05 * i, 2) for i in range(18)] | |
| best = (-1, 0.5, None) | |
| for t in grid: | |
| p = W >= t | |
| tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum()) | |
| f1 = 2 * tp / max(1, 2 * tp + fp + fn) | |
| u = f1 - 0.15 * float(p.mean()) | |
| if u > best[0]: | |
| best = (u, t, (f1, float(p.mean()), tp, fp, fn)) | |
| return best | |
| def sweep_with_c5(W, Y, Wv, Yv): | |
| """Pick threshold maximizing unified-val utility; among thresholds within | |
| 0.03 utility of the best, tie-break by challenge5-valid F1 (8 clips).""" | |
| grid = [round(0.05 + 0.05 * i, 2) for i in range(18)] | |
| scored = [] | |
| for t in grid: | |
| p = W >= t | |
| tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum()) | |
| f1 = 2 * tp / max(1, 2 * tp + fp + fn) | |
| u = f1 - 0.15 * float(p.mean()) | |
| pv = Wv >= t | |
| tpv = int((pv & (Yv > 0.5)).sum()); fpv = int((pv & (Yv <= 0.5)).sum()); fnv = int((~pv & (Yv > 0.5)).sum()) | |
| f1v = 2 * tpv / max(1, 2 * tpv + fpv + fnv) | |
| scored.append((u, t, f1, float(p.mean()), f1v, pv.mean())) | |
| best_u = max(x[0] for x in scored) | |
| cands = [x for x in scored if x[0] >= best_u - 0.03] | |
| cands.sort(key=lambda x: (x[4], -x[5])) # best c5 F1, then lowest duty | |
| u, t, f1, duty, f1v, dutyv = cands[0] | |
| return u, t, (f1, duty, f1v, dutyv) | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| log_path = args.out / "train_log.txt" | |
| logf = open(log_path, "a") | |
| def log(msg): | |
| print(msg, flush=True) | |
| logf.write(msg + "\n"); logf.flush() | |
| best_u, best_t, best_state = -1, 0.5, None | |
| for ep in range(1, args.epochs + 1): | |
| model.train() | |
| t0 = time.time() | |
| n_seen = 0 | |
| scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda") | |
| for xs, wake, cls, _ in train_loader: | |
| opt.zero_grad() | |
| with torch.amp.autocast("cuda", enabled=device.type == "cuda"): | |
| wl, cl = model(xs.to(device)) | |
| loss = bce(wl, wake.to(device)) + 0.5 * ce(cl, cls.to(device)) | |
| scaler.scale(loss).backward() | |
| scaler.step(opt) | |
| scaler.update() | |
| n_seen += len(xs) | |
| sched.step() | |
| W, Y, C, P, _ = eval_loader(val_loader, model) | |
| Wv, Yv, Cv, Pv, _ = eval_loader(c5v_loader, model) | |
| u, t, stats = sweep_with_c5(W, Y, Wv, Yv) | |
| cls_acc_v = float((Pv == Cv).mean()) | |
| log(f"[ep {ep}/{args.epochs}] {time.time()-t0:.0f}s rows={n_seen} " | |
| f"unified_val: F1={stats[0]:.4f} duty={stats[1]:.3f} @thr={t:.2f} u={u:.4f} | " | |
| f"c5_valid: F1={stats[2]:.4f} duty={stats[3]:.3f} cls_acc={cls_acc_v:.3f}") | |
| if u > best_u: | |
| best_u, best_t = u, t | |
| best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} | |
| torch.save(best_state, args.out / "best.pt") | |
| if best_state is not None: | |
| model.load_state_dict(best_state) | |
| model.eval().cpu() | |
| # final metrics | |
| W, Y, C, P, _ = eval_loader(val_loader, model) | |
| Wv, Yv, Cv, Pv, ids_v = eval_loader(c5v_loader, model) | |
| u, t, stats = sweep_with_c5(W, Y, Wv, Yv) | |
| pv = Wv >= best_t | |
| tp = int((pv & (Yv > 0.5)).sum()); fp = int((pv & (Yv <= 0.5)).sum()); fn = int((~pv & (Yv > 0.5)).sum()) | |
| f1v = 2 * tp / max(1, 2 * tp + fp + fn) | |
| dutyv = float(pv.mean()) | |
| cls_acc_v = float((Pv == Cv).mean()) | |
| log(f"[final] thr={best_t:.2f} c5_valid wake_F1={f1v:.4f} duty={dutyv:.3f} cls_acc={cls_acc_v:.4f} " | |
| f"(tp={tp} fp={fp} fn={fn}) unified_val_u={best_u:.4f}") | |
| # ---------- ONNX export with threshold metadata ---------- | |
| import onnx | |
| dummy = torch.rand(1, 3, 224, 224) | |
| onnx_path = args.out / "audio_wake_cnn.onnx" | |
| torch.onnx.export(model, (dummy,), str(onnx_path), opset_version=17, | |
| input_names=["spectrogram"], output_names=["wake_logit", "sound_class_logits"], | |
| dynamo=False) | |
| m = onnx.load(str(onnx_path)) | |
| md = {e.key: e.value for e in m.metadata_props} | |
| md["wake_threshold"] = str(best_t) | |
| md["arch"] = args.arch | |
| onnx.helper.set_model_props(m, md) | |
| onnx.save(m, str(onnx_path)) | |
| onnx_mb = onnx_path.stat().st_size / 1e6 | |
| log(f"[onnx] saved {onnx_mb:.2f} MB, threshold={best_t:.2f}") | |
| # ---------- ORT latency ---------- | |
| import onnxruntime as ort | |
| sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"]) | |
| x = np.random.rand(1, 3, 224, 224).astype(np.float32) | |
| for _ in range(10): | |
| sess.run(None, {"spectrogram": x}) | |
| ts = [] | |
| for _ in range(100): | |
| t0 = time.time() | |
| sess.run(None, {"spectrogram": x}) | |
| ts.append((time.time() - t0) * 1000) | |
| lat_ms = float(np.mean(ts)) | |
| log(f"[ort] CPU latency mean={lat_ms:.2f} ms p50={float(np.percentile(ts,50)):.2f}") | |
| # ---------- predictions on challenge5 valid + test ---------- | |
| def predict(loader): | |
| outs = [] | |
| with torch.no_grad(): | |
| for xs, wake, cls, sids in loader: | |
| wl, cl = model(xs) | |
| wp = torch.sigmoid(wl).numpy() | |
| cp = cl.argmax(1).numpy() | |
| for sid, w, c in zip(sids, wp, cp): | |
| clip = sid.split("__")[1].replace(".wav", "") | |
| outs.append({"id": clip, "wake_triggered": bool(w >= best_t), | |
| "sound_class": SOUND_CLASSES[int(c)]}) | |
| return outs | |
| preds_valid = predict(c5v_loader) | |
| preds_test = predict(c5t_loader) | |
| (args.out / "preds_valid.jsonl").write_text( | |
| "\n".join(json.dumps(p) for p in preds_valid) + "\n") | |
| (args.out / "preds_test.jsonl").write_text( | |
| "\n".join(json.dumps(p) for p in preds_test) + "\n") | |
| # ---------- publish ---------- | |
| metrics = { | |
| "arch": args.arch, "seed": args.seed, "epochs": args.epochs, "batch": args.batch, | |
| "lr": args.lr, "params": n_params, "onnx_mb": round(onnx_mb, 2), | |
| "wake_threshold": best_t, | |
| "c5_valid": {"wake_F1": round(f1v, 4), "optical_duty_cycle": round(dutyv, 4), | |
| "sound_class_accuracy": round(cls_acc_v, 4), | |
| "optical_inactivity_pct": round((1 - dutyv) * 100, 2), "tp": tp, "fp": fp, "fn": fn}, | |
| "unified_val": {"utility": round(best_u, 4), "thr": best_t, | |
| "F1": round(stats[0], 4), "duty": round(stats[1], 4)}, | |
| "ort_cpu_latency_ms": round(lat_ms, 2), | |
| "train_rows": len(train_d), "val_rows": len(val_d), | |
| "dataset": args.dataset, | |
| } | |
| (args.out / "metrics.json").write_text(json.dumps(metrics, indent=2)) | |
| repo = f"Tonic/challenge5-acoustic-{args.arch}-s{args.seed}" | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| api.create_repo(repo, repo_type="model", exist_ok=True, private=False) | |
| try: | |
| api.update_repo_settings(repo_id=repo, repo_type="model", gated="manual") | |
| except Exception as e: | |
| log(f"[publish] gating note: {str(e)[:80]}") | |
| card = f"""--- | |
| license: apache-2.0 | |
| tags: [acoustic-wake, c-uas, challenge-5, onnx, drone-detection] | |
| --- | |
| # Challenge-5 acoustic wake model: {args.arch} (seed {args.seed}) | |
| Two-head CNN over 224x224 log-mel windows: `wake_logit` + `sound_class_logits` (4 classes), | |
| ImageNet normalization baked in-graph, `wake_threshold` stored in ONNX metadata. | |
| ## Metrics (challenge5 valid, {len(c5_valid)} clips) | |
| ```json | |
| {json.dumps(metrics['c5_valid'], indent=2)} | |
| ``` | |
| ORT CPU latency: {lat_ms:.2f} ms/window | params {n_params/1e6:.2f}M | ONNX {onnx_mb:.2f} MB | |
| Trained on {args.dataset} `windows_5s` (train, {len(train_d)} rows) + challenge5 train clips. | |
| Predictions: `preds_valid.jsonl`, `preds_test.jsonl` (challenge submission format). | |
| """ | |
| (args.out / "README.md").write_text(card) | |
| api.upload_folder(folder_path=str(args.out), path_in_repo="", repo_id=repo, repo_type="model", | |
| commit_message=f"{args.arch} seed {args.seed}: F1={f1v:.3f} acc={cls_acc_v:.3f} thr={best_t:.2f}") | |
| log(f"[publish] https://huggingface.co/{repo}") | |
| # append to shared leaderboard dataset repo | |
| try: | |
| api.create_repo(RESULTS_REPO, repo_type="dataset", exist_ok=True, private=False) | |
| row = json.dumps(metrics) + "\n" | |
| try: | |
| old = api.hf_hub_download(RESULTS_REPO, "results.jsonl", repo_type="dataset") | |
| data = open(old, "rb").read() + row.encode() | |
| except Exception: | |
| data = row.encode() | |
| api.upload_file(path_or_fileobj=data, path_in_repo="results.jsonl", | |
| repo_id=RESULTS_REPO, repo_type="dataset", | |
| commit_message=f"add {args.arch} s{args.seed}") | |
| log(f"[publish] leaderboard {RESULTS_REPO} updated") | |
| except Exception as e: | |
| log(f"[publish] leaderboard append failed: {str(e)[:120]}") | |
| log(f"[done] arch={args.arch} seed={args.seed} F1={f1v:.4f} acc={cls_acc_v:.4f} duty={dutyv:.3f} thr={best_t:.2f} lat={lat_ms:.2f}ms") | |
| if __name__ == "__main__": | |
| main() | |