training script
Browse files- job_train.py +378 -0
job_train.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.11"
|
| 3 |
+
# dependencies = [
|
| 4 |
+
# "torch>=2.4",
|
| 5 |
+
# "torchvision>=0.19",
|
| 6 |
+
# "timm>=1.0",
|
| 7 |
+
# "datasets>=3.0",
|
| 8 |
+
# "pyarrow",
|
| 9 |
+
# "onnx>=1.16",
|
| 10 |
+
# "onnxruntime>=1.18",
|
| 11 |
+
# "pillow",
|
| 12 |
+
# "numpy",
|
| 13 |
+
# "huggingface_hub>=0.30",
|
| 14 |
+
# ]
|
| 15 |
+
# ///
|
| 16 |
+
"""Challenge-5 acoustic wake model: train on the unified dataset, evaluate, export ONNX, publish to HF.
|
| 17 |
+
|
| 18 |
+
Runs as a Hugging Face Job (uv). One arch x one seed per job.
|
| 19 |
+
|
| 20 |
+
Usage: job_train.py --arch mobilenet_v3_small --seed 42 [--epochs 10] [--batch 64]
|
| 21 |
+
"""
|
| 22 |
+
import argparse, io, json, math, os, random, time
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
|
| 27 |
+
ARCHES = ("mobilenet_v3_small", "shufflenet_v2_x0_5", "squeezenet1_1",
|
| 28 |
+
"efficientnet_lite0", "ghostnet_050", "cnn6_custom")
|
| 29 |
+
SOUND_CLASSES = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor")
|
| 30 |
+
CLASS_NAMES_UNIFIED = ("drone_buzz", "vehicle_mechanical", "explosion", "ambient_outdoor", "unknown")
|
| 31 |
+
DATASET = "Tonic/unified-drone-acoustic-cuas"
|
| 32 |
+
RESULTS_REPO = "Tonic/challenge5-acoustic-results"
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def build_backbone(arch: str, pretrained: bool):
|
| 36 |
+
"""Returns (backbone_module, feat_dim). Backbone maps [B,3,224,224] -> [B,feat_dim]."""
|
| 37 |
+
import torch.nn as nn
|
| 38 |
+
from torchvision import models
|
| 39 |
+
if arch == "mobilenet_v3_small":
|
| 40 |
+
m = models.mobilenet_v3_small(weights="IMAGENET1K_V1" if pretrained else None)
|
| 41 |
+
feat = m.classifier[0].in_features
|
| 42 |
+
m.classifier = nn.Identity()
|
| 43 |
+
return m, feat
|
| 44 |
+
if arch == "shufflenet_v2_x0_5":
|
| 45 |
+
m = models.shufflenet_v2_x0_5(weights="IMAGENET1K_V1" if pretrained else None)
|
| 46 |
+
feat = m.fc.in_features
|
| 47 |
+
m.fc = nn.Identity()
|
| 48 |
+
return m, feat
|
| 49 |
+
if arch == "squeezenet1_1":
|
| 50 |
+
m = models.squeezenet1_1(weights="IMAGENET1K_V1" if pretrained else None)
|
| 51 |
+
feat = 512
|
| 52 |
+
class Sq(nn.Module):
|
| 53 |
+
def __init__(self, m):
|
| 54 |
+
super().__init__()
|
| 55 |
+
self.features = m.features
|
| 56 |
+
def forward(self, x):
|
| 57 |
+
return self.features(x).mean(dim=(2, 3))
|
| 58 |
+
return Sq(m), feat
|
| 59 |
+
if arch in ("efficientnet_lite0", "ghostnet_050"):
|
| 60 |
+
import timm
|
| 61 |
+
names = {"efficientnet_lite0": ["tf_efficientnet_lite0", "efficientnet_lite0"],
|
| 62 |
+
"ghostnet_050": ["ghostnet_050", "ghostnet_100"]}[arch]
|
| 63 |
+
last = None
|
| 64 |
+
for n in names:
|
| 65 |
+
try:
|
| 66 |
+
m = timm.create_model(n, pretrained=pretrained, num_classes=0)
|
| 67 |
+
return m, m.num_features
|
| 68 |
+
except Exception as e:
|
| 69 |
+
last = e
|
| 70 |
+
raise RuntimeError(f"timm model for {arch} failed: {last}")
|
| 71 |
+
if arch == "cnn6_custom":
|
| 72 |
+
import torch.nn as nn
|
| 73 |
+
ch = (32, 64, 128, 256, 256, 512)
|
| 74 |
+
blocks = []
|
| 75 |
+
for c in ch:
|
| 76 |
+
blocks += [nn.Conv2d(blocks[-1].out_channels if blocks else 3, c, 3, padding=1, bias=False),
|
| 77 |
+
nn.BatchNorm2d(c), nn.ReLU(inplace=True),
|
| 78 |
+
nn.Conv2d(c, c, 3, padding=1, groups=c, bias=False), # depthwise
|
| 79 |
+
nn.BatchNorm2d(c), nn.ReLU(inplace=True),
|
| 80 |
+
nn.Conv2d(c, c, 1, bias=False), nn.BatchNorm2d(c), nn.ReLU(inplace=True),
|
| 81 |
+
nn.MaxPool2d(2)]
|
| 82 |
+
class CNN6(nn.Module):
|
| 83 |
+
def __init__(self):
|
| 84 |
+
super().__init__()
|
| 85 |
+
self.net = nn.Sequential(*blocks)
|
| 86 |
+
def forward(self, x):
|
| 87 |
+
return self.net(x).mean(dim=(2, 3))
|
| 88 |
+
return CNN6(), ch[-1]
|
| 89 |
+
raise ValueError(arch)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def main():
|
| 93 |
+
import torch
|
| 94 |
+
import torch.nn as nn
|
| 95 |
+
|
| 96 |
+
ap = argparse.ArgumentParser()
|
| 97 |
+
ap.add_argument("--arch", choices=ARCHES, required=True)
|
| 98 |
+
ap.add_argument("--seed", type=int, default=42)
|
| 99 |
+
ap.add_argument("--epochs", type=int, default=10)
|
| 100 |
+
ap.add_argument("--batch", type=int, default=64)
|
| 101 |
+
ap.add_argument("--lr", type=float, default=3e-4)
|
| 102 |
+
ap.add_argument("--max-train-rows", type=int, default=120000)
|
| 103 |
+
ap.add_argument("--val-rows", type=int, default=4000)
|
| 104 |
+
ap.add_argument("--pretrained", type=int, default=1)
|
| 105 |
+
ap.add_argument("--out", type=Path, default=Path("/tmp/c5_out"))
|
| 106 |
+
args = ap.parse_args()
|
| 107 |
+
|
| 108 |
+
random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
|
| 109 |
+
if torch.cuda.is_available():
|
| 110 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 111 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 112 |
+
print(f"[job] arch={args.arch} seed={args.seed} device={device} "
|
| 113 |
+
f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
|
| 114 |
+
|
| 115 |
+
# ---------- data ----------
|
| 116 |
+
from datasets import load_dataset
|
| 117 |
+
t0 = time.time()
|
| 118 |
+
ds = load_dataset(DATASET, "windows_5s")
|
| 119 |
+
print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
|
| 120 |
+
unames = ds["train"].features["sound_class"].names
|
| 121 |
+
assert unames == list(CLASS_NAMES_UNIFIED), unames
|
| 122 |
+
|
| 123 |
+
def to_xy(split, source=None, labeled=True, cap=None):
|
| 124 |
+
d = ds[split]
|
| 125 |
+
if source is not None:
|
| 126 |
+
d = d.filter(lambda r: r["source"] == source, num_proc=4)
|
| 127 |
+
if labeled:
|
| 128 |
+
d = d.filter(lambda r: r["labeled"], num_proc=4)
|
| 129 |
+
if cap and len(d) > cap:
|
| 130 |
+
d = d.shuffle(seed=args.seed).select(range(cap))
|
| 131 |
+
return d
|
| 132 |
+
|
| 133 |
+
train_d = to_xy("train", cap=args.max_train_rows)
|
| 134 |
+
val_d = to_xy("validation", cap=args.val_rows)
|
| 135 |
+
c5_valid = to_xy("validation", source="challenge5", labeled=True)
|
| 136 |
+
c5_test = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
|
| 137 |
+
print(f"[data] train={len(train_d)} val={len(val_d)} c5_valid={len(c5_valid)} c5_test={len(c5_test)}", flush=True)
|
| 138 |
+
|
| 139 |
+
from PIL import Image
|
| 140 |
+
from torch.utils.data import DataLoader
|
| 141 |
+
|
| 142 |
+
def collate(batch):
|
| 143 |
+
xs = torch.stack([torch.from_numpy(np.asarray(r["spectrogram"], dtype=np.float32) / 255.0)
|
| 144 |
+
.permute(2, 0, 1) for r in batch])
|
| 145 |
+
wake = torch.tensor([float(r["has_drone_audio"]) for r in batch])
|
| 146 |
+
cls = torch.tensor([SOUND_CLASSES.index(CLASS_NAMES_UNIFIED[r["sound_class"]])
|
| 147 |
+
if r["sound_class"] < 4 else 0 for r in batch], dtype=torch.long)
|
| 148 |
+
ids = [r["sample_id"] for r in batch]
|
| 149 |
+
return xs, wake, cls, ids
|
| 150 |
+
|
| 151 |
+
train_loader = DataLoader(train_d, batch_size=args.batch, shuffle=True, num_workers=4,
|
| 152 |
+
collate_fn=collate, drop_last=True)
|
| 153 |
+
val_loader = DataLoader(val_d, batch_size=256, shuffle=False, num_workers=4, collate_fn=collate)
|
| 154 |
+
c5v_loader = DataLoader(c5_valid, batch_size=16, shuffle=False, collate_fn=collate)
|
| 155 |
+
c5t_loader = DataLoader(c5_test, batch_size=16, shuffle=False, collate_fn=collate)
|
| 156 |
+
|
| 157 |
+
# ---------- model ----------
|
| 158 |
+
class WakeCNN(nn.Module):
|
| 159 |
+
def __init__(self, arch, pretrained):
|
| 160 |
+
super().__init__()
|
| 161 |
+
self.register_buffer("mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
|
| 162 |
+
self.register_buffer("std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
|
| 163 |
+
self.backbone, feat = build_backbone(arch, pretrained)
|
| 164 |
+
self.head_wake = nn.Linear(feat, 1)
|
| 165 |
+
self.head_class = nn.Linear(feat, 4)
|
| 166 |
+
|
| 167 |
+
def forward(self, x):
|
| 168 |
+
x = (x - self.mean) / self.std
|
| 169 |
+
f = self.backbone(x)
|
| 170 |
+
return self.head_wake(f).squeeze(-1), self.head_class(f)
|
| 171 |
+
|
| 172 |
+
model = WakeCNN(args.arch, bool(args.pretrained)).to(device)
|
| 173 |
+
n_params = sum(p.numel() for p in model.parameters())
|
| 174 |
+
print(f"[model] {args.arch} params={n_params/1e6:.2f}M", flush=True)
|
| 175 |
+
|
| 176 |
+
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
|
| 177 |
+
sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs)
|
| 178 |
+
pos = float(np.mean([r["has_drone_audio"] for r in train_d.select(range(2000))]))
|
| 179 |
+
pos_weight = torch.tensor([min(2.0, max(1.0, ((1 - pos) / max(pos, 1e-3)) ** 0.5))], device=device)
|
| 180 |
+
print(f"[train] pos_rate~{pos:.3f} pos_weight={float(pos_weight):.3f}", flush=True)
|
| 181 |
+
bce = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
|
| 182 |
+
ce = nn.CrossEntropyLoss()
|
| 183 |
+
|
| 184 |
+
def eval_loader(loader, model):
|
| 185 |
+
model.eval()
|
| 186 |
+
W, Y, C, P, ids = [], [], [], [], []
|
| 187 |
+
with torch.no_grad():
|
| 188 |
+
for xs, wake, cls, sids in loader:
|
| 189 |
+
wl, cl = model(xs.to(device))
|
| 190 |
+
W.append(torch.sigmoid(wl).cpu()); Y.append(wake); C.append(cls)
|
| 191 |
+
P.append(cl.argmax(1).cpu()); ids += sids
|
| 192 |
+
return (torch.cat(W).numpy(), torch.cat(Y).numpy(), torch.cat(C).numpy(),
|
| 193 |
+
torch.cat(P).numpy(), ids)
|
| 194 |
+
|
| 195 |
+
def sweep(W, Y, grid=None):
|
| 196 |
+
grid = grid or [round(0.30 + 0.05 * i, 2) for i in range(11)]
|
| 197 |
+
best = (-1, 0.5, None)
|
| 198 |
+
for t in grid:
|
| 199 |
+
p = W >= t
|
| 200 |
+
tp = int((p & (Y > 0.5)).sum()); fp = int((p & (Y <= 0.5)).sum()); fn = int((~p & (Y > 0.5)).sum())
|
| 201 |
+
f1 = 2 * tp / max(1, 2 * tp + fp + fn)
|
| 202 |
+
u = f1 - 0.15 * float(p.mean())
|
| 203 |
+
if u > best[0]:
|
| 204 |
+
best = (u, t, (f1, float(p.mean()), tp, fp, fn))
|
| 205 |
+
return best
|
| 206 |
+
|
| 207 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 208 |
+
log_path = args.out / "train_log.txt"
|
| 209 |
+
logf = open(log_path, "a")
|
| 210 |
+
|
| 211 |
+
def log(msg):
|
| 212 |
+
print(msg, flush=True)
|
| 213 |
+
logf.write(msg + "\n"); logf.flush()
|
| 214 |
+
|
| 215 |
+
best_u, best_t, best_state = -1, 0.5, None
|
| 216 |
+
for ep in range(1, args.epochs + 1):
|
| 217 |
+
model.train()
|
| 218 |
+
t0 = time.time()
|
| 219 |
+
n_seen = 0
|
| 220 |
+
scaler = torch.amp.GradScaler("cuda", enabled=device.type == "cuda")
|
| 221 |
+
for xs, wake, cls, _ in train_loader:
|
| 222 |
+
opt.zero_grad()
|
| 223 |
+
with torch.amp.autocast("cuda", enabled=device.type == "cuda"):
|
| 224 |
+
wl, cl = model(xs.to(device))
|
| 225 |
+
loss = bce(wl, wake.to(device)) + 0.5 * ce(cl, cls.to(device))
|
| 226 |
+
scaler.scale(loss).backward()
|
| 227 |
+
scaler.step(opt)
|
| 228 |
+
scaler.update()
|
| 229 |
+
n_seen += len(xs)
|
| 230 |
+
sched.step()
|
| 231 |
+
W, Y, C, P, _ = eval_loader(val_loader, model)
|
| 232 |
+
u, t, stats = sweep(W, Y)
|
| 233 |
+
Wv, Yv, Cv, Pv, _ = eval_loader(c5v_loader, model)
|
| 234 |
+
uv, tv, statsv = sweep(Wv, Yv)
|
| 235 |
+
cls_acc_v = float((Pv == Cv).mean())
|
| 236 |
+
log(f"[ep {ep}/{args.epochs}] {time.time()-t0:.0f}s rows={n_seen} "
|
| 237 |
+
f"unified_val: F1={stats[0]:.4f} duty={stats[1]:.3f} @thr={t:.2f} u={u:.4f} | "
|
| 238 |
+
f"c5_valid: F1={statsv[0]:.4f} duty={statsv[1]:.3f} @thr={tv:.2f} cls_acc={cls_acc_v:.3f}")
|
| 239 |
+
if u > best_u:
|
| 240 |
+
best_u, best_t = u, t
|
| 241 |
+
best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
|
| 242 |
+
torch.save(best_state, args.out / "best.pt")
|
| 243 |
+
|
| 244 |
+
if best_state is not None:
|
| 245 |
+
model.load_state_dict(best_state)
|
| 246 |
+
model.eval().cpu()
|
| 247 |
+
|
| 248 |
+
# final metrics
|
| 249 |
+
W, Y, C, P, _ = eval_loader(val_loader, model)
|
| 250 |
+
u, t, stats = sweep(W, Y)
|
| 251 |
+
Wv, Yv, Cv, Pv, ids_v = eval_loader(c5v_loader, model)
|
| 252 |
+
pv = Wv >= best_t
|
| 253 |
+
tp = int((pv & (Yv > 0.5)).sum()); fp = int((pv & (Yv <= 0.5)).sum()); fn = int((~pv & (Yv > 0.5)).sum())
|
| 254 |
+
f1v = 2 * tp / max(1, 2 * tp + fp + fn)
|
| 255 |
+
dutyv = float(pv.mean())
|
| 256 |
+
cls_acc_v = float((Pv == Cv).mean())
|
| 257 |
+
log(f"[final] thr={best_t:.2f} c5_valid wake_F1={f1v:.4f} duty={dutyv:.3f} cls_acc={cls_acc_v:.4f} "
|
| 258 |
+
f"(tp={tp} fp={fp} fn={fn}) unified_val_u={best_u:.4f}")
|
| 259 |
+
|
| 260 |
+
# ---------- ONNX export with threshold metadata ----------
|
| 261 |
+
import onnx
|
| 262 |
+
dummy = torch.rand(1, 3, 224, 224)
|
| 263 |
+
onnx_path = args.out / "audio_wake_cnn.onnx"
|
| 264 |
+
torch.onnx.export(model, (dummy,), str(onnx_path), opset_version=17,
|
| 265 |
+
input_names=["spectrogram"], output_names=["wake_logit", "sound_class_logits"],
|
| 266 |
+
dynamo=False)
|
| 267 |
+
m = onnx.load(str(onnx_path))
|
| 268 |
+
md = {e.key: e.value for e in m.metadata_props}
|
| 269 |
+
md["wake_threshold"] = str(best_t)
|
| 270 |
+
md["arch"] = args.arch
|
| 271 |
+
onnx.helper.set_model_props(m, md)
|
| 272 |
+
onnx.save(m, str(onnx_path))
|
| 273 |
+
onnx_mb = onnx_path.stat().st_size / 1e6
|
| 274 |
+
log(f"[onnx] saved {onnx_mb:.2f} MB, threshold={best_t:.2f}")
|
| 275 |
+
|
| 276 |
+
# ---------- ORT latency ----------
|
| 277 |
+
import onnxruntime as ort
|
| 278 |
+
sess = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"])
|
| 279 |
+
x = np.random.rand(1, 3, 224, 224).astype(np.float32)
|
| 280 |
+
for _ in range(10):
|
| 281 |
+
sess.run(None, {"spectrogram": x})
|
| 282 |
+
ts = []
|
| 283 |
+
for _ in range(100):
|
| 284 |
+
t0 = time.time()
|
| 285 |
+
sess.run(None, {"spectrogram": x})
|
| 286 |
+
ts.append((time.time() - t0) * 1000)
|
| 287 |
+
lat_ms = float(np.mean(ts))
|
| 288 |
+
log(f"[ort] CPU latency mean={lat_ms:.2f} ms p50={float(np.percentile(ts,50)):.2f}")
|
| 289 |
+
|
| 290 |
+
# ---------- predictions on challenge5 valid + test ----------
|
| 291 |
+
def predict(loader):
|
| 292 |
+
outs = []
|
| 293 |
+
with torch.no_grad():
|
| 294 |
+
for xs, wake, cls, sids in loader:
|
| 295 |
+
wl, cl = model(xs)
|
| 296 |
+
wp = torch.sigmoid(wl).numpy()
|
| 297 |
+
cp = cl.argmax(1).numpy()
|
| 298 |
+
for sid, w, c in zip(sids, wp, cp):
|
| 299 |
+
clip = sid.split("__")[1].replace(".wav", "")
|
| 300 |
+
outs.append({"id": clip, "wake_triggered": bool(w >= best_t),
|
| 301 |
+
"sound_class": SOUND_CLASSES[int(c)]})
|
| 302 |
+
return outs
|
| 303 |
+
|
| 304 |
+
preds_valid = predict(c5v_loader)
|
| 305 |
+
preds_test = predict(c5t_loader)
|
| 306 |
+
(args.out / "preds_valid.jsonl").write_text(
|
| 307 |
+
"\n".join(json.dumps(p) for p in preds_valid) + "\n")
|
| 308 |
+
(args.out / "preds_test.jsonl").write_text(
|
| 309 |
+
"\n".join(json.dumps(p) for p in preds_test) + "\n")
|
| 310 |
+
|
| 311 |
+
# ---------- publish ----------
|
| 312 |
+
metrics = {
|
| 313 |
+
"arch": args.arch, "seed": args.seed, "epochs": args.epochs, "batch": args.batch,
|
| 314 |
+
"lr": args.lr, "params": n_params, "onnx_mb": round(onnx_mb, 2),
|
| 315 |
+
"wake_threshold": best_t,
|
| 316 |
+
"c5_valid": {"wake_F1": round(f1v, 4), "optical_duty_cycle": round(dutyv, 4),
|
| 317 |
+
"sound_class_accuracy": round(cls_acc_v, 4),
|
| 318 |
+
"optical_inactivity_pct": round((1 - dutyv) * 100, 2), "tp": tp, "fp": fp, "fn": fn},
|
| 319 |
+
"unified_val": {"utility": round(best_u, 4), "thr": best_t,
|
| 320 |
+
"F1": round(stats[0], 4), "duty": round(stats[1], 4)},
|
| 321 |
+
"ort_cpu_latency_ms": round(lat_ms, 2),
|
| 322 |
+
"train_rows": len(train_d), "val_rows": len(val_d),
|
| 323 |
+
"dataset": DATASET,
|
| 324 |
+
}
|
| 325 |
+
(args.out / "metrics.json").write_text(json.dumps(metrics, indent=2))
|
| 326 |
+
|
| 327 |
+
repo = f"Tonic/challenge5-acoustic-{args.arch}-s{args.seed}"
|
| 328 |
+
from huggingface_hub import HfApi
|
| 329 |
+
api = HfApi()
|
| 330 |
+
api.create_repo(repo, repo_type="model", exist_ok=True, private=False)
|
| 331 |
+
try:
|
| 332 |
+
api.update_repo_settings(repo_id=repo, repo_type="model", gated="manual")
|
| 333 |
+
except Exception as e:
|
| 334 |
+
log(f"[publish] gating note: {str(e)[:80]}")
|
| 335 |
+
card = f"""---
|
| 336 |
+
license: apache-2.0
|
| 337 |
+
tags: [acoustic-wake, c-uas, challenge-5, onnx, drone-detection]
|
| 338 |
+
---
|
| 339 |
+
# Challenge-5 acoustic wake model: {args.arch} (seed {args.seed})
|
| 340 |
+
|
| 341 |
+
Two-head CNN over 224x224 log-mel windows: `wake_logit` + `sound_class_logits` (4 classes),
|
| 342 |
+
ImageNet normalization baked in-graph, `wake_threshold` stored in ONNX metadata.
|
| 343 |
+
|
| 344 |
+
## Metrics (challenge5 valid, {len(c5_valid)} clips)
|
| 345 |
+
```json
|
| 346 |
+
{json.dumps(metrics['c5_valid'], indent=2)}
|
| 347 |
+
```
|
| 348 |
+
ORT CPU latency: {lat_ms:.2f} ms/window | params {n_params/1e6:.2f}M | ONNX {onnx_mb:.2f} MB
|
| 349 |
+
|
| 350 |
+
Trained on {DATASET} `windows_5s` (train, {len(train_d)} rows) + challenge5 train clips.
|
| 351 |
+
Predictions: `preds_valid.jsonl`, `preds_test.jsonl` (challenge submission format).
|
| 352 |
+
"""
|
| 353 |
+
(args.out / "README.md").write_text(card)
|
| 354 |
+
api.upload_folder(folder_path=str(args.out), path_in_repo="", repo_id=repo, repo_type="model",
|
| 355 |
+
commit_message=f"{args.arch} seed {args.seed}: F1={f1v:.3f} acc={cls_acc_v:.3f} thr={best_t:.2f}")
|
| 356 |
+
log(f"[publish] https://huggingface.co/{repo}")
|
| 357 |
+
|
| 358 |
+
# append to shared leaderboard dataset repo
|
| 359 |
+
try:
|
| 360 |
+
api.create_repo(RESULTS_REPO, repo_type="dataset", exist_ok=True, private=False)
|
| 361 |
+
row = json.dumps(metrics) + "\n"
|
| 362 |
+
try:
|
| 363 |
+
old = api.hf_hub_download(RESULTS_REPO, "results.jsonl", repo_type="dataset")
|
| 364 |
+
data = open(old, "rb").read() + row.encode()
|
| 365 |
+
except Exception:
|
| 366 |
+
data = row.encode()
|
| 367 |
+
api.upload_file(path_or_fileobj=data, path_in_repo="results.jsonl",
|
| 368 |
+
repo_id=RESULTS_REPO, repo_type="dataset",
|
| 369 |
+
commit_message=f"add {args.arch} s{args.seed}")
|
| 370 |
+
log(f"[publish] leaderboard {RESULTS_REPO} updated")
|
| 371 |
+
except Exception as e:
|
| 372 |
+
log(f"[publish] leaderboard append failed: {str(e)[:120]}")
|
| 373 |
+
|
| 374 |
+
log(f"[done] arch={args.arch} seed={args.seed} F1={f1v:.4f} acc={cls_acc_v:.4f} duty={dutyv:.3f} thr={best_t:.2f} lat={lat_ms:.2f}ms")
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
if __name__ == "__main__":
|
| 378 |
+
main()
|