"""Build the unified drone-acoustic dataset as parquet shards. FULL sources, worker-local writes. Configs: clips - one row per source clip, full-length 16 kHz mono WAV windows_5s - 5.0 s windows with 224x224 log-mel PNG + metadata + clip_sample_id Each worker process writes its own shard files (pid-tagged), so no large payloads cross process boundaries. Run per source: build_unified.py [...] """ import json, os, sys, csv, glob, re for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", "BLIS_NUM_THREADS"): os.environ.setdefault(_v, "1") from multiprocessing import Pool import numpy as np import pyarrow as pa import pyarrow.parquet as pq from unified_schema import (CLASSES, CLASS_ID, SPLITS, SPLIT_ID, SOURCE_REPOS, TARGET_SR, WINDOW_SEC, WINDOW_SAMPLES, assign_split, resample, wav_bytes, read_audio_any, logmel_png, make_windows) ROOT = os.path.dirname(os.path.abspath(__file__)) CHALLENGE = os.path.dirname(ROOT) OUT = os.path.join(ROOT, "build", "unified") BATCH = 500 NPROC = max(1, (os.cpu_count() or 4) - 1) META_FIELDS = [ ("drone_model", pa.string()), ("distance_m", pa.float32()), ("altitude_m", pa.float32()), ("azimuth_deg", pa.float32()), ("yaw_deg", pa.float32()), ("speed_mps", pa.float32()), ("throttle", pa.string()), ("environment", pa.string()), ("non_drone_source", pa.string()), ("weather", pa.string()), ] META_DEFAULT = dict(drone_model="", distance_m=-1.0, altitude_m=-1.0, azimuth_deg=-1.0, yaw_deg=-1.0, speed_mps=-1.0, throttle="", environment="", non_drone_source="", weather="") BASE_FIELDS = [ ("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()), ("source_file", pa.string()), ("orig_sampling_rate", pa.int32()), ("duration_s", pa.float32()), ("window_index", pa.int32()), ("window_start_s", pa.float32()), ("window_sec", pa.float32()), ("sound_class", pa.int8()), ("has_drone_audio", pa.bool_()), ("labeled", pa.bool_()), ("source_label", pa.string()), ("group_id", pa.string()), ("split", pa.int8()), ("text", pa.string()), ] + META_FIELDS CLIP_FIELDS = [("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()), ("source_file", pa.string()), ("audio", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))] + BASE_FIELDS[4:] WINDOW_FIELDS = BASE_FIELDS + [("clip_sample_id", pa.string()), ("spectrogram", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))] _W = None # per-process writer class LocalWriter: def __init__(self): self.buf = {} self.shards = {} def _flush(self, key): rows = self.buf.get(key) if not rows: return config, split, source = key fields = WINDOW_FIELDS if config == "windows_5s" else CLIP_FIELDS cols = {name: [] for name, _ in fields} for r in rows: for name, _ in fields: cols[name].append(r.get(name)) arrays = [] for name, typ in fields: if name in ("audio", "spectrogram"): arrays.append(pa.StructArray.from_arrays( [pa.array([c[0] for c in cols[name]], type=pa.binary()), pa.array([c[1] for c in cols[name]], type=pa.string())], names=["bytes", "path"])) else: arrays.append(pa.array(cols[name], type=typ)) table = pa.Table.from_arrays(arrays, schema=pa.schema(fields)) d = os.path.join(OUT, config) os.makedirs(d, exist_ok=True) n = self.shards.get(key, 0) pq.write_table(table, os.path.join(d, f"{split}-{source}-p{os.getpid()}-{n:05d}.parquet"), compression="zstd") self.shards[key] = n + 1 self.buf[key] = [] def add(self, config, row): key = (config, SPLITS[row["split"]], row["source"]) self.buf.setdefault(key, []).append(row) if len(self.buf[key]) >= BATCH: self._flush(key) def close(self): for key in list(self.buf): self._flush(key) def _init_worker(): global _W _W = LocalWriter() from multiprocessing.util import Finalize Finalize(None, _W.close, exitpriority=16) def emit_rows(source, source_file, x, sr, sound_class, has_drone, labeled, source_label, group_id, split, text="", meta=None): """Build clip row + window rows and hand them to the process-local writer.""" if x.ndim > 1: x = x.mean(axis=1) x16 = resample(x, sr) dur = len(x16) / TARGET_SR sid = f"{source}__{source_file.replace('/', '__').replace(' ', '_')}" m = dict(META_DEFAULT) spec_png = (meta or {}).get("_spec_png") m.update({k: v for k, v in (meta or {}).items() if k != "_spec_png"}) base = dict(source=source, source_repo=SOURCE_REPOS[source], source_file=source_file, orig_sampling_rate=int(sr), sound_class=CLASS_ID[sound_class], has_drone_audio=bool(has_drone), labeled=bool(labeled), source_label=source_label, group_id=group_id, split=SPLIT_ID[split], text=text, **m) _W.add("clips", dict(base, sample_id=sid, audio=(wav_bytes(x16), (os.path.basename(source_file) or "clip.wav")), duration_s=float(dur), window_index=0, window_start_s=0.0, window_sec=float(dur))) for i, start_s, chunk in make_windows(x16): chunk = np.pad(chunk, (0, max(0, WINDOW_SAMPLES - len(chunk)))) wdur = min(WINDOW_SEC, dur - start_s) if i == 0 and spec_png: with open(spec_png, "rb") as fh: spec = fh.read() # use the challenge's own precomputed PNG verbatim else: spec = logmel_png(chunk) _W.add("windows_5s", dict(base, sample_id=f"{sid}__w{i:04d}", clip_sample_id=sid, duration_s=float(max(wdur, 0.0)), window_index=i, window_start_s=float(start_s), window_sec=WINDOW_SEC, spectrogram=(spec, f"w{i:04d}.png"))) def load_locator(loc): """loc: ('path', p) or ('pqrow', parquet_path, row_idx).""" if loc[0] == "path": return read_audio_any(loc[1]) _, p, i = loc t = pq.read_table(p) a = t.column("audio")[i].as_py() return read_audio_any(a["bytes"]) def _work(t): try: x, sr = load_locator(t["loc"]) emit_rows(t["source"], t["source_file"], x, sr, t["sound_class"], t["has_drone"], t["labeled"], t["source_label"], t["group_id"], t["split"], t.get("text", ""), t.get("meta")) except Exception as e: print("ERR", t["source"], t["source_file"], str(e)[:120], flush=True) return 1 def run_tasks(tasks, tag): """tasks: list of dicts with 'loc' + label/meta fields. Worker-local writing.""" n = 0 pool = Pool(NPROC, initializer=_init_worker) for _ in pool.imap_unordered(_work, tasks, chunksize=4): n += 1 if n % 5000 == 0: print(f"[{tag}] {n}/{len(tasks)}", flush=True) pool.close() pool.join() # clean worker shutdown so atexit flushes buffers print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True) def T(source, source_file, loc, sound_class="ambient_outdoor", has_drone=False, labeled=True, source_label="", group_id="", split="train", text="", meta=None): return dict(source=source, source_file=source_file, loc=loc, sound_class=sound_class, has_drone=has_drone, labeled=labeled, source_label=source_label, group_id=group_id, split=split, text=text, meta=meta) # ---------- sources ---------- def ingest_challenge5(): tasks = [] for split_name, split in (("train", "train"), ("valid", "validation"), ("test", "test")): for line in open(os.path.join(CHALLENGE, "dataset", split_name, "labels.jsonl"), encoding="utf-8"): if not line.strip(): continue r = json.loads(line) labeled = "sound_class" in r cls = r.get("sound_class", "unknown") tasks.append(T("challenge5", r["id"] + ".wav", ("path", os.path.join(CHALLENGE, r["audio"])), sound_class=cls, has_drone=bool(r.get("has_drone_audio", False)), labeled=labeled, source_label=cls if labeled else "held-out", group_id=r["id"], split=split, text=f"challenge-5 {split_name}; sound_class={cls}; has_drone_audio={r.get('has_drone_audio', False)}", meta=dict(environment="outdoor field microphone", non_drone_source="" if r.get("has_drone_audio") else cls, _spec_png=os.path.join(CHALLENGE, r["spectrogram"]) if r.get("spectrogram") else None))) run_tasks(tasks, "challenge5") def ingest_itu(): root = os.path.join(ROOT, "raw", "itu-aris") groups = json.load(open(os.path.join(root, "groups.json"))) seg2group = {s["file"]: (s["label"], s["group"]) for s in groups["segments"]} assign = json.load(open(os.path.join(root, "splits", "split_interval_disjoint.json")))["assignment"] tasks = [] for f, (label, group) in seg2group.items(): split = {"train": "train", "val": "validation", "test": "test"}.get(assign.get(f, "train"), "train") tasks.append(T("itu-aris", f, ("path", os.path.join(root, f)), sound_class="drone_buzz" if label == "drone" else "ambient_outdoor", has_drone=label == "drone", source_label=label, group_id=group, split=split, text=f"ITU ARIS Lab 1s segment, group={group}; single 3.12h outdoor session, drone ranges few m to ~2 km, multiple UAV models", meta=dict(environment="outdoor field, single continuous session", non_drone_source="" if label == "drone" else "background"))) run_tasks(tasks, "itu-aris") def ingest_bucket_drone(): root = os.path.join(ROOT, "raw", "bucket", "drone") tasks = [] with open(os.path.join(root, "metadata.csv"), newline="", encoding="utf-8") as fh: for row in csv.DictReader(fh): fn = row["file_name"] p = os.path.join(root, fn) if not os.path.exists(p): continue label = row["label"] stem = os.path.basename(fn)[:-4] group = stem.rsplit("-", 1)[0] if "-" in stem else stem tasks.append(T("bucket-drone", fn, ("path", p), sound_class="drone_buzz" if label == "1" else "ambient_outdoor", has_drone=label == "1", source_label=f"label={label}", group_id=group, split=assign_split("bucket-drone", group), text="yaroslav2005 bucket drone/ (file,label csv)", meta=dict(non_drone_source="" if label == "1" else "non-drone audio"))) run_tasks(tasks, "bucket-drone") def ingest_test2(): root = os.path.join(ROOT, "raw", "bucket", "drone_all_test_datasets_2", "data") tasks = [] for f in sorted(glob.glob(os.path.join(root, "*.parquet"))): shard = os.path.basename(f).split("-")[0] try: md = pq.ParquetFile(f).metadata n = md.num_rows except Exception as e: print("skip corrupt shard", f, str(e)[:60], flush=True) continue for i in range(n): tasks.append(T("drone-test2", f"{shard}/row{i}", ("pqrow", f, i), sound_class="?", has_drone=None, source_label="?", # filled in worker group_id="", split="train", text="", meta={"_shard": f, "_row": i})) # labels need the parquet; fill them in worker via _shard/_row run_pq_tasks(tasks, "drone-test2") def parse_das_path(fp, data_type): meta = {} m = re.search(r"(drone\d+)(?:-only)?/", fp) if m: meta["drone_model"] = m.group(1) m = re.search(r"mic-dist-(\d+)cm", fp) if m: meta["distance_m"] = float(m.group(1)) / 100.0 m = re.search(r"throttle-(\w+)", fp) if m: meta["throttle"] = m.group(1) m = re.search(r"speaker-loudness-(\d+)db", fp) if m: meta["non_drone_source"] = f"speaker playback @ {m.group(1)} dB" m = re.search(r"speaker-dist-(\d+)m", fp) if m and "distance_m" not in meta: meta["distance_m"] = float(m.group(1)) if "room" in fp: meta["environment"] = "indoor room (speaker + drone)" if data_type == "source-only": meta["non_drone_source"] = meta.get("non_drone_source", "speaker playback (source only)") meta["environment"] = "indoor room" return meta def ingest_das(): root = os.path.join(ROOT, "raw", "droneaudioset") tasks = [] for f in sorted(glob.glob(os.path.join(root, "**", "*.parquet"), recursive=True)): rel = os.path.relpath(f, root) cfg = rel.split(os.sep)[0] try: md = pq.ParquetFile(f).metadata n = md.num_rows except Exception as e: print("skip corrupt shard", rel, str(e)[:60], flush=True) continue for i in range(n): tasks.append(T("droneaudioset", f"{cfg}/row{i}", ("pqrow", f, i), sound_class="?", has_drone=None, source_label="?", group_id="", split="train", text="", meta={"_shard": f, "_row": i})) run_pq_tasks(tasks, "droneaudioset") def ingest_dads(): root = os.path.join(ROOT, "raw", "dads_full", "data") tasks = [] for f in sorted(glob.glob(os.path.join(root, "*.parquet"))): try: md = pq.ParquetFile(f).metadata n = md.num_rows except Exception as e: print("skip corrupt shard", f, str(e)[:60], flush=True) continue for i in range(n): tasks.append(T("dads", f"row{i}", ("pqrow", f, i), sound_class="?", has_drone=None, source_label="?", group_id="", split="train", text="", meta={"_shard": f, "_row": i})) run_pq_tasks(tasks, "dads") Z15_MODELS = {"d1": "DJI Matrice 300", "d2": "DJI Mavic 3", "d3": "DJI Mavic Air 2S", "d4": "DJI Mavic Air 2", "d5": "DJI Mavic Mini 2", "d6": "DJI Mavic 2 Pro", "d7": "DJI Mavic 2 Pro", "d8": "DJI Mavic 3", "d9": "DJI Phantom 4", "d10": "DJI Mavic 2 Zoom", "d11": "DJI Mavic Mini 2", "d12": "Yuneec H520", "d13": "Yuneec H520E RTK", "d14": "S900", "d15": "X6D", "d16": "Y6", "d17": "DJI Phantom 4"} Z15_PLACE = {"230315": ("Kielce (urban)", "5C, 1014 hPa, 51% RH, wind 22 km/h"), "230415": ("Gdańsk (urban)", "10C, 1015 hPa, 78% RH, wind 25 km/h"), "230416": ("Dębogórze near Gdańsk (rural)", "6C, 1022 hPa, 93% RH, wind 18 km/h"), "230417": ("Dębogórze near Gdańsk (rural)", "7C, 1030 hPa, 80% RH, wind 22 km/h"), "230418": ("Łapalice near Gdańsk (rural)", "8C, 1033 hPa, 90% RH, wind 25 km/h")} def parse_z15_name(name): base = os.path.basename(name).lower().replace(".wav", "") m = re.match(r"(x[46])_(d\d+)_([a-z0-9]+)_(\d+)m_(\d+)m_(\d{6})_(\d+)(_sekwencja)?$", base) meta = {"environment": "outdoor"} if not m: return meta, "uav_hover" _, dno, model_key, alt, dist, date, num, seq = m.groups() meta["drone_model"] = Z15_MODELS.get(dno, model_key) meta["altitude_m"] = float(alt) meta["distance_m"] = float(dist) place, weather = Z15_PLACE.get(date, ("unknown", "")) meta["environment"] = f"outdoor {place}" meta["weather"] = weather if seq: meta["non_drone_source"] = "speech commands (EN/PL) in drone presence" return meta, "speech_sequence_in_drone_presence" return meta, "uav_hover" def ingest_zenodo_miesikowska(): root = os.path.join(ROOT, "raw", "zenodo", "15190811") tasks = [] for f in sorted(glob.glob(os.path.join(root, "*.wav"))): name = os.path.basename(f) meta, label = parse_z15_name(name) group = re.sub(r"_\d+(_sekwencja)?\.wav$", "", name.lower()) tasks.append(T("zenodo-15190811", name, ("path", f), sound_class="drone_buzz", has_drone=True, source_label=label, group_id=group, split=assign_split("zenodo-15190811", group), text="Zenodo 15190811 (Miesikowska, Kielce Univ. Tech., CC-BY-4.0); OLYMPUS LS11 outdoor recording", meta=meta)) run_tasks(tasks, "zenodo-15190811") ROT2YAW = {"Front": 0.0, "Right": 90.0, "Back": 180.0, "Left": 270.0} def ingest_uavirbase(): root = os.path.join(ROOT, "raw", "zenodo", "15391924") tasks = [] for f in sorted(glob.glob(os.path.join(root, "*", "output.wav"))): sess = os.path.basename(os.path.dirname(f)) lp = os.path.join(os.path.dirname(f), "label.json") label = json.load(open(lp)) if os.path.exists(lp) else {} drone = label.get("drone", {}) wx = (label.get("weather_data") or {}).get("measurements", {}) weather = "; ".join(f"{k}={v}" for k, v in wx.items() if k in ("air temperature", "air humidity", "wind speed", "wind direction")) is_drone = drone.get("sound_source") == "Drone" meta = dict( drone_model=drone.get("type") or "", distance_m=float(drone.get("distance")) if drone.get("distance") else -1.0, altitude_m=float(drone.get("height")) if drone.get("height") else -1.0, azimuth_deg=float(drone.get("azimuth")) if drone.get("azimuth") is not None else -1.0, yaw_deg=ROT2YAW.get(drone.get("rotation") or "", -1.0), speed_mps=0.0 if drone.get("movement") == "Static" else -1.0, environment="outdoor open field (mic array, 52.26N 20.90E Poland)", weather=weather, non_drone_source="" if is_drone else "ambient noise (no UAV)") tasks.append(T("uavirbase", f"{sess}/output.wav", ("path", f), sound_class="drone_buzz" if is_drone else "ambient_outdoor", has_drone=is_drone, source_label="uav_flight" if is_drone else "ambient_only", group_id=sess, split=assign_split("uavirbase", sess), text=f"UaVirBASE session {sess}; label={json.dumps(drone)[:200]}", meta=meta)) run_tasks(tasks, "uavirbase") DD_MODELS = {"T1": "DJI Matrice 300 RTK", "T2": "DJI Phantom 4 Pro Plus", "T3": "Mavic 2 Enterprise Dual", "T6": "DJI Phantom 4 Pro V2.0"} FACING2YAW = {"front": 0.0, "right": 90.0, "l/r": 90.0, "back": 180.0, "left": 270.0} def parse_dd_setup(setup): meta = {} if not setup: return meta s = setup.lower() m = re.search(r"(\d+)\s*m\s+distance", s) if m: meta["distance_m"] = float(m.group(1)) m = re.search(r"(\d+)\s*m\s+above", s) if m: meta["altitude_m"] = float(m.group(1)) for facing, yaw in FACING2YAW.items(): if f"facing {facing}" in s: meta["yaw_deg"] = yaw break if "towards" in s: meta["azimuth_deg"] = 0.0 meta["speed_mps"] = -2.0 elif "away" in s: meta["azimuth_deg"] = 180.0 meta["speed_mps"] = -2.0 elif "parallel" in s: meta["azimuth_deg"] = 90.0 meta["speed_mps"] = -2.0 elif "stationary" in s: meta["speed_mps"] = 0.0 elif "random" in s: meta["speed_mps"] = -2.0 return meta def ingest_dronedata(): root = os.path.join(ROOT, "raw", "bucket", "DroneData", "extracted") manifest = os.path.join(ROOT, "build", "dronedata_manifest.json") if not os.path.exists(manifest): print("no dronedata manifest, skipping") return man = json.load(open(manifest)) import pandas as pd try: cs = pd.read_excel(os.path.join(ROOT, "raw", "bucket", "DroneData", "Data Collection.xlsx"), sheet_name="Collection state") setup = {str(r["File name"]): str(r["Setup"]) for _, r in cs.iterrows()} except Exception as e: print("xlsx read failed:", e) setup = {} tasks = [] for f in sorted(glob.glob(os.path.join(root, "**", "*.wav"), recursive=True)): rel = os.path.relpath(f, root).replace(os.sep, "/") code = man.get(rel) if code is None: continue sess_file = rel.split("/")[1][:-4] sess = sess_file.rsplit("_", 1)[0] if code == "NO": cls, drone, nds = "ambient_outdoor", False, "no object (ambient)" elif code == "OO": cls, drone, nds = "unknown", False, "other object (person running / non-UAV)" else: cls, drone, nds = "drone_buzz", True, "" meta = parse_dd_setup(setup.get(sess_file, "")) meta["drone_model"] = DD_MODELS.get(code, "") meta["environment"] = "outdoor radar test site" meta["non_drone_source"] = nds tasks.append(T("dronedata", rel, ("path", f), sound_class=cls, has_drone=drone, source_label=f"class_code={code}", group_id=sess, split=assign_split("dronedata", sess), text=f"DroneData session {sess_file} class {code}; setup={setup.get(sess_file, '')}", meta=meta)) run_tasks(tasks, "dronedata") # ---- pqrow sources need labels from the parquet; wrap load to fill them ---- _PQ_CACHE = {} _RG_CACHE = {} def pq_row_fields(shard, i): pf = _PQ_CACHE.get(shard) if pf is None: pf = pq.ParquetFile(shard) if len(_PQ_CACHE) > 2: _PQ_CACHE.clear() _RG_CACHE.clear() _PQ_CACHE[shard] = pf md = pf.metadata rg = 0 base = 0 if md.num_row_groups > 1: acc = 0 for g in range(md.num_row_groups): acc += md.row_group(g).num_rows if i < acc: rg = g base = acc - md.row_group(g).num_rows break key = (shard, rg) if key not in _RG_CACHE: if len(_RG_CACHE) > 3: _RG_CACHE.clear() _RG_CACHE[key] = pf.read_row_group(rg) # keep as arrow Table (cheap, lazy-ish) return _RG_CACHE[key], i - base def decode_audio_cell(audio_col, i): """audio_col: ChunkedArray/StructArray of audio structs. Returns (x float32, sr, path). The array field may be nested list> (Array2D, das) — decode via arrow flatten (zero-copy) instead of as_py() to avoid huge Python lists. """ sa = audio_col.combine_chunks() if isinstance(audio_col, pa.ChunkedArray) else audio_col names = list(sa.type.names) if "bytes" in names: b = sa.field("bytes")[i] if b.is_valid and b.as_py() is not None: return read_audio_any(b.as_py()) + (sa.field("path")[i].as_py() if "path" in names else None,) row = sa.field("array")[i] a = row.values if isinstance(row, pa.Scalar) else row m = len(row) while pa.types.is_list(a.type) or pa.types.is_large_list(a.type): a = a.flatten() x = np.asarray(a.to_numpy(zero_copy_only=False), dtype=np.float32).reshape(m, -1) sr = int(sa.field("sampling_rate")[i].as_py()) path = sa.field("path")[i].as_py() if "path" in names else None return x, sr, path def _work_pq(t): """Variant of work() for parquet-row tasks (dads/test2/das): labels come from the shard.""" tab, i = pq_row_fields(t["meta"]["_shard"], t["meta"]["_row"]) x, sr, apath = decode_audio_cell(tab.column("audio"), i) if t["source"] == "dads": label = int(tab.column("label")[i].as_py()) path = apath or f"row{i}.wav" emit_rows("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}", x, sr, "drone_buzz" if label == 1 else "ambient_outdoor", label == 1, True, f"label={label}", f"{os.path.basename(t['meta']['_shard'])}/{path}", assign_split("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}"), f"DADS sample; source path={path}", dict(non_drone_source="" if label == 1 else "no-drone (mixed sources)")) elif t["source"] == "drone-test2": label = int(tab.column("label")[i].as_py()) shard = re.sub(r"-\d+-of-\d+\.parquet$", "", os.path.basename(t["meta"]["_shard"])) path = apath or f"{i}.wav" stem = path[:-4] if path.endswith(".wav") else path group = f"{shard}__{stem}" # group by file, not row: chunks of one file stay together emit_rows("drone-test2", f"{shard}/{path}#r{i}", x, sr, "drone_buzz" if label == 1 else "ambient_outdoor", label == 1, True, f"label={label} ({'drone' if label == 1 else 'other'})", group, assign_split("drone-test2", group), f"external test shard {shard}", dict(environment="parabolic-mic test recording" if shard.startswith("parabole") else "", non_drone_source="" if label == 1 else "other (unspecified)")) elif t["source"] == "droneaudioset": dt = tab.column("data_type")[i].as_py() fp = tab.column("file_path")[i].as_py() cfg = os.path.relpath(t["meta"]["_shard"], os.path.join(ROOT, "raw", "droneaudioset")).split(os.sep)[0] group = os.path.dirname(fp) or fp emit_rows("droneaudioset", f"{cfg}/{fp}", x, sr, "drone_buzz" if dt != "source-only" else "ambient_outdoor", dt != "source-only", True, f"data_type={dt}", group, assign_split("droneaudioset", group), f"DroneAudioSet {cfg}; file_path={fp}", parse_das_path(fp, dt)) return 1 def run_pq_tasks(tasks, tag): n = 0 pool = Pool(NPROC, initializer=_init_worker) for _ in pool.imap_unordered(_work_pq, tasks, chunksize=16): n += 1 if n % 5000 == 0: print(f"[{tag}] {n}/{len(tasks)}", flush=True) pool.close() pool.join() print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True) if __name__ == "__main__": which = sys.argv[1:] or ["challenge5", "itu", "bucket_drone", "test2", "das", "dads", "zenodo15190811", "uavirbase", "dronedata"] state_p = os.path.join(ROOT, "build", "build_state.json") state = json.load(open(state_p)) if os.path.exists(state_p) else {} which = [w for w in which if not state.get(w)] print("running:", which, "skipped:", [k for k in state if k not in which], flush=True) if "challenge5" in which: ingest_challenge5() if "itu" in which: ingest_itu() if "bucket_drone" in which: ingest_bucket_drone() if "test2" in which: ingest_test2() # labels filled in worker if "das" in which: ingest_das() if "dads" in which: ingest_dads() if "zenodo15190811" in which: ingest_zenodo_miesikowska() if "uavirbase" in which: ingest_uavirbase() if "dronedata" in which: ingest_dronedata() for w in which: state[w] = True json.dump(state, open(state_p, "w")) print("BUILD DONE", flush=True)