Download build_unified.py from Tonic/challenge5-acoustic-code: direct link, hf CLI and curl.
- Browser
- Download file 27.4 kB
-
https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/build_unified.py
- Command line
-
hf download hf://Tonic/challenge5-acoustic-code/build_unified.py
-
curl -L -o build_unified.py https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/build_unified.py
27.4 kB
| """Build the unified drone-acoustic dataset as parquet shards. FULL sources, worker-local writes. | |
| Configs: | |
| clips - one row per source clip, full-length 16 kHz mono WAV | |
| windows_5s - 5.0 s windows with 224x224 log-mel PNG + metadata + clip_sample_id | |
| Each worker process writes its own shard files (pid-tagged), so no large payloads | |
| cross process boundaries. Run per source: build_unified.py <source> [...] | |
| """ | |
| import json, os, sys, csv, glob, re | |
| for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", | |
| "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", "BLIS_NUM_THREADS"): | |
| os.environ.setdefault(_v, "1") | |
| from multiprocessing import Pool | |
| import numpy as np | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| from unified_schema import (CLASSES, CLASS_ID, SPLITS, SPLIT_ID, SOURCE_REPOS, TARGET_SR, | |
| WINDOW_SEC, WINDOW_SAMPLES, assign_split, resample, wav_bytes, | |
| read_audio_any, logmel_png, make_windows) | |
| ROOT = os.path.dirname(os.path.abspath(__file__)) | |
| CHALLENGE = os.path.dirname(ROOT) | |
| OUT = os.path.join(ROOT, "build", "unified") | |
| BATCH = 500 | |
| NPROC = max(1, (os.cpu_count() or 4) - 1) | |
| META_FIELDS = [ | |
| ("drone_model", pa.string()), ("distance_m", pa.float32()), ("altitude_m", pa.float32()), | |
| ("azimuth_deg", pa.float32()), ("yaw_deg", pa.float32()), ("speed_mps", pa.float32()), | |
| ("throttle", pa.string()), ("environment", pa.string()), ("non_drone_source", pa.string()), | |
| ("weather", pa.string()), | |
| ] | |
| META_DEFAULT = dict(drone_model="", distance_m=-1.0, altitude_m=-1.0, azimuth_deg=-1.0, | |
| yaw_deg=-1.0, speed_mps=-1.0, throttle="", environment="", | |
| non_drone_source="", weather="") | |
| BASE_FIELDS = [ | |
| ("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()), | |
| ("source_file", pa.string()), | |
| ("orig_sampling_rate", pa.int32()), ("duration_s", pa.float32()), | |
| ("window_index", pa.int32()), ("window_start_s", pa.float32()), ("window_sec", pa.float32()), | |
| ("sound_class", pa.int8()), ("has_drone_audio", pa.bool_()), ("labeled", pa.bool_()), | |
| ("source_label", pa.string()), ("group_id", pa.string()), ("split", pa.int8()), | |
| ("text", pa.string()), | |
| ] + META_FIELDS | |
| CLIP_FIELDS = [("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()), | |
| ("source_file", pa.string()), | |
| ("audio", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))] + BASE_FIELDS[4:] | |
| WINDOW_FIELDS = BASE_FIELDS + [("clip_sample_id", pa.string()), | |
| ("spectrogram", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))] | |
| _W = None # per-process writer | |
| class LocalWriter: | |
| def __init__(self): | |
| self.buf = {} | |
| self.shards = {} | |
| def _flush(self, key): | |
| rows = self.buf.get(key) | |
| if not rows: | |
| return | |
| config, split, source = key | |
| fields = WINDOW_FIELDS if config == "windows_5s" else CLIP_FIELDS | |
| cols = {name: [] for name, _ in fields} | |
| for r in rows: | |
| for name, _ in fields: | |
| cols[name].append(r.get(name)) | |
| arrays = [] | |
| for name, typ in fields: | |
| if name in ("audio", "spectrogram"): | |
| arrays.append(pa.StructArray.from_arrays( | |
| [pa.array([c[0] for c in cols[name]], type=pa.binary()), | |
| pa.array([c[1] for c in cols[name]], type=pa.string())], | |
| names=["bytes", "path"])) | |
| else: | |
| arrays.append(pa.array(cols[name], type=typ)) | |
| table = pa.Table.from_arrays(arrays, schema=pa.schema(fields)) | |
| d = os.path.join(OUT, config) | |
| os.makedirs(d, exist_ok=True) | |
| n = self.shards.get(key, 0) | |
| pq.write_table(table, os.path.join(d, f"{split}-{source}-p{os.getpid()}-{n:05d}.parquet"), | |
| compression="zstd") | |
| self.shards[key] = n + 1 | |
| self.buf[key] = [] | |
| def add(self, config, row): | |
| key = (config, SPLITS[row["split"]], row["source"]) | |
| self.buf.setdefault(key, []).append(row) | |
| if len(self.buf[key]) >= BATCH: | |
| self._flush(key) | |
| def close(self): | |
| for key in list(self.buf): | |
| self._flush(key) | |
| def _init_worker(): | |
| global _W | |
| _W = LocalWriter() | |
| from multiprocessing.util import Finalize | |
| Finalize(None, _W.close, exitpriority=16) | |
| def emit_rows(source, source_file, x, sr, sound_class, has_drone, labeled, source_label, | |
| group_id, split, text="", meta=None): | |
| """Build clip row + window rows and hand them to the process-local writer.""" | |
| if x.ndim > 1: | |
| x = x.mean(axis=1) | |
| x16 = resample(x, sr) | |
| dur = len(x16) / TARGET_SR | |
| sid = f"{source}__{source_file.replace('/', '__').replace(' ', '_')}" | |
| m = dict(META_DEFAULT) | |
| spec_png = (meta or {}).get("_spec_png") | |
| m.update({k: v for k, v in (meta or {}).items() if k != "_spec_png"}) | |
| base = dict(source=source, source_repo=SOURCE_REPOS[source], source_file=source_file, | |
| orig_sampling_rate=int(sr), sound_class=CLASS_ID[sound_class], | |
| has_drone_audio=bool(has_drone), labeled=bool(labeled), | |
| source_label=source_label, group_id=group_id, split=SPLIT_ID[split], text=text, **m) | |
| _W.add("clips", dict(base, sample_id=sid, | |
| audio=(wav_bytes(x16), (os.path.basename(source_file) or "clip.wav")), | |
| duration_s=float(dur), window_index=0, window_start_s=0.0, | |
| window_sec=float(dur))) | |
| for i, start_s, chunk in make_windows(x16): | |
| chunk = np.pad(chunk, (0, max(0, WINDOW_SAMPLES - len(chunk)))) | |
| wdur = min(WINDOW_SEC, dur - start_s) | |
| if i == 0 and spec_png: | |
| with open(spec_png, "rb") as fh: | |
| spec = fh.read() # use the challenge's own precomputed PNG verbatim | |
| else: | |
| spec = logmel_png(chunk) | |
| _W.add("windows_5s", dict(base, sample_id=f"{sid}__w{i:04d}", clip_sample_id=sid, | |
| duration_s=float(max(wdur, 0.0)), window_index=i, | |
| window_start_s=float(start_s), window_sec=WINDOW_SEC, | |
| spectrogram=(spec, f"w{i:04d}.png"))) | |
| def load_locator(loc): | |
| """loc: ('path', p) or ('pqrow', parquet_path, row_idx).""" | |
| if loc[0] == "path": | |
| return read_audio_any(loc[1]) | |
| _, p, i = loc | |
| t = pq.read_table(p) | |
| a = t.column("audio")[i].as_py() | |
| return read_audio_any(a["bytes"]) | |
| def _work(t): | |
| try: | |
| x, sr = load_locator(t["loc"]) | |
| emit_rows(t["source"], t["source_file"], x, sr, t["sound_class"], t["has_drone"], | |
| t["labeled"], t["source_label"], t["group_id"], t["split"], t.get("text", ""), | |
| t.get("meta")) | |
| except Exception as e: | |
| print("ERR", t["source"], t["source_file"], str(e)[:120], flush=True) | |
| return 1 | |
| def run_tasks(tasks, tag): | |
| """tasks: list of dicts with 'loc' + label/meta fields. Worker-local writing.""" | |
| n = 0 | |
| pool = Pool(NPROC, initializer=_init_worker) | |
| for _ in pool.imap_unordered(_work, tasks, chunksize=4): | |
| n += 1 | |
| if n % 5000 == 0: | |
| print(f"[{tag}] {n}/{len(tasks)}", flush=True) | |
| pool.close() | |
| pool.join() # clean worker shutdown so atexit flushes buffers | |
| print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True) | |
| def T(source, source_file, loc, sound_class="ambient_outdoor", has_drone=False, labeled=True, | |
| source_label="", group_id="", split="train", text="", meta=None): | |
| return dict(source=source, source_file=source_file, loc=loc, sound_class=sound_class, | |
| has_drone=has_drone, labeled=labeled, source_label=source_label, | |
| group_id=group_id, split=split, text=text, meta=meta) | |
| # ---------- sources ---------- | |
| def ingest_challenge5(): | |
| tasks = [] | |
| for split_name, split in (("train", "train"), ("valid", "validation"), ("test", "test")): | |
| for line in open(os.path.join(CHALLENGE, "dataset", split_name, "labels.jsonl"), encoding="utf-8"): | |
| if not line.strip(): | |
| continue | |
| r = json.loads(line) | |
| labeled = "sound_class" in r | |
| cls = r.get("sound_class", "unknown") | |
| tasks.append(T("challenge5", r["id"] + ".wav", ("path", os.path.join(CHALLENGE, r["audio"])), | |
| sound_class=cls, has_drone=bool(r.get("has_drone_audio", False)), | |
| labeled=labeled, source_label=cls if labeled else "held-out", | |
| group_id=r["id"], split=split, | |
| text=f"challenge-5 {split_name}; sound_class={cls}; has_drone_audio={r.get('has_drone_audio', False)}", | |
| meta=dict(environment="outdoor field microphone", | |
| non_drone_source="" if r.get("has_drone_audio") else cls, | |
| _spec_png=os.path.join(CHALLENGE, r["spectrogram"]) if r.get("spectrogram") else None))) | |
| run_tasks(tasks, "challenge5") | |
| def ingest_itu(): | |
| root = os.path.join(ROOT, "raw", "itu-aris") | |
| groups = json.load(open(os.path.join(root, "groups.json"))) | |
| seg2group = {s["file"]: (s["label"], s["group"]) for s in groups["segments"]} | |
| assign = json.load(open(os.path.join(root, "splits", "split_interval_disjoint.json")))["assignment"] | |
| tasks = [] | |
| for f, (label, group) in seg2group.items(): | |
| split = {"train": "train", "val": "validation", "test": "test"}.get(assign.get(f, "train"), "train") | |
| tasks.append(T("itu-aris", f, ("path", os.path.join(root, f)), | |
| sound_class="drone_buzz" if label == "drone" else "ambient_outdoor", | |
| has_drone=label == "drone", source_label=label, group_id=group, split=split, | |
| text=f"ITU ARIS Lab 1s segment, group={group}; single 3.12h outdoor session, drone ranges few m to ~2 km, multiple UAV models", | |
| meta=dict(environment="outdoor field, single continuous session", | |
| non_drone_source="" if label == "drone" else "background"))) | |
| run_tasks(tasks, "itu-aris") | |
| def ingest_bucket_drone(): | |
| root = os.path.join(ROOT, "raw", "bucket", "drone") | |
| tasks = [] | |
| with open(os.path.join(root, "metadata.csv"), newline="", encoding="utf-8") as fh: | |
| for row in csv.DictReader(fh): | |
| fn = row["file_name"] | |
| p = os.path.join(root, fn) | |
| if not os.path.exists(p): | |
| continue | |
| label = row["label"] | |
| stem = os.path.basename(fn)[:-4] | |
| group = stem.rsplit("-", 1)[0] if "-" in stem else stem | |
| tasks.append(T("bucket-drone", fn, ("path", p), | |
| sound_class="drone_buzz" if label == "1" else "ambient_outdoor", | |
| has_drone=label == "1", source_label=f"label={label}", group_id=group, | |
| split=assign_split("bucket-drone", group), | |
| text="yaroslav2005 bucket drone/ (file,label csv)", | |
| meta=dict(non_drone_source="" if label == "1" else "non-drone audio"))) | |
| run_tasks(tasks, "bucket-drone") | |
| def ingest_test2(): | |
| root = os.path.join(ROOT, "raw", "bucket", "drone_all_test_datasets_2", "data") | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "*.parquet"))): | |
| shard = os.path.basename(f).split("-")[0] | |
| try: | |
| md = pq.ParquetFile(f).metadata | |
| n = md.num_rows | |
| except Exception as e: | |
| print("skip corrupt shard", f, str(e)[:60], flush=True) | |
| continue | |
| for i in range(n): | |
| tasks.append(T("drone-test2", f"{shard}/row{i}", ("pqrow", f, i), | |
| sound_class="?", has_drone=None, source_label="?", # filled in worker | |
| group_id="", split="train", text="", meta={"_shard": f, "_row": i})) | |
| # labels need the parquet; fill them in worker via _shard/_row | |
| run_pq_tasks(tasks, "drone-test2") | |
| def parse_das_path(fp, data_type): | |
| meta = {} | |
| m = re.search(r"(drone\d+)(?:-only)?/", fp) | |
| if m: | |
| meta["drone_model"] = m.group(1) | |
| m = re.search(r"mic-dist-(\d+)cm", fp) | |
| if m: | |
| meta["distance_m"] = float(m.group(1)) / 100.0 | |
| m = re.search(r"throttle-(\w+)", fp) | |
| if m: | |
| meta["throttle"] = m.group(1) | |
| m = re.search(r"speaker-loudness-(\d+)db", fp) | |
| if m: | |
| meta["non_drone_source"] = f"speaker playback @ {m.group(1)} dB" | |
| m = re.search(r"speaker-dist-(\d+)m", fp) | |
| if m and "distance_m" not in meta: | |
| meta["distance_m"] = float(m.group(1)) | |
| if "room" in fp: | |
| meta["environment"] = "indoor room (speaker + drone)" | |
| if data_type == "source-only": | |
| meta["non_drone_source"] = meta.get("non_drone_source", "speaker playback (source only)") | |
| meta["environment"] = "indoor room" | |
| return meta | |
| def ingest_das(): | |
| root = os.path.join(ROOT, "raw", "droneaudioset") | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "**", "*.parquet"), recursive=True)): | |
| rel = os.path.relpath(f, root) | |
| cfg = rel.split(os.sep)[0] | |
| try: | |
| md = pq.ParquetFile(f).metadata | |
| n = md.num_rows | |
| except Exception as e: | |
| print("skip corrupt shard", rel, str(e)[:60], flush=True) | |
| continue | |
| for i in range(n): | |
| tasks.append(T("droneaudioset", f"{cfg}/row{i}", ("pqrow", f, i), | |
| sound_class="?", has_drone=None, source_label="?", group_id="", | |
| split="train", text="", meta={"_shard": f, "_row": i})) | |
| run_pq_tasks(tasks, "droneaudioset") | |
| def ingest_dads(): | |
| root = os.path.join(ROOT, "raw", "dads_full", "data") | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "*.parquet"))): | |
| try: | |
| md = pq.ParquetFile(f).metadata | |
| n = md.num_rows | |
| except Exception as e: | |
| print("skip corrupt shard", f, str(e)[:60], flush=True) | |
| continue | |
| for i in range(n): | |
| tasks.append(T("dads", f"row{i}", ("pqrow", f, i), | |
| sound_class="?", has_drone=None, source_label="?", group_id="", | |
| split="train", text="", meta={"_shard": f, "_row": i})) | |
| run_pq_tasks(tasks, "dads") | |
| Z15_MODELS = {"d1": "DJI Matrice 300", "d2": "DJI Mavic 3", "d3": "DJI Mavic Air 2S", | |
| "d4": "DJI Mavic Air 2", "d5": "DJI Mavic Mini 2", "d6": "DJI Mavic 2 Pro", | |
| "d7": "DJI Mavic 2 Pro", "d8": "DJI Mavic 3", "d9": "DJI Phantom 4", | |
| "d10": "DJI Mavic 2 Zoom", "d11": "DJI Mavic Mini 2", "d12": "Yuneec H520", | |
| "d13": "Yuneec H520E RTK", "d14": "S900", "d15": "X6D", "d16": "Y6", | |
| "d17": "DJI Phantom 4"} | |
| Z15_PLACE = {"230315": ("Kielce (urban)", "5C, 1014 hPa, 51% RH, wind 22 km/h"), | |
| "230415": ("Gdańsk (urban)", "10C, 1015 hPa, 78% RH, wind 25 km/h"), | |
| "230416": ("Dębogórze near Gdańsk (rural)", "6C, 1022 hPa, 93% RH, wind 18 km/h"), | |
| "230417": ("Dębogórze near Gdańsk (rural)", "7C, 1030 hPa, 80% RH, wind 22 km/h"), | |
| "230418": ("Łapalice near Gdańsk (rural)", "8C, 1033 hPa, 90% RH, wind 25 km/h")} | |
| def parse_z15_name(name): | |
| base = os.path.basename(name).lower().replace(".wav", "") | |
| m = re.match(r"(x[46])_(d\d+)_([a-z0-9]+)_(\d+)m_(\d+)m_(\d{6})_(\d+)(_sekwencja)?$", base) | |
| meta = {"environment": "outdoor"} | |
| if not m: | |
| return meta, "uav_hover" | |
| _, dno, model_key, alt, dist, date, num, seq = m.groups() | |
| meta["drone_model"] = Z15_MODELS.get(dno, model_key) | |
| meta["altitude_m"] = float(alt) | |
| meta["distance_m"] = float(dist) | |
| place, weather = Z15_PLACE.get(date, ("unknown", "")) | |
| meta["environment"] = f"outdoor {place}" | |
| meta["weather"] = weather | |
| if seq: | |
| meta["non_drone_source"] = "speech commands (EN/PL) in drone presence" | |
| return meta, "speech_sequence_in_drone_presence" | |
| return meta, "uav_hover" | |
| def ingest_zenodo_miesikowska(): | |
| root = os.path.join(ROOT, "raw", "zenodo", "15190811") | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "*.wav"))): | |
| name = os.path.basename(f) | |
| meta, label = parse_z15_name(name) | |
| group = re.sub(r"_\d+(_sekwencja)?\.wav$", "", name.lower()) | |
| tasks.append(T("zenodo-15190811", name, ("path", f), sound_class="drone_buzz", has_drone=True, | |
| source_label=label, group_id=group, split=assign_split("zenodo-15190811", group), | |
| text="Zenodo 15190811 (Miesikowska, Kielce Univ. Tech., CC-BY-4.0); OLYMPUS LS11 outdoor recording", | |
| meta=meta)) | |
| run_tasks(tasks, "zenodo-15190811") | |
| ROT2YAW = {"Front": 0.0, "Right": 90.0, "Back": 180.0, "Left": 270.0} | |
| def ingest_uavirbase(): | |
| root = os.path.join(ROOT, "raw", "zenodo", "15391924") | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "*", "output.wav"))): | |
| sess = os.path.basename(os.path.dirname(f)) | |
| lp = os.path.join(os.path.dirname(f), "label.json") | |
| label = json.load(open(lp)) if os.path.exists(lp) else {} | |
| drone = label.get("drone", {}) | |
| wx = (label.get("weather_data") or {}).get("measurements", {}) | |
| weather = "; ".join(f"{k}={v}" for k, v in wx.items() if k in | |
| ("air temperature", "air humidity", "wind speed", "wind direction")) | |
| is_drone = drone.get("sound_source") == "Drone" | |
| meta = dict( | |
| drone_model=drone.get("type") or "", | |
| distance_m=float(drone.get("distance")) if drone.get("distance") else -1.0, | |
| altitude_m=float(drone.get("height")) if drone.get("height") else -1.0, | |
| azimuth_deg=float(drone.get("azimuth")) if drone.get("azimuth") is not None else -1.0, | |
| yaw_deg=ROT2YAW.get(drone.get("rotation") or "", -1.0), | |
| speed_mps=0.0 if drone.get("movement") == "Static" else -1.0, | |
| environment="outdoor open field (mic array, 52.26N 20.90E Poland)", | |
| weather=weather, | |
| non_drone_source="" if is_drone else "ambient noise (no UAV)") | |
| tasks.append(T("uavirbase", f"{sess}/output.wav", ("path", f), | |
| sound_class="drone_buzz" if is_drone else "ambient_outdoor", | |
| has_drone=is_drone, source_label="uav_flight" if is_drone else "ambient_only", | |
| group_id=sess, split=assign_split("uavirbase", sess), | |
| text=f"UaVirBASE session {sess}; label={json.dumps(drone)[:200]}", meta=meta)) | |
| run_tasks(tasks, "uavirbase") | |
| DD_MODELS = {"T1": "DJI Matrice 300 RTK", "T2": "DJI Phantom 4 Pro Plus", | |
| "T3": "Mavic 2 Enterprise Dual", "T6": "DJI Phantom 4 Pro V2.0"} | |
| FACING2YAW = {"front": 0.0, "right": 90.0, "l/r": 90.0, "back": 180.0, "left": 270.0} | |
| def parse_dd_setup(setup): | |
| meta = {} | |
| if not setup: | |
| return meta | |
| s = setup.lower() | |
| m = re.search(r"(\d+)\s*m\s+distance", s) | |
| if m: | |
| meta["distance_m"] = float(m.group(1)) | |
| m = re.search(r"(\d+)\s*m\s+above", s) | |
| if m: | |
| meta["altitude_m"] = float(m.group(1)) | |
| for facing, yaw in FACING2YAW.items(): | |
| if f"facing {facing}" in s: | |
| meta["yaw_deg"] = yaw | |
| break | |
| if "towards" in s: | |
| meta["azimuth_deg"] = 0.0 | |
| meta["speed_mps"] = -2.0 | |
| elif "away" in s: | |
| meta["azimuth_deg"] = 180.0 | |
| meta["speed_mps"] = -2.0 | |
| elif "parallel" in s: | |
| meta["azimuth_deg"] = 90.0 | |
| meta["speed_mps"] = -2.0 | |
| elif "stationary" in s: | |
| meta["speed_mps"] = 0.0 | |
| elif "random" in s: | |
| meta["speed_mps"] = -2.0 | |
| return meta | |
| def ingest_dronedata(): | |
| root = os.path.join(ROOT, "raw", "bucket", "DroneData", "extracted") | |
| manifest = os.path.join(ROOT, "build", "dronedata_manifest.json") | |
| if not os.path.exists(manifest): | |
| print("no dronedata manifest, skipping") | |
| return | |
| man = json.load(open(manifest)) | |
| import pandas as pd | |
| try: | |
| cs = pd.read_excel(os.path.join(ROOT, "raw", "bucket", "DroneData", "Data Collection.xlsx"), | |
| sheet_name="Collection state") | |
| setup = {str(r["File name"]): str(r["Setup"]) for _, r in cs.iterrows()} | |
| except Exception as e: | |
| print("xlsx read failed:", e) | |
| setup = {} | |
| tasks = [] | |
| for f in sorted(glob.glob(os.path.join(root, "**", "*.wav"), recursive=True)): | |
| rel = os.path.relpath(f, root).replace(os.sep, "/") | |
| code = man.get(rel) | |
| if code is None: | |
| continue | |
| sess_file = rel.split("/")[1][:-4] | |
| sess = sess_file.rsplit("_", 1)[0] | |
| if code == "NO": | |
| cls, drone, nds = "ambient_outdoor", False, "no object (ambient)" | |
| elif code == "OO": | |
| cls, drone, nds = "unknown", False, "other object (person running / non-UAV)" | |
| else: | |
| cls, drone, nds = "drone_buzz", True, "" | |
| meta = parse_dd_setup(setup.get(sess_file, "")) | |
| meta["drone_model"] = DD_MODELS.get(code, "") | |
| meta["environment"] = "outdoor radar test site" | |
| meta["non_drone_source"] = nds | |
| tasks.append(T("dronedata", rel, ("path", f), sound_class=cls, has_drone=drone, | |
| source_label=f"class_code={code}", group_id=sess, | |
| split=assign_split("dronedata", sess), | |
| text=f"DroneData session {sess_file} class {code}; setup={setup.get(sess_file, '')}", | |
| meta=meta)) | |
| run_tasks(tasks, "dronedata") | |
| # ---- pqrow sources need labels from the parquet; wrap load to fill them ---- | |
| _PQ_CACHE = {} | |
| _RG_CACHE = {} | |
| def pq_row_fields(shard, i): | |
| pf = _PQ_CACHE.get(shard) | |
| if pf is None: | |
| pf = pq.ParquetFile(shard) | |
| if len(_PQ_CACHE) > 2: | |
| _PQ_CACHE.clear() | |
| _RG_CACHE.clear() | |
| _PQ_CACHE[shard] = pf | |
| md = pf.metadata | |
| rg = 0 | |
| base = 0 | |
| if md.num_row_groups > 1: | |
| acc = 0 | |
| for g in range(md.num_row_groups): | |
| acc += md.row_group(g).num_rows | |
| if i < acc: | |
| rg = g | |
| base = acc - md.row_group(g).num_rows | |
| break | |
| key = (shard, rg) | |
| if key not in _RG_CACHE: | |
| if len(_RG_CACHE) > 3: | |
| _RG_CACHE.clear() | |
| _RG_CACHE[key] = pf.read_row_group(rg) # keep as arrow Table (cheap, lazy-ish) | |
| return _RG_CACHE[key], i - base | |
| def decode_audio_cell(audio_col, i): | |
| """audio_col: ChunkedArray/StructArray of audio structs. Returns (x float32, sr, path). | |
| The array field may be nested list<list<double>> (Array2D, das) — decode via | |
| arrow flatten (zero-copy) instead of as_py() to avoid huge Python lists. | |
| """ | |
| sa = audio_col.combine_chunks() if isinstance(audio_col, pa.ChunkedArray) else audio_col | |
| names = list(sa.type.names) | |
| if "bytes" in names: | |
| b = sa.field("bytes")[i] | |
| if b.is_valid and b.as_py() is not None: | |
| return read_audio_any(b.as_py()) + (sa.field("path")[i].as_py() if "path" in names else None,) | |
| row = sa.field("array")[i] | |
| a = row.values if isinstance(row, pa.Scalar) else row | |
| m = len(row) | |
| while pa.types.is_list(a.type) or pa.types.is_large_list(a.type): | |
| a = a.flatten() | |
| x = np.asarray(a.to_numpy(zero_copy_only=False), dtype=np.float32).reshape(m, -1) | |
| sr = int(sa.field("sampling_rate")[i].as_py()) | |
| path = sa.field("path")[i].as_py() if "path" in names else None | |
| return x, sr, path | |
| def _work_pq(t): | |
| """Variant of work() for parquet-row tasks (dads/test2/das): labels come from the shard.""" | |
| tab, i = pq_row_fields(t["meta"]["_shard"], t["meta"]["_row"]) | |
| x, sr, apath = decode_audio_cell(tab.column("audio"), i) | |
| if t["source"] == "dads": | |
| label = int(tab.column("label")[i].as_py()) | |
| path = apath or f"row{i}.wav" | |
| emit_rows("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}", x, sr, "drone_buzz" if label == 1 else "ambient_outdoor", | |
| label == 1, True, f"label={label}", f"{os.path.basename(t['meta']['_shard'])}/{path}", assign_split("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}"), | |
| f"DADS sample; source path={path}", | |
| dict(non_drone_source="" if label == 1 else "no-drone (mixed sources)")) | |
| elif t["source"] == "drone-test2": | |
| label = int(tab.column("label")[i].as_py()) | |
| shard = re.sub(r"-\d+-of-\d+\.parquet$", "", os.path.basename(t["meta"]["_shard"])) | |
| path = apath or f"{i}.wav" | |
| stem = path[:-4] if path.endswith(".wav") else path | |
| group = f"{shard}__{stem}" # group by file, not row: chunks of one file stay together | |
| emit_rows("drone-test2", f"{shard}/{path}#r{i}", x, sr, | |
| "drone_buzz" if label == 1 else "ambient_outdoor", label == 1, True, | |
| f"label={label} ({'drone' if label == 1 else 'other'})", group, | |
| assign_split("drone-test2", group), f"external test shard {shard}", | |
| dict(environment="parabolic-mic test recording" if shard.startswith("parabole") else "", | |
| non_drone_source="" if label == 1 else "other (unspecified)")) | |
| elif t["source"] == "droneaudioset": | |
| dt = tab.column("data_type")[i].as_py() | |
| fp = tab.column("file_path")[i].as_py() | |
| cfg = os.path.relpath(t["meta"]["_shard"], os.path.join(ROOT, "raw", "droneaudioset")).split(os.sep)[0] | |
| group = os.path.dirname(fp) or fp | |
| emit_rows("droneaudioset", f"{cfg}/{fp}", x, sr, | |
| "drone_buzz" if dt != "source-only" else "ambient_outdoor", dt != "source-only", True, | |
| f"data_type={dt}", group, assign_split("droneaudioset", group), | |
| f"DroneAudioSet {cfg}; file_path={fp}", parse_das_path(fp, dt)) | |
| return 1 | |
| def run_pq_tasks(tasks, tag): | |
| n = 0 | |
| pool = Pool(NPROC, initializer=_init_worker) | |
| for _ in pool.imap_unordered(_work_pq, tasks, chunksize=16): | |
| n += 1 | |
| if n % 5000 == 0: | |
| print(f"[{tag}] {n}/{len(tasks)}", flush=True) | |
| pool.close() | |
| pool.join() | |
| print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True) | |
| if __name__ == "__main__": | |
| which = sys.argv[1:] or ["challenge5", "itu", "bucket_drone", "test2", "das", "dads", | |
| "zenodo15190811", "uavirbase", "dronedata"] | |
| state_p = os.path.join(ROOT, "build", "build_state.json") | |
| state = json.load(open(state_p)) if os.path.exists(state_p) else {} | |
| which = [w for w in which if not state.get(w)] | |
| print("running:", which, "skipped:", [k for k in state if k not in which], flush=True) | |
| if "challenge5" in which: ingest_challenge5() | |
| if "itu" in which: ingest_itu() | |
| if "bucket_drone" in which: ingest_bucket_drone() | |
| if "test2" in which: ingest_test2() # labels filled in worker | |
| if "das" in which: ingest_das() | |
| if "dads" in which: ingest_dads() | |
| if "zenodo15190811" in which: ingest_zenodo_miesikowska() | |
| if "uavirbase" in which: ingest_uavirbase() | |
| if "dronedata" in which: ingest_dronedata() | |
| for w in which: | |
| state[w] = True | |
| json.dump(state, open(state_p, "w")) | |
| print("BUILD DONE", flush=True) | |