challenge5-acoustic-code / build_unified.py
Tonic's picture
Upload build_unified.py with huggingface_hub
9c56095 verified
Raw History Blame Contribute Delete
27.4 kB
"""Build the unified drone-acoustic dataset as parquet shards. FULL sources, worker-local writes.
Configs:
clips - one row per source clip, full-length 16 kHz mono WAV
windows_5s - 5.0 s windows with 224x224 log-mel PNG + metadata + clip_sample_id
Each worker process writes its own shard files (pid-tagged), so no large payloads
cross process boundaries. Run per source: build_unified.py <source> [...]
"""
import json, os, sys, csv, glob, re
for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS",
"NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", "BLIS_NUM_THREADS"):
os.environ.setdefault(_v, "1")
from multiprocessing import Pool
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
from unified_schema import (CLASSES, CLASS_ID, SPLITS, SPLIT_ID, SOURCE_REPOS, TARGET_SR,
WINDOW_SEC, WINDOW_SAMPLES, assign_split, resample, wav_bytes,
read_audio_any, logmel_png, make_windows)
ROOT = os.path.dirname(os.path.abspath(__file__))
CHALLENGE = os.path.dirname(ROOT)
OUT = os.path.join(ROOT, "build", "unified")
BATCH = 500
NPROC = max(1, (os.cpu_count() or 4) - 1)
META_FIELDS = [
("drone_model", pa.string()), ("distance_m", pa.float32()), ("altitude_m", pa.float32()),
("azimuth_deg", pa.float32()), ("yaw_deg", pa.float32()), ("speed_mps", pa.float32()),
("throttle", pa.string()), ("environment", pa.string()), ("non_drone_source", pa.string()),
("weather", pa.string()),
]
META_DEFAULT = dict(drone_model="", distance_m=-1.0, altitude_m=-1.0, azimuth_deg=-1.0,
yaw_deg=-1.0, speed_mps=-1.0, throttle="", environment="",
non_drone_source="", weather="")
BASE_FIELDS = [
("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()),
("source_file", pa.string()),
("orig_sampling_rate", pa.int32()), ("duration_s", pa.float32()),
("window_index", pa.int32()), ("window_start_s", pa.float32()), ("window_sec", pa.float32()),
("sound_class", pa.int8()), ("has_drone_audio", pa.bool_()), ("labeled", pa.bool_()),
("source_label", pa.string()), ("group_id", pa.string()), ("split", pa.int8()),
("text", pa.string()),
] + META_FIELDS
CLIP_FIELDS = [("sample_id", pa.string()), ("source", pa.string()), ("source_repo", pa.string()),
("source_file", pa.string()),
("audio", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))] + BASE_FIELDS[4:]
WINDOW_FIELDS = BASE_FIELDS + [("clip_sample_id", pa.string()),
("spectrogram", pa.struct([("bytes", pa.binary()), ("path", pa.string())]))]
_W = None # per-process writer
class LocalWriter:
def __init__(self):
self.buf = {}
self.shards = {}
def _flush(self, key):
rows = self.buf.get(key)
if not rows:
return
config, split, source = key
fields = WINDOW_FIELDS if config == "windows_5s" else CLIP_FIELDS
cols = {name: [] for name, _ in fields}
for r in rows:
for name, _ in fields:
cols[name].append(r.get(name))
arrays = []
for name, typ in fields:
if name in ("audio", "spectrogram"):
arrays.append(pa.StructArray.from_arrays(
[pa.array([c[0] for c in cols[name]], type=pa.binary()),
pa.array([c[1] for c in cols[name]], type=pa.string())],
names=["bytes", "path"]))
else:
arrays.append(pa.array(cols[name], type=typ))
table = pa.Table.from_arrays(arrays, schema=pa.schema(fields))
d = os.path.join(OUT, config)
os.makedirs(d, exist_ok=True)
n = self.shards.get(key, 0)
pq.write_table(table, os.path.join(d, f"{split}-{source}-p{os.getpid()}-{n:05d}.parquet"),
compression="zstd")
self.shards[key] = n + 1
self.buf[key] = []
def add(self, config, row):
key = (config, SPLITS[row["split"]], row["source"])
self.buf.setdefault(key, []).append(row)
if len(self.buf[key]) >= BATCH:
self._flush(key)
def close(self):
for key in list(self.buf):
self._flush(key)
def _init_worker():
global _W
_W = LocalWriter()
from multiprocessing.util import Finalize
Finalize(None, _W.close, exitpriority=16)
def emit_rows(source, source_file, x, sr, sound_class, has_drone, labeled, source_label,
group_id, split, text="", meta=None):
"""Build clip row + window rows and hand them to the process-local writer."""
if x.ndim > 1:
x = x.mean(axis=1)
x16 = resample(x, sr)
dur = len(x16) / TARGET_SR
sid = f"{source}__{source_file.replace('/', '__').replace(' ', '_')}"
m = dict(META_DEFAULT)
spec_png = (meta or {}).get("_spec_png")
m.update({k: v for k, v in (meta or {}).items() if k != "_spec_png"})
base = dict(source=source, source_repo=SOURCE_REPOS[source], source_file=source_file,
orig_sampling_rate=int(sr), sound_class=CLASS_ID[sound_class],
has_drone_audio=bool(has_drone), labeled=bool(labeled),
source_label=source_label, group_id=group_id, split=SPLIT_ID[split], text=text, **m)
_W.add("clips", dict(base, sample_id=sid,
audio=(wav_bytes(x16), (os.path.basename(source_file) or "clip.wav")),
duration_s=float(dur), window_index=0, window_start_s=0.0,
window_sec=float(dur)))
for i, start_s, chunk in make_windows(x16):
chunk = np.pad(chunk, (0, max(0, WINDOW_SAMPLES - len(chunk))))
wdur = min(WINDOW_SEC, dur - start_s)
if i == 0 and spec_png:
with open(spec_png, "rb") as fh:
spec = fh.read() # use the challenge's own precomputed PNG verbatim
else:
spec = logmel_png(chunk)
_W.add("windows_5s", dict(base, sample_id=f"{sid}__w{i:04d}", clip_sample_id=sid,
duration_s=float(max(wdur, 0.0)), window_index=i,
window_start_s=float(start_s), window_sec=WINDOW_SEC,
spectrogram=(spec, f"w{i:04d}.png")))
def load_locator(loc):
"""loc: ('path', p) or ('pqrow', parquet_path, row_idx)."""
if loc[0] == "path":
return read_audio_any(loc[1])
_, p, i = loc
t = pq.read_table(p)
a = t.column("audio")[i].as_py()
return read_audio_any(a["bytes"])
def _work(t):
try:
x, sr = load_locator(t["loc"])
emit_rows(t["source"], t["source_file"], x, sr, t["sound_class"], t["has_drone"],
t["labeled"], t["source_label"], t["group_id"], t["split"], t.get("text", ""),
t.get("meta"))
except Exception as e:
print("ERR", t["source"], t["source_file"], str(e)[:120], flush=True)
return 1
def run_tasks(tasks, tag):
"""tasks: list of dicts with 'loc' + label/meta fields. Worker-local writing."""
n = 0
pool = Pool(NPROC, initializer=_init_worker)
for _ in pool.imap_unordered(_work, tasks, chunksize=4):
n += 1
if n % 5000 == 0:
print(f"[{tag}] {n}/{len(tasks)}", flush=True)
pool.close()
pool.join() # clean worker shutdown so atexit flushes buffers
print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True)
def T(source, source_file, loc, sound_class="ambient_outdoor", has_drone=False, labeled=True,
source_label="", group_id="", split="train", text="", meta=None):
return dict(source=source, source_file=source_file, loc=loc, sound_class=sound_class,
has_drone=has_drone, labeled=labeled, source_label=source_label,
group_id=group_id, split=split, text=text, meta=meta)
# ---------- sources ----------
def ingest_challenge5():
tasks = []
for split_name, split in (("train", "train"), ("valid", "validation"), ("test", "test")):
for line in open(os.path.join(CHALLENGE, "dataset", split_name, "labels.jsonl"), encoding="utf-8"):
if not line.strip():
continue
r = json.loads(line)
labeled = "sound_class" in r
cls = r.get("sound_class", "unknown")
tasks.append(T("challenge5", r["id"] + ".wav", ("path", os.path.join(CHALLENGE, r["audio"])),
sound_class=cls, has_drone=bool(r.get("has_drone_audio", False)),
labeled=labeled, source_label=cls if labeled else "held-out",
group_id=r["id"], split=split,
text=f"challenge-5 {split_name}; sound_class={cls}; has_drone_audio={r.get('has_drone_audio', False)}",
meta=dict(environment="outdoor field microphone",
non_drone_source="" if r.get("has_drone_audio") else cls,
_spec_png=os.path.join(CHALLENGE, r["spectrogram"]) if r.get("spectrogram") else None)))
run_tasks(tasks, "challenge5")
def ingest_itu():
root = os.path.join(ROOT, "raw", "itu-aris")
groups = json.load(open(os.path.join(root, "groups.json")))
seg2group = {s["file"]: (s["label"], s["group"]) for s in groups["segments"]}
assign = json.load(open(os.path.join(root, "splits", "split_interval_disjoint.json")))["assignment"]
tasks = []
for f, (label, group) in seg2group.items():
split = {"train": "train", "val": "validation", "test": "test"}.get(assign.get(f, "train"), "train")
tasks.append(T("itu-aris", f, ("path", os.path.join(root, f)),
sound_class="drone_buzz" if label == "drone" else "ambient_outdoor",
has_drone=label == "drone", source_label=label, group_id=group, split=split,
text=f"ITU ARIS Lab 1s segment, group={group}; single 3.12h outdoor session, drone ranges few m to ~2 km, multiple UAV models",
meta=dict(environment="outdoor field, single continuous session",
non_drone_source="" if label == "drone" else "background")))
run_tasks(tasks, "itu-aris")
def ingest_bucket_drone():
root = os.path.join(ROOT, "raw", "bucket", "drone")
tasks = []
with open(os.path.join(root, "metadata.csv"), newline="", encoding="utf-8") as fh:
for row in csv.DictReader(fh):
fn = row["file_name"]
p = os.path.join(root, fn)
if not os.path.exists(p):
continue
label = row["label"]
stem = os.path.basename(fn)[:-4]
group = stem.rsplit("-", 1)[0] if "-" in stem else stem
tasks.append(T("bucket-drone", fn, ("path", p),
sound_class="drone_buzz" if label == "1" else "ambient_outdoor",
has_drone=label == "1", source_label=f"label={label}", group_id=group,
split=assign_split("bucket-drone", group),
text="yaroslav2005 bucket drone/ (file,label csv)",
meta=dict(non_drone_source="" if label == "1" else "non-drone audio")))
run_tasks(tasks, "bucket-drone")
def ingest_test2():
root = os.path.join(ROOT, "raw", "bucket", "drone_all_test_datasets_2", "data")
tasks = []
for f in sorted(glob.glob(os.path.join(root, "*.parquet"))):
shard = os.path.basename(f).split("-")[0]
try:
md = pq.ParquetFile(f).metadata
n = md.num_rows
except Exception as e:
print("skip corrupt shard", f, str(e)[:60], flush=True)
continue
for i in range(n):
tasks.append(T("drone-test2", f"{shard}/row{i}", ("pqrow", f, i),
sound_class="?", has_drone=None, source_label="?", # filled in worker
group_id="", split="train", text="", meta={"_shard": f, "_row": i}))
# labels need the parquet; fill them in worker via _shard/_row
run_pq_tasks(tasks, "drone-test2")
def parse_das_path(fp, data_type):
meta = {}
m = re.search(r"(drone\d+)(?:-only)?/", fp)
if m:
meta["drone_model"] = m.group(1)
m = re.search(r"mic-dist-(\d+)cm", fp)
if m:
meta["distance_m"] = float(m.group(1)) / 100.0
m = re.search(r"throttle-(\w+)", fp)
if m:
meta["throttle"] = m.group(1)
m = re.search(r"speaker-loudness-(\d+)db", fp)
if m:
meta["non_drone_source"] = f"speaker playback @ {m.group(1)} dB"
m = re.search(r"speaker-dist-(\d+)m", fp)
if m and "distance_m" not in meta:
meta["distance_m"] = float(m.group(1))
if "room" in fp:
meta["environment"] = "indoor room (speaker + drone)"
if data_type == "source-only":
meta["non_drone_source"] = meta.get("non_drone_source", "speaker playback (source only)")
meta["environment"] = "indoor room"
return meta
def ingest_das():
root = os.path.join(ROOT, "raw", "droneaudioset")
tasks = []
for f in sorted(glob.glob(os.path.join(root, "**", "*.parquet"), recursive=True)):
rel = os.path.relpath(f, root)
cfg = rel.split(os.sep)[0]
try:
md = pq.ParquetFile(f).metadata
n = md.num_rows
except Exception as e:
print("skip corrupt shard", rel, str(e)[:60], flush=True)
continue
for i in range(n):
tasks.append(T("droneaudioset", f"{cfg}/row{i}", ("pqrow", f, i),
sound_class="?", has_drone=None, source_label="?", group_id="",
split="train", text="", meta={"_shard": f, "_row": i}))
run_pq_tasks(tasks, "droneaudioset")
def ingest_dads():
root = os.path.join(ROOT, "raw", "dads_full", "data")
tasks = []
for f in sorted(glob.glob(os.path.join(root, "*.parquet"))):
try:
md = pq.ParquetFile(f).metadata
n = md.num_rows
except Exception as e:
print("skip corrupt shard", f, str(e)[:60], flush=True)
continue
for i in range(n):
tasks.append(T("dads", f"row{i}", ("pqrow", f, i),
sound_class="?", has_drone=None, source_label="?", group_id="",
split="train", text="", meta={"_shard": f, "_row": i}))
run_pq_tasks(tasks, "dads")
Z15_MODELS = {"d1": "DJI Matrice 300", "d2": "DJI Mavic 3", "d3": "DJI Mavic Air 2S",
"d4": "DJI Mavic Air 2", "d5": "DJI Mavic Mini 2", "d6": "DJI Mavic 2 Pro",
"d7": "DJI Mavic 2 Pro", "d8": "DJI Mavic 3", "d9": "DJI Phantom 4",
"d10": "DJI Mavic 2 Zoom", "d11": "DJI Mavic Mini 2", "d12": "Yuneec H520",
"d13": "Yuneec H520E RTK", "d14": "S900", "d15": "X6D", "d16": "Y6",
"d17": "DJI Phantom 4"}
Z15_PLACE = {"230315": ("Kielce (urban)", "5C, 1014 hPa, 51% RH, wind 22 km/h"),
"230415": ("Gdańsk (urban)", "10C, 1015 hPa, 78% RH, wind 25 km/h"),
"230416": ("Dębogórze near Gdańsk (rural)", "6C, 1022 hPa, 93% RH, wind 18 km/h"),
"230417": ("Dębogórze near Gdańsk (rural)", "7C, 1030 hPa, 80% RH, wind 22 km/h"),
"230418": ("Łapalice near Gdańsk (rural)", "8C, 1033 hPa, 90% RH, wind 25 km/h")}
def parse_z15_name(name):
base = os.path.basename(name).lower().replace(".wav", "")
m = re.match(r"(x[46])_(d\d+)_([a-z0-9]+)_(\d+)m_(\d+)m_(\d{6})_(\d+)(_sekwencja)?$", base)
meta = {"environment": "outdoor"}
if not m:
return meta, "uav_hover"
_, dno, model_key, alt, dist, date, num, seq = m.groups()
meta["drone_model"] = Z15_MODELS.get(dno, model_key)
meta["altitude_m"] = float(alt)
meta["distance_m"] = float(dist)
place, weather = Z15_PLACE.get(date, ("unknown", ""))
meta["environment"] = f"outdoor {place}"
meta["weather"] = weather
if seq:
meta["non_drone_source"] = "speech commands (EN/PL) in drone presence"
return meta, "speech_sequence_in_drone_presence"
return meta, "uav_hover"
def ingest_zenodo_miesikowska():
root = os.path.join(ROOT, "raw", "zenodo", "15190811")
tasks = []
for f in sorted(glob.glob(os.path.join(root, "*.wav"))):
name = os.path.basename(f)
meta, label = parse_z15_name(name)
group = re.sub(r"_\d+(_sekwencja)?\.wav$", "", name.lower())
tasks.append(T("zenodo-15190811", name, ("path", f), sound_class="drone_buzz", has_drone=True,
source_label=label, group_id=group, split=assign_split("zenodo-15190811", group),
text="Zenodo 15190811 (Miesikowska, Kielce Univ. Tech., CC-BY-4.0); OLYMPUS LS11 outdoor recording",
meta=meta))
run_tasks(tasks, "zenodo-15190811")
ROT2YAW = {"Front": 0.0, "Right": 90.0, "Back": 180.0, "Left": 270.0}
def ingest_uavirbase():
root = os.path.join(ROOT, "raw", "zenodo", "15391924")
tasks = []
for f in sorted(glob.glob(os.path.join(root, "*", "output.wav"))):
sess = os.path.basename(os.path.dirname(f))
lp = os.path.join(os.path.dirname(f), "label.json")
label = json.load(open(lp)) if os.path.exists(lp) else {}
drone = label.get("drone", {})
wx = (label.get("weather_data") or {}).get("measurements", {})
weather = "; ".join(f"{k}={v}" for k, v in wx.items() if k in
("air temperature", "air humidity", "wind speed", "wind direction"))
is_drone = drone.get("sound_source") == "Drone"
meta = dict(
drone_model=drone.get("type") or "",
distance_m=float(drone.get("distance")) if drone.get("distance") else -1.0,
altitude_m=float(drone.get("height")) if drone.get("height") else -1.0,
azimuth_deg=float(drone.get("azimuth")) if drone.get("azimuth") is not None else -1.0,
yaw_deg=ROT2YAW.get(drone.get("rotation") or "", -1.0),
speed_mps=0.0 if drone.get("movement") == "Static" else -1.0,
environment="outdoor open field (mic array, 52.26N 20.90E Poland)",
weather=weather,
non_drone_source="" if is_drone else "ambient noise (no UAV)")
tasks.append(T("uavirbase", f"{sess}/output.wav", ("path", f),
sound_class="drone_buzz" if is_drone else "ambient_outdoor",
has_drone=is_drone, source_label="uav_flight" if is_drone else "ambient_only",
group_id=sess, split=assign_split("uavirbase", sess),
text=f"UaVirBASE session {sess}; label={json.dumps(drone)[:200]}", meta=meta))
run_tasks(tasks, "uavirbase")
DD_MODELS = {"T1": "DJI Matrice 300 RTK", "T2": "DJI Phantom 4 Pro Plus",
"T3": "Mavic 2 Enterprise Dual", "T6": "DJI Phantom 4 Pro V2.0"}
FACING2YAW = {"front": 0.0, "right": 90.0, "l/r": 90.0, "back": 180.0, "left": 270.0}
def parse_dd_setup(setup):
meta = {}
if not setup:
return meta
s = setup.lower()
m = re.search(r"(\d+)\s*m\s+distance", s)
if m:
meta["distance_m"] = float(m.group(1))
m = re.search(r"(\d+)\s*m\s+above", s)
if m:
meta["altitude_m"] = float(m.group(1))
for facing, yaw in FACING2YAW.items():
if f"facing {facing}" in s:
meta["yaw_deg"] = yaw
break
if "towards" in s:
meta["azimuth_deg"] = 0.0
meta["speed_mps"] = -2.0
elif "away" in s:
meta["azimuth_deg"] = 180.0
meta["speed_mps"] = -2.0
elif "parallel" in s:
meta["azimuth_deg"] = 90.0
meta["speed_mps"] = -2.0
elif "stationary" in s:
meta["speed_mps"] = 0.0
elif "random" in s:
meta["speed_mps"] = -2.0
return meta
def ingest_dronedata():
root = os.path.join(ROOT, "raw", "bucket", "DroneData", "extracted")
manifest = os.path.join(ROOT, "build", "dronedata_manifest.json")
if not os.path.exists(manifest):
print("no dronedata manifest, skipping")
return
man = json.load(open(manifest))
import pandas as pd
try:
cs = pd.read_excel(os.path.join(ROOT, "raw", "bucket", "DroneData", "Data Collection.xlsx"),
sheet_name="Collection state")
setup = {str(r["File name"]): str(r["Setup"]) for _, r in cs.iterrows()}
except Exception as e:
print("xlsx read failed:", e)
setup = {}
tasks = []
for f in sorted(glob.glob(os.path.join(root, "**", "*.wav"), recursive=True)):
rel = os.path.relpath(f, root).replace(os.sep, "/")
code = man.get(rel)
if code is None:
continue
sess_file = rel.split("/")[1][:-4]
sess = sess_file.rsplit("_", 1)[0]
if code == "NO":
cls, drone, nds = "ambient_outdoor", False, "no object (ambient)"
elif code == "OO":
cls, drone, nds = "unknown", False, "other object (person running / non-UAV)"
else:
cls, drone, nds = "drone_buzz", True, ""
meta = parse_dd_setup(setup.get(sess_file, ""))
meta["drone_model"] = DD_MODELS.get(code, "")
meta["environment"] = "outdoor radar test site"
meta["non_drone_source"] = nds
tasks.append(T("dronedata", rel, ("path", f), sound_class=cls, has_drone=drone,
source_label=f"class_code={code}", group_id=sess,
split=assign_split("dronedata", sess),
text=f"DroneData session {sess_file} class {code}; setup={setup.get(sess_file, '')}",
meta=meta))
run_tasks(tasks, "dronedata")
# ---- pqrow sources need labels from the parquet; wrap load to fill them ----
_PQ_CACHE = {}
_RG_CACHE = {}
def pq_row_fields(shard, i):
pf = _PQ_CACHE.get(shard)
if pf is None:
pf = pq.ParquetFile(shard)
if len(_PQ_CACHE) > 2:
_PQ_CACHE.clear()
_RG_CACHE.clear()
_PQ_CACHE[shard] = pf
md = pf.metadata
rg = 0
base = 0
if md.num_row_groups > 1:
acc = 0
for g in range(md.num_row_groups):
acc += md.row_group(g).num_rows
if i < acc:
rg = g
base = acc - md.row_group(g).num_rows
break
key = (shard, rg)
if key not in _RG_CACHE:
if len(_RG_CACHE) > 3:
_RG_CACHE.clear()
_RG_CACHE[key] = pf.read_row_group(rg) # keep as arrow Table (cheap, lazy-ish)
return _RG_CACHE[key], i - base
def decode_audio_cell(audio_col, i):
"""audio_col: ChunkedArray/StructArray of audio structs. Returns (x float32, sr, path).
The array field may be nested list<list<double>> (Array2D, das) — decode via
arrow flatten (zero-copy) instead of as_py() to avoid huge Python lists.
"""
sa = audio_col.combine_chunks() if isinstance(audio_col, pa.ChunkedArray) else audio_col
names = list(sa.type.names)
if "bytes" in names:
b = sa.field("bytes")[i]
if b.is_valid and b.as_py() is not None:
return read_audio_any(b.as_py()) + (sa.field("path")[i].as_py() if "path" in names else None,)
row = sa.field("array")[i]
a = row.values if isinstance(row, pa.Scalar) else row
m = len(row)
while pa.types.is_list(a.type) or pa.types.is_large_list(a.type):
a = a.flatten()
x = np.asarray(a.to_numpy(zero_copy_only=False), dtype=np.float32).reshape(m, -1)
sr = int(sa.field("sampling_rate")[i].as_py())
path = sa.field("path")[i].as_py() if "path" in names else None
return x, sr, path
def _work_pq(t):
"""Variant of work() for parquet-row tasks (dads/test2/das): labels come from the shard."""
tab, i = pq_row_fields(t["meta"]["_shard"], t["meta"]["_row"])
x, sr, apath = decode_audio_cell(tab.column("audio"), i)
if t["source"] == "dads":
label = int(tab.column("label")[i].as_py())
path = apath or f"row{i}.wav"
emit_rows("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}", x, sr, "drone_buzz" if label == 1 else "ambient_outdoor",
label == 1, True, f"label={label}", f"{os.path.basename(t['meta']['_shard'])}/{path}", assign_split("dads", f"{os.path.basename(t['meta']['_shard'])}/{path}"),
f"DADS sample; source path={path}",
dict(non_drone_source="" if label == 1 else "no-drone (mixed sources)"))
elif t["source"] == "drone-test2":
label = int(tab.column("label")[i].as_py())
shard = re.sub(r"-\d+-of-\d+\.parquet$", "", os.path.basename(t["meta"]["_shard"]))
path = apath or f"{i}.wav"
stem = path[:-4] if path.endswith(".wav") else path
group = f"{shard}__{stem}" # group by file, not row: chunks of one file stay together
emit_rows("drone-test2", f"{shard}/{path}#r{i}", x, sr,
"drone_buzz" if label == 1 else "ambient_outdoor", label == 1, True,
f"label={label} ({'drone' if label == 1 else 'other'})", group,
assign_split("drone-test2", group), f"external test shard {shard}",
dict(environment="parabolic-mic test recording" if shard.startswith("parabole") else "",
non_drone_source="" if label == 1 else "other (unspecified)"))
elif t["source"] == "droneaudioset":
dt = tab.column("data_type")[i].as_py()
fp = tab.column("file_path")[i].as_py()
cfg = os.path.relpath(t["meta"]["_shard"], os.path.join(ROOT, "raw", "droneaudioset")).split(os.sep)[0]
group = os.path.dirname(fp) or fp
emit_rows("droneaudioset", f"{cfg}/{fp}", x, sr,
"drone_buzz" if dt != "source-only" else "ambient_outdoor", dt != "source-only", True,
f"data_type={dt}", group, assign_split("droneaudioset", group),
f"DroneAudioSet {cfg}; file_path={fp}", parse_das_path(fp, dt))
return 1
def run_pq_tasks(tasks, tag):
n = 0
pool = Pool(NPROC, initializer=_init_worker)
for _ in pool.imap_unordered(_work_pq, tasks, chunksize=16):
n += 1
if n % 5000 == 0:
print(f"[{tag}] {n}/{len(tasks)}", flush=True)
pool.close()
pool.join()
print(f"[{tag}] DONE {n}/{len(tasks)}", flush=True)
if __name__ == "__main__":
which = sys.argv[1:] or ["challenge5", "itu", "bucket_drone", "test2", "das", "dads",
"zenodo15190811", "uavirbase", "dronedata"]
state_p = os.path.join(ROOT, "build", "build_state.json")
state = json.load(open(state_p)) if os.path.exists(state_p) else {}
which = [w for w in which if not state.get(w)]
print("running:", which, "skipped:", [k for k in state if k not in which], flush=True)
if "challenge5" in which: ingest_challenge5()
if "itu" in which: ingest_itu()
if "bucket_drone" in which: ingest_bucket_drone()
if "test2" in which: ingest_test2() # labels filled in worker
if "das" in which: ingest_das()
if "dads" in which: ingest_dads()
if "zenodo15190811" in which: ingest_zenodo_miesikowska()
if "uavirbase" in which: ingest_uavirbase()
if "dronedata" in which: ingest_dronedata()
for w in which:
state[w] = True
json.dump(state, open(state_p, "w"))
print("BUILD DONE", flush=True)