Download as_extract.py from Tonic/challenge5-acoustic-code: direct link, hf CLI and curl.
- Browser
- Download file 4.43 kB
-
https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/as_extract.py
- Command line
-
hf download hf://Tonic/challenge5-acoustic-code/as_extract.py
-
curl -L -o as_extract.py https://huggingface.co/Tonic/challenge5-acoustic-code/resolve/main/as_extract.py
4.43 kB
| # /// script | |
| # requires-python = ">=3.11" | |
| # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "soundfile", "numpy", "scipy"] | |
| # /// | |
| """Extract explosion-family + vehicle/mechanical segments from agkphysics/AudioSet | |
| via HfFileSystem range reads (label columns first; audio column only for shards with | |
| matches). Pushes gated staging dataset Tonic/raw-audioset-explosion (16 kHz mono wav). | |
| """ | |
| import io, os | |
| import numpy as np | |
| import soundfile as sf | |
| import pyarrow.parquet as pq | |
| from huggingface_hub import HfApi, HfFileSystem | |
| from datasets import Dataset, Features, Audio, Value, Sequence | |
| EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"} | |
| VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle", | |
| "Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle", | |
| "Car passing by", "Race car, auto racing", "Airplane, aeroplane"} | |
| CAP_EXPL, CAP_VEH = 3000, 3000 | |
| TARGET_SR = 16000 | |
| api = HfApi(token=os.environ["HF_TOKEN"]) | |
| fs = HfFileSystem(token=os.environ["HF_TOKEN"]) | |
| REPO = "agkphysics/AudioSet" | |
| SPLITS = ("bal_train", "eval", "unbal_train") | |
| files = [f for f in api.list_repo_files(REPO, repo_type="dataset") | |
| if f.startswith("data/") and f.endswith(".parquet") | |
| and f.split("/")[1] in SPLITS] | |
| print(f"{len(files)} parquet shards", flush=True) | |
| rows = [] | |
| n_expl = n_veh = 0 | |
| def encode_wav(raw): | |
| arr, sr = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False) | |
| if arr.ndim > 1: | |
| arr = arr.mean(axis=1) | |
| if sr != TARGET_SR: | |
| g = np.gcd(sr, TARGET_SR) | |
| from scipy.signal import resample_poly | |
| arr = resample_poly(arr, TARGET_SR // g, sr // g) | |
| buf = io.BytesIO() | |
| sf.write(buf, arr.astype(np.float32), TARGET_SR, format="WAV", subtype="PCM_16") | |
| return buf.getvalue() | |
| for fi, f in enumerate(files): | |
| if n_expl >= CAP_EXPL and n_veh >= CAP_VEH: | |
| break | |
| with fs.open(f"datasets/{REPO}/{f}", "rb") as fh: | |
| pf = pq.ParquetFile(fh) | |
| for rg in range(pf.num_row_groups): | |
| lab = pf.read_row_group(rg, columns=["video_id", "human_labels", "labels"]) | |
| vids = lab.column("video_id").to_pylist() | |
| hls = lab.column("human_labels").to_pylist() | |
| lbs = lab.column("labels").to_pylist() | |
| match = [] | |
| for i, hl in enumerate(hls): | |
| hl = set(hl or []) | |
| if n_expl < CAP_EXPL and hl & EXPL: | |
| match.append((i, "explosion")) | |
| elif n_veh < CAP_VEH and hl & VEH: | |
| match.append((i, "vehicle_mechanical")) | |
| if not match: | |
| continue | |
| full = pf.read_row_group(rg) # all columns; struct audio reads fine here | |
| acol = full.column("audio").combine_chunks() | |
| for i, cls in match: | |
| raw = acol[i].as_py()["bytes"] | |
| hl = hls[i] or [] | |
| rows.append({ | |
| "video_id": vids[i], | |
| "labels": lbs[i] or [], | |
| "human_labels": list(hl), | |
| "sound_class": cls, | |
| "non_drone_source": "audioset:" + ",".join(sorted(hl)), | |
| "audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"}, | |
| }) | |
| if cls == "explosion": | |
| n_expl += 1 | |
| else: | |
| n_veh += 1 | |
| print(f"[{fi}/{len(files)}] {f} rg{rg}: {len(match)} matches (expl={n_expl} veh={n_veh})", flush=True) | |
| if fi % 50 == 0: | |
| print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True) | |
| print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True) | |
| feats = Features({ | |
| "video_id": Value("string"), | |
| "labels": Sequence(Value("string")), | |
| "human_labels": Sequence(Value("string")), | |
| "sound_class": Value("string"), | |
| "non_drone_source": Value("string"), | |
| "audio": Audio(sampling_rate=TARGET_SR), | |
| }) | |
| out = Dataset.from_list(rows, features=feats) | |
| out.push_to_hub("Tonic/raw-audioset-explosion", config_name="clips", token=os.environ["HF_TOKEN"]) | |
| api.update_repo_settings(repo_id="Tonic/raw-audioset-explosion", repo_type="dataset", gated="manual") | |
| print("PUSHED Tonic/raw-audioset-explosion (gated)", flush=True) | |