# /// script # requires-python = ">=3.11" # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "soundfile", "numpy", "scipy"] # /// """Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel.""" import os as _os PART = int(_os.environ.get("PART", "0")) NPARTS = int(_os.environ.get("NPARTS", "4")) CAP_EXPL = int(_os.environ.get("CAP_EXPL", "1500")) CAP_VEH = int(_os.environ.get("CAP_VEH", "1500")) CONFIG = _os.environ.get("CONFIG") or f"clips_p{PART}" import io, os import numpy as np import soundfile as sf import pyarrow.parquet as pq from huggingface_hub import HfApi, HfFileSystem from datasets import Dataset, Features, Value, Sequence EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"} VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle", "Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle", "Car passing by", "Race car, auto racing", "Airplane, aeroplane"} TARGET_SR = 16000 api = HfApi(token=os.environ["HF_TOKEN"]) fs = HfFileSystem(token=os.environ["HF_TOKEN"]) REPO = "agkphysics/AudioSet" SPLITS = ("bal_train", "eval", "unbal_train") files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset") if f.startswith("data/") and f.endswith(".parquet") and f.split("/")[1] in SPLITS] files = [f for i, f in enumerate(files_all) if i % NPARTS == PART] print(f"{len(files)} parquet shards", flush=True) rows = [] n_expl = n_veh = 0 def encode_wav(raw): arr, sr = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False) if arr.ndim > 1: arr = arr.mean(axis=1) if sr != TARGET_SR: g = np.gcd(sr, TARGET_SR) from scipy.signal import resample_poly arr = resample_poly(arr, TARGET_SR // g, sr // g) buf = io.BytesIO() sf.write(buf, arr.astype(np.float32), TARGET_SR, format="WAV", subtype="PCM_16") return buf.getvalue() for fi, f in enumerate(files): if n_expl >= CAP_EXPL and n_veh >= CAP_VEH: break with fs.open(f"datasets/{REPO}/{f}", "rb") as fh: pf = pq.ParquetFile(fh) for rg in range(pf.num_row_groups): lab = pf.read_row_group(rg, columns=["video_id", "human_labels", "labels"]) vids = lab.column("video_id").to_pylist() hls = lab.column("human_labels").to_pylist() lbs = lab.column("labels").to_pylist() match = [] for i, hl in enumerate(hls): hl = set(hl or []) if n_expl < CAP_EXPL and hl & EXPL: match.append((i, "explosion")) elif n_veh < CAP_VEH and hl & VEH: match.append((i, "vehicle_mechanical")) if not match: continue full = pf.read_row_group(rg) # all columns; struct audio reads fine here acol = full.column("audio").combine_chunks() for i, cls in match: raw = acol[i].as_py()["bytes"] hl = hls[i] or [] rows.append({ "video_id": vids[i], "labels": lbs[i] or [], "human_labels": list(hl), "sound_class": cls, "non_drone_source": "audioset:" + ",".join(sorted(hl)), "audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"}, }) if cls == "explosion": n_expl += 1 else: n_veh += 1 print(f"[{fi}/{len(files)}] {f} rg{rg}: {len(match)} matches (expl={n_expl} veh={n_veh})", flush=True) if fi % 50 == 0: print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True) print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True) # incremental, resumable push: parquet shards straight into the staging repo import pyarrow as pa, pyarrow.parquet as pq from huggingface_hub import CommitOperationAdd STAGE = "Tonic/raw-audioset-explosion" try: api.create_repo(STAGE, repo_type="dataset", exist_ok=True) api.update_repo_settings(repo_id=STAGE, repo_type="dataset", gated="manual") except Exception as e: print("stage repo setup skipped:", repr(e)[:200], flush=True) existing = set(api.list_repo_files(STAGE, repo_type="dataset")) COLS = ["video_id", "labels", "human_labels", "sound_class", "non_drone_source", "audio"] shard = 0 batch = [] def flush(): global shard, batch if not batch: return tbl = pa.table({ "video_id": [r["video_id"] for r in batch], "labels": [r["labels"] for r in batch], "human_labels": [r["human_labels"] for r in batch], "sound_class": [r["sound_class"] for r in batch], "non_drone_source": [r["non_drone_source"] for r in batch], "audio": [r["audio"] for r in batch], }) while True: rp = f"{CONFIG}/data/part-{shard:05d}.parquet" if rp not in existing: break shard += 1 buf = io.BytesIO() pq.write_table(tbl, buf, compression="zstd") api.create_commit(repo_id=STAGE, repo_type="dataset", token=os.environ["HF_TOKEN"], operations=[CommitOperationAdd(path_in_repo=rp, path_or_fileobj=buf.getvalue())], commit_message=f"{CONFIG} part {shard} ({len(batch)} rows)") print(f"pushed {rp} rows={len(batch)} (expl={n_expl} veh={n_veh})", flush=True) shard += 1 batch = [] # stream rows into batches during scan is complex; instead re-scan cheaply is not possible, # so flush at end in chunks: CH = 400 for i in range(0, len(rows), CH): batch = rows[i:i + CH] flush() print(f"PUSHED {STAGE} config={CONFIG} shards={shard} rows={len(rows)} (gated)", flush=True)