challenge5-acoustic-code / as_extract_part.py
Tonic's picture
fix: create staging repo if missing before push (404 fix)
4ca7161 verified
Raw History Blame Contribute Delete
5.92 kB
# /// script
# requires-python = ">=3.11"
# dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "soundfile", "numpy", "scipy"]
# ///
"""Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i
and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel."""
import os as _os
PART = int(_os.environ.get("PART", "0"))
NPARTS = int(_os.environ.get("NPARTS", "4"))
CAP_EXPL = int(_os.environ.get("CAP_EXPL", "1500"))
CAP_VEH = int(_os.environ.get("CAP_VEH", "1500"))
CONFIG = _os.environ.get("CONFIG") or f"clips_p{PART}"
import io, os
import numpy as np
import soundfile as sf
import pyarrow.parquet as pq
from huggingface_hub import HfApi, HfFileSystem
from datasets import Dataset, Features, Value, Sequence
EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"}
VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
"Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle",
"Car passing by", "Race car, auto racing", "Airplane, aeroplane"}
TARGET_SR = 16000
api = HfApi(token=os.environ["HF_TOKEN"])
fs = HfFileSystem(token=os.environ["HF_TOKEN"])
REPO = "agkphysics/AudioSet"
SPLITS = ("bal_train", "eval", "unbal_train")
files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset")
if f.startswith("data/") and f.endswith(".parquet")
and f.split("/")[1] in SPLITS]
files = [f for i, f in enumerate(files_all) if i % NPARTS == PART]
print(f"{len(files)} parquet shards", flush=True)
rows = []
n_expl = n_veh = 0
def encode_wav(raw):
arr, sr = sf.read(io.BytesIO(raw), dtype="float32", always_2d=False)
if arr.ndim > 1:
arr = arr.mean(axis=1)
if sr != TARGET_SR:
g = np.gcd(sr, TARGET_SR)
from scipy.signal import resample_poly
arr = resample_poly(arr, TARGET_SR // g, sr // g)
buf = io.BytesIO()
sf.write(buf, arr.astype(np.float32), TARGET_SR, format="WAV", subtype="PCM_16")
return buf.getvalue()
for fi, f in enumerate(files):
if n_expl >= CAP_EXPL and n_veh >= CAP_VEH:
break
with fs.open(f"datasets/{REPO}/{f}", "rb") as fh:
pf = pq.ParquetFile(fh)
for rg in range(pf.num_row_groups):
lab = pf.read_row_group(rg, columns=["video_id", "human_labels", "labels"])
vids = lab.column("video_id").to_pylist()
hls = lab.column("human_labels").to_pylist()
lbs = lab.column("labels").to_pylist()
match = []
for i, hl in enumerate(hls):
hl = set(hl or [])
if n_expl < CAP_EXPL and hl & EXPL:
match.append((i, "explosion"))
elif n_veh < CAP_VEH and hl & VEH:
match.append((i, "vehicle_mechanical"))
if not match:
continue
full = pf.read_row_group(rg) # all columns; struct audio reads fine here
acol = full.column("audio").combine_chunks()
for i, cls in match:
raw = acol[i].as_py()["bytes"]
hl = hls[i] or []
rows.append({
"video_id": vids[i],
"labels": lbs[i] or [],
"human_labels": list(hl),
"sound_class": cls,
"non_drone_source": "audioset:" + ",".join(sorted(hl)),
"audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"},
})
if cls == "explosion":
n_expl += 1
else:
n_veh += 1
print(f"[{fi}/{len(files)}] {f} rg{rg}: {len(match)} matches (expl={n_expl} veh={n_veh})", flush=True)
if fi % 50 == 0:
print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
# incremental, resumable push: parquet shards straight into the staging repo
import pyarrow as pa, pyarrow.parquet as pq
from huggingface_hub import CommitOperationAdd
STAGE = "Tonic/raw-audioset-explosion"
try:
api.create_repo(STAGE, repo_type="dataset", exist_ok=True)
api.update_repo_settings(repo_id=STAGE, repo_type="dataset", gated="manual")
except Exception as e:
print("stage repo setup skipped:", repr(e)[:200], flush=True)
existing = set(api.list_repo_files(STAGE, repo_type="dataset"))
COLS = ["video_id", "labels", "human_labels", "sound_class", "non_drone_source", "audio"]
shard = 0
batch = []
def flush():
global shard, batch
if not batch:
return
tbl = pa.table({
"video_id": [r["video_id"] for r in batch],
"labels": [r["labels"] for r in batch],
"human_labels": [r["human_labels"] for r in batch],
"sound_class": [r["sound_class"] for r in batch],
"non_drone_source": [r["non_drone_source"] for r in batch],
"audio": [r["audio"] for r in batch],
})
while True:
rp = f"{CONFIG}/data/part-{shard:05d}.parquet"
if rp not in existing:
break
shard += 1
buf = io.BytesIO()
pq.write_table(tbl, buf, compression="zstd")
api.create_commit(repo_id=STAGE, repo_type="dataset", token=os.environ["HF_TOKEN"],
operations=[CommitOperationAdd(path_in_repo=rp, path_or_fileobj=buf.getvalue())],
commit_message=f"{CONFIG} part {shard} ({len(batch)} rows)")
print(f"pushed {rp} rows={len(batch)} (expl={n_expl} veh={n_veh})", flush=True)
shard += 1
batch = []
# stream rows into batches during scan is complex; instead re-scan cheaply is not possible,
# so flush at end in chunks:
CH = 400
for i in range(0, len(rows), CH):
batch = rows[i:i + CH]
flush()
print(f"PUSHED {STAGE} config={CONFIG} shards={shard} rows={len(rows)} (gated)", flush=True)