fix: read staging parquet via pyarrow (avoid datasets Audio/torchcodec ImportError)
Browse files- build_audioset_job.py +108 -91
build_audioset_job.py
CHANGED
|
@@ -1,91 +1,108 @@
|
|
| 1 |
-
# /// script
|
| 2 |
-
# requires-python = ">=3.11"
|
| 3 |
-
# dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "numpy", "soundfile", "pandas"]
|
| 4 |
-
# ///
|
| 5 |
-
"""Build unified shards for the `audioset` source from the staging repo
|
| 6 |
-
Tonic/raw-audioset-explosion, and upload them directly to
|
| 7 |
-
Tonic/unified-drone-acoustic-cuas (shards under clips/ and windows_5s/).
|
| 8 |
-
|
| 9 |
-
Run as HF Job (cpu-upgrade) after as_extract.py completed.
|
| 10 |
-
"""
|
| 11 |
-
import io, os, sys, glob, time
|
| 12 |
-
|
| 13 |
-
W = "/tmp/c5build"
|
| 14 |
-
os.makedirs(W, exist_ok=True)
|
| 15 |
-
sys.path.insert(0, W)
|
| 16 |
-
|
| 17 |
-
from huggingface_hub import HfApi, CommitOperationAdd
|
| 18 |
-
api = HfApi(token=os.environ["HF_TOKEN"])
|
| 19 |
-
for fn in ("unified_schema.py", "build_unified.py"):
|
| 20 |
-
p = api.download_repo_file if False else None
|
| 21 |
-
from huggingface_hub import hf_hub_download
|
| 22 |
-
for fn in ("unified_schema.py", "build_unified.py"):
|
| 23 |
-
hf_hub_download("Tonic/challenge5-acoustic-code", fn, repo_type="model",
|
| 24 |
-
local_dir=W, token=os.environ["HF_TOKEN"])
|
| 25 |
-
|
| 26 |
-
import build_unified as B
|
| 27 |
-
from unified_schema import assign_split, read_audio_any
|
| 28 |
-
|
| 29 |
-
B.ROOT = W
|
| 30 |
-
B.OUT = os.path.join(W, "build", "unified")
|
| 31 |
-
B.NPROC = max(1, (os.cpu_count() or 4) - 1)
|
| 32 |
-
|
| 33 |
-
# bytes locators
|
| 34 |
-
def load_locator(loc):
|
| 35 |
-
if loc[0] == "bytes":
|
| 36 |
-
return read_audio_any(loc[1])
|
| 37 |
-
return read_audio_any(loc[1]) if loc[0] == "path" else None
|
| 38 |
-
B.load_locator = load_locator
|
| 39 |
-
|
| 40 |
-
from datasets import load_dataset
|
| 41 |
-
|
| 42 |
-
from
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
print("staging
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.11"
|
| 3 |
+
# dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "numpy", "soundfile", "pandas"]
|
| 4 |
+
# ///
|
| 5 |
+
"""Build unified shards for the `audioset` source from the staging repo
|
| 6 |
+
Tonic/raw-audioset-explosion, and upload them directly to
|
| 7 |
+
Tonic/unified-drone-acoustic-cuas (shards under clips/ and windows_5s/).
|
| 8 |
+
|
| 9 |
+
Run as HF Job (cpu-upgrade) after as_extract.py completed.
|
| 10 |
+
"""
|
| 11 |
+
import io, os, sys, glob, time
|
| 12 |
+
|
| 13 |
+
W = "/tmp/c5build"
|
| 14 |
+
os.makedirs(W, exist_ok=True)
|
| 15 |
+
sys.path.insert(0, W)
|
| 16 |
+
|
| 17 |
+
from huggingface_hub import HfApi, CommitOperationAdd
|
| 18 |
+
api = HfApi(token=os.environ["HF_TOKEN"])
|
| 19 |
+
for fn in ("unified_schema.py", "build_unified.py"):
|
| 20 |
+
p = api.download_repo_file if False else None
|
| 21 |
+
from huggingface_hub import hf_hub_download
|
| 22 |
+
for fn in ("unified_schema.py", "build_unified.py"):
|
| 23 |
+
hf_hub_download("Tonic/challenge5-acoustic-code", fn, repo_type="model",
|
| 24 |
+
local_dir=W, token=os.environ["HF_TOKEN"])
|
| 25 |
+
|
| 26 |
+
import build_unified as B
|
| 27 |
+
from unified_schema import assign_split, read_audio_any
|
| 28 |
+
|
| 29 |
+
B.ROOT = W
|
| 30 |
+
B.OUT = os.path.join(W, "build", "unified")
|
| 31 |
+
B.NPROC = max(1, (os.cpu_count() or 4) - 1)
|
| 32 |
+
|
| 33 |
+
# bytes locators
|
| 34 |
+
def load_locator(loc):
|
| 35 |
+
if loc[0] == "bytes":
|
| 36 |
+
return read_audio_any(loc[1])
|
| 37 |
+
return read_audio_any(loc[1]) if loc[0] == "path" else None
|
| 38 |
+
B.load_locator = load_locator
|
| 39 |
+
|
| 40 |
+
from datasets import load_dataset
|
| 41 |
+
import pyarrow.parquet as _pq
|
| 42 |
+
from huggingface_hub import HfFileSystem as _HFS
|
| 43 |
+
_fs = _HFS(token=os.environ["HF_TOKEN"])
|
| 44 |
+
_files = sorted(f for f in api.list_repo_files("Tonic/raw-audioset-explosion", repo_type="dataset")
|
| 45 |
+
if f.startswith("clips") and f.endswith(".parquet"))
|
| 46 |
+
print("staging files:", len(_files), flush=True)
|
| 47 |
+
_rows = []
|
| 48 |
+
for _f in _files:
|
| 49 |
+
with _fs.open(f"datasets/Tonic/raw-audioset-explosion/{_f}", "rb") as _fh:
|
| 50 |
+
_tbl = _pq.ParquetFile(_fh).read(columns=["video_id", "human_labels", "sound_class", "non_drone_source", "audio"])
|
| 51 |
+
_ac = _tbl.column("audio").combine_chunks()
|
| 52 |
+
for _i in range(_tbl.num_rows):
|
| 53 |
+
_a = _ac[_i].as_py()
|
| 54 |
+
_rows.append({"video_id": _tbl.column("video_id")[_i].as_py(),
|
| 55 |
+
"human_labels": _tbl.column("human_labels")[_i].as_py() or [],
|
| 56 |
+
"sound_class": _tbl.column("sound_class")[_i].as_py(),
|
| 57 |
+
"non_drone_source": _tbl.column("non_drone_source")[_i].as_py() or "",
|
| 58 |
+
"audio": {"bytes": _a["bytes"] if isinstance(_a, dict) else _a}})
|
| 59 |
+
class _DS:
|
| 60 |
+
def __len__(self): return len(_rows)
|
| 61 |
+
def __getitem__(self, i): return _rows[i]
|
| 62 |
+
ds = _DS()
|
| 63 |
+
print("staging rows:", len(ds), flush=True)
|
| 64 |
+
|
| 65 |
+
tasks = []
|
| 66 |
+
for i in range(len(ds)):
|
| 67 |
+
r = ds[i]
|
| 68 |
+
a = r["audio"]
|
| 69 |
+
raw = a["bytes"] if isinstance(a, dict) else None
|
| 70 |
+
if raw is None:
|
| 71 |
+
import soundfile as sf
|
| 72 |
+
arr, sr = r["audio"]["array"], r["audio"]["sampling_rate"]
|
| 73 |
+
buf = io.BytesIO(); sf.write(buf, arr, sr, format="WAV", subtype="PCM_16")
|
| 74 |
+
raw = buf.getvalue()
|
| 75 |
+
hl = r.get("human_labels") or []
|
| 76 |
+
vid = r["video_id"]
|
| 77 |
+
tasks.append(dict(
|
| 78 |
+
source="audioset", source_file=f"{vid}.wav", loc=("bytes", raw),
|
| 79 |
+
sound_class=r["sound_class"], has_drone=False, labeled=True,
|
| 80 |
+
source_label=",".join(hl), group_id=f"audioset-{vid}", split="train",
|
| 81 |
+
text=f"AudioSet YouTube {vid} labels={','.join(hl)}",
|
| 82 |
+
meta=dict(environment="YouTube recording (AudioSet)",
|
| 83 |
+
non_drone_source=r.get("non_drone_source", ""))))
|
| 84 |
+
for t in tasks:
|
| 85 |
+
t["split"] = assign_split(t["source"], t["group_id"])
|
| 86 |
+
print("tasks:", len(tasks), flush=True)
|
| 87 |
+
B.run_tasks(tasks, "audioset")
|
| 88 |
+
|
| 89 |
+
# upload shards straight to the unified dataset repo
|
| 90 |
+
REPO = "Tonic/unified-drone-acoustic-cuas"
|
| 91 |
+
files = sorted(glob.glob(os.path.join(B.OUT, "*", "*.parquet")))
|
| 92 |
+
print("uploading", len(files), "audioset shards", flush=True)
|
| 93 |
+
t0 = time.time()
|
| 94 |
+
CH = 40
|
| 95 |
+
for c0 in range(0, len(files), CH):
|
| 96 |
+
chunk = files[c0:c0 + CH]
|
| 97 |
+
ops = [CommitOperationAdd(path_in_repo=os.path.relpath(f, B.OUT).replace(os.sep, "/"),
|
| 98 |
+
path_or_fileobj=f) for f in chunk]
|
| 99 |
+
for attempt in range(6):
|
| 100 |
+
try:
|
| 101 |
+
api.create_commit(repo_id=REPO, repo_type="dataset", operations=ops,
|
| 102 |
+
commit_message=f"audioset shards {c0}-{c0+len(chunk)-1}", num_threads=8)
|
| 103 |
+
break
|
| 104 |
+
except Exception as e:
|
| 105 |
+
print("retry", c0, str(e)[:120], flush=True)
|
| 106 |
+
time.sleep(min(2 ** attempt, 30))
|
| 107 |
+
print(f" {min(c0+CH,len(files))}/{len(files)} ({time.time()-t0:.0f}s)", flush=True)
|
| 108 |
+
print("AUDIOSET UPLOAD DONE", flush=True)
|