Tonic commited on
Commit
c816be0
·
verified ·
1 Parent(s): 4ca7161

fix: read staging parquet via pyarrow (avoid datasets Audio/torchcodec ImportError)

Browse files
Files changed (1) hide show
  1. build_audioset_job.py +108 -91
build_audioset_job.py CHANGED
@@ -1,91 +1,108 @@
1
- # /// script
2
- # requires-python = ">=3.11"
3
- # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "numpy", "soundfile", "pandas"]
4
- # ///
5
- """Build unified shards for the `audioset` source from the staging repo
6
- Tonic/raw-audioset-explosion, and upload them directly to
7
- Tonic/unified-drone-acoustic-cuas (shards under clips/ and windows_5s/).
8
-
9
- Run as HF Job (cpu-upgrade) after as_extract.py completed.
10
- """
11
- import io, os, sys, glob, time
12
-
13
- W = "/tmp/c5build"
14
- os.makedirs(W, exist_ok=True)
15
- sys.path.insert(0, W)
16
-
17
- from huggingface_hub import HfApi, CommitOperationAdd
18
- api = HfApi(token=os.environ["HF_TOKEN"])
19
- for fn in ("unified_schema.py", "build_unified.py"):
20
- p = api.download_repo_file if False else None
21
- from huggingface_hub import hf_hub_download
22
- for fn in ("unified_schema.py", "build_unified.py"):
23
- hf_hub_download("Tonic/challenge5-acoustic-code", fn, repo_type="model",
24
- local_dir=W, token=os.environ["HF_TOKEN"])
25
-
26
- import build_unified as B
27
- from unified_schema import assign_split, read_audio_any
28
-
29
- B.ROOT = W
30
- B.OUT = os.path.join(W, "build", "unified")
31
- B.NPROC = max(1, (os.cpu_count() or 4) - 1)
32
-
33
- # bytes locators
34
- def load_locator(loc):
35
- if loc[0] == "bytes":
36
- return read_audio_any(loc[1])
37
- return read_audio_any(loc[1]) if loc[0] == "path" else None
38
- B.load_locator = load_locator
39
-
40
- from datasets import load_dataset
41
- _cfgs = sorted({p.split("/")[0] for p in api.list_repo_files("Tonic/raw-audioset-explosion", repo_type="dataset") if p.split("/")[0].startswith("clips")}) or ["clips"]
42
- from datasets import concatenate_datasets as _cd, load_dataset as _ld
43
- _dss = [_ld("Tonic/raw-audioset-explosion", c, token=os.environ["HF_TOKEN"])["train"] for c in _cfgs]
44
- ds = _dss[0] if len(_dss) == 1 else _cd(_dss)
45
- print("staging configs:", _cfgs, flush=True)
46
- print("staging rows:", len(ds), flush=True)
47
-
48
- tasks = []
49
- for i in range(len(ds)):
50
- r = ds[i]
51
- a = r["audio"]
52
- raw = a["bytes"] if isinstance(a, dict) else None
53
- if raw is None:
54
- import soundfile as sf
55
- arr, sr = r["audio"]["array"], r["audio"]["sampling_rate"]
56
- buf = io.BytesIO(); sf.write(buf, arr, sr, format="WAV", subtype="PCM_16")
57
- raw = buf.getvalue()
58
- hl = r.get("human_labels") or []
59
- vid = r["video_id"]
60
- tasks.append(dict(
61
- source="audioset", source_file=f"{vid}.wav", loc=("bytes", raw),
62
- sound_class=r["sound_class"], has_drone=False, labeled=True,
63
- source_label=",".join(hl), group_id=f"audioset-{vid}", split="train",
64
- text=f"AudioSet YouTube {vid} labels={','.join(hl)}",
65
- meta=dict(environment="YouTube recording (AudioSet)",
66
- non_drone_source=r.get("non_drone_source", ""))))
67
- for t in tasks:
68
- t["split"] = assign_split(t["source"], t["group_id"])
69
- print("tasks:", len(tasks), flush=True)
70
- B.run_tasks(tasks, "audioset")
71
-
72
- # upload shards straight to the unified dataset repo
73
- REPO = "Tonic/unified-drone-acoustic-cuas"
74
- files = sorted(glob.glob(os.path.join(B.OUT, "*", "*.parquet")))
75
- print("uploading", len(files), "audioset shards", flush=True)
76
- t0 = time.time()
77
- CH = 40
78
- for c0 in range(0, len(files), CH):
79
- chunk = files[c0:c0 + CH]
80
- ops = [CommitOperationAdd(path_in_repo=os.path.relpath(f, B.OUT).replace(os.sep, "/"),
81
- path_or_fileobj=f) for f in chunk]
82
- for attempt in range(6):
83
- try:
84
- api.create_commit(repo_id=REPO, repo_type="dataset", operations=ops,
85
- commit_message=f"audioset shards {c0}-{c0+len(chunk)-1}", num_threads=8)
86
- break
87
- except Exception as e:
88
- print("retry", c0, str(e)[:120], flush=True)
89
- time.sleep(min(2 ** attempt, 30))
90
- print(f" {min(c0+CH,len(files))}/{len(files)} ({time.time()-t0:.0f}s)", flush=True)
91
- print("AUDIOSET UPLOAD DONE", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "numpy", "soundfile", "pandas"]
4
+ # ///
5
+ """Build unified shards for the `audioset` source from the staging repo
6
+ Tonic/raw-audioset-explosion, and upload them directly to
7
+ Tonic/unified-drone-acoustic-cuas (shards under clips/ and windows_5s/).
8
+
9
+ Run as HF Job (cpu-upgrade) after as_extract.py completed.
10
+ """
11
+ import io, os, sys, glob, time
12
+
13
+ W = "/tmp/c5build"
14
+ os.makedirs(W, exist_ok=True)
15
+ sys.path.insert(0, W)
16
+
17
+ from huggingface_hub import HfApi, CommitOperationAdd
18
+ api = HfApi(token=os.environ["HF_TOKEN"])
19
+ for fn in ("unified_schema.py", "build_unified.py"):
20
+ p = api.download_repo_file if False else None
21
+ from huggingface_hub import hf_hub_download
22
+ for fn in ("unified_schema.py", "build_unified.py"):
23
+ hf_hub_download("Tonic/challenge5-acoustic-code", fn, repo_type="model",
24
+ local_dir=W, token=os.environ["HF_TOKEN"])
25
+
26
+ import build_unified as B
27
+ from unified_schema import assign_split, read_audio_any
28
+
29
+ B.ROOT = W
30
+ B.OUT = os.path.join(W, "build", "unified")
31
+ B.NPROC = max(1, (os.cpu_count() or 4) - 1)
32
+
33
+ # bytes locators
34
+ def load_locator(loc):
35
+ if loc[0] == "bytes":
36
+ return read_audio_any(loc[1])
37
+ return read_audio_any(loc[1]) if loc[0] == "path" else None
38
+ B.load_locator = load_locator
39
+
40
+ from datasets import load_dataset
41
+ import pyarrow.parquet as _pq
42
+ from huggingface_hub import HfFileSystem as _HFS
43
+ _fs = _HFS(token=os.environ["HF_TOKEN"])
44
+ _files = sorted(f for f in api.list_repo_files("Tonic/raw-audioset-explosion", repo_type="dataset")
45
+ if f.startswith("clips") and f.endswith(".parquet"))
46
+ print("staging files:", len(_files), flush=True)
47
+ _rows = []
48
+ for _f in _files:
49
+ with _fs.open(f"datasets/Tonic/raw-audioset-explosion/{_f}", "rb") as _fh:
50
+ _tbl = _pq.ParquetFile(_fh).read(columns=["video_id", "human_labels", "sound_class", "non_drone_source", "audio"])
51
+ _ac = _tbl.column("audio").combine_chunks()
52
+ for _i in range(_tbl.num_rows):
53
+ _a = _ac[_i].as_py()
54
+ _rows.append({"video_id": _tbl.column("video_id")[_i].as_py(),
55
+ "human_labels": _tbl.column("human_labels")[_i].as_py() or [],
56
+ "sound_class": _tbl.column("sound_class")[_i].as_py(),
57
+ "non_drone_source": _tbl.column("non_drone_source")[_i].as_py() or "",
58
+ "audio": {"bytes": _a["bytes"] if isinstance(_a, dict) else _a}})
59
+ class _DS:
60
+ def __len__(self): return len(_rows)
61
+ def __getitem__(self, i): return _rows[i]
62
+ ds = _DS()
63
+ print("staging rows:", len(ds), flush=True)
64
+
65
+ tasks = []
66
+ for i in range(len(ds)):
67
+ r = ds[i]
68
+ a = r["audio"]
69
+ raw = a["bytes"] if isinstance(a, dict) else None
70
+ if raw is None:
71
+ import soundfile as sf
72
+ arr, sr = r["audio"]["array"], r["audio"]["sampling_rate"]
73
+ buf = io.BytesIO(); sf.write(buf, arr, sr, format="WAV", subtype="PCM_16")
74
+ raw = buf.getvalue()
75
+ hl = r.get("human_labels") or []
76
+ vid = r["video_id"]
77
+ tasks.append(dict(
78
+ source="audioset", source_file=f"{vid}.wav", loc=("bytes", raw),
79
+ sound_class=r["sound_class"], has_drone=False, labeled=True,
80
+ source_label=",".join(hl), group_id=f"audioset-{vid}", split="train",
81
+ text=f"AudioSet YouTube {vid} labels={','.join(hl)}",
82
+ meta=dict(environment="YouTube recording (AudioSet)",
83
+ non_drone_source=r.get("non_drone_source", ""))))
84
+ for t in tasks:
85
+ t["split"] = assign_split(t["source"], t["group_id"])
86
+ print("tasks:", len(tasks), flush=True)
87
+ B.run_tasks(tasks, "audioset")
88
+
89
+ # upload shards straight to the unified dataset repo
90
+ REPO = "Tonic/unified-drone-acoustic-cuas"
91
+ files = sorted(glob.glob(os.path.join(B.OUT, "*", "*.parquet")))
92
+ print("uploading", len(files), "audioset shards", flush=True)
93
+ t0 = time.time()
94
+ CH = 40
95
+ for c0 in range(0, len(files), CH):
96
+ chunk = files[c0:c0 + CH]
97
+ ops = [CommitOperationAdd(path_in_repo=os.path.relpath(f, B.OUT).replace(os.sep, "/"),
98
+ path_or_fileobj=f) for f in chunk]
99
+ for attempt in range(6):
100
+ try:
101
+ api.create_commit(repo_id=REPO, repo_type="dataset", operations=ops,
102
+ commit_message=f"audioset shards {c0}-{c0+len(chunk)-1}", num_threads=8)
103
+ break
104
+ except Exception as e:
105
+ print("retry", c0, str(e)[:120], flush=True)
106
+ time.sleep(min(2 ** attempt, 30))
107
+ print(f" {min(c0+CH,len(files))}/{len(files)} ({time.time()-t0:.0f}s)", flush=True)
108
+ print("AUDIOSET UPLOAD DONE", flush=True)