Tonic commited on
Commit
9fa4578
·
verified ·
1 Parent(s): 9c56095

Upload build_audioset_job.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. build_audioset_job.py +87 -0
build_audioset_job.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pyarrow", "numpy", "soundfile", "pandas"]
4
+ # ///
5
+ """Build unified shards for the `audioset` source from the staging repo
6
+ Tonic/raw-audioset-explosion, and upload them directly to
7
+ Tonic/unified-drone-acoustic-cuas (shards under clips/ and windows_5s/).
8
+
9
+ Run as HF Job (cpu-upgrade) after as_extract.py completed.
10
+ """
11
+ import io, os, sys, glob, time
12
+
13
+ W = "/tmp/c5build"
14
+ os.makedirs(W, exist_ok=True)
15
+ sys.path.insert(0, W)
16
+
17
+ from huggingface_hub import HfApi, CommitOperationAdd
18
+ api = HfApi(token=os.environ["HF_TOKEN"])
19
+ for fn in ("unified_schema.py", "build_unified.py"):
20
+ p = api.download_repo_file if False else None
21
+ from huggingface_hub import hf_hub_download
22
+ for fn in ("unified_schema.py", "build_unified.py"):
23
+ hf_hub_download("Tonic/challenge5-acoustic-code", fn, repo_type="model",
24
+ local_dir=W, token=os.environ["HF_TOKEN"])
25
+
26
+ import build_unified as B
27
+ from unified_schema import assign_split, read_audio_any
28
+
29
+ B.ROOT = W
30
+ B.OUT = os.path.join(W, "build", "unified")
31
+ B.NPROC = max(1, (os.cpu_count() or 4) - 1)
32
+
33
+ # bytes locators
34
+ def load_locator(loc):
35
+ if loc[0] == "bytes":
36
+ return read_audio_any(loc[1])
37
+ return read_audio_any(loc[1]) if loc[0] == "path" else None
38
+ B.load_locator = load_locator
39
+
40
+ from datasets import load_dataset
41
+ ds = load_dataset("Tonic/raw-audioset-explosion", "clips", token=os.environ["HF_TOKEN"])["train"]
42
+ print("staging rows:", len(ds), flush=True)
43
+
44
+ tasks = []
45
+ for i in range(len(ds)):
46
+ r = ds[i]
47
+ a = r["audio"]
48
+ raw = a["bytes"] if isinstance(a, dict) else None
49
+ if raw is None:
50
+ import soundfile as sf
51
+ arr, sr = r["audio"]["array"], r["audio"]["sampling_rate"]
52
+ buf = io.BytesIO(); sf.write(buf, arr, sr, format="WAV", subtype="PCM_16")
53
+ raw = buf.getvalue()
54
+ hl = r.get("human_labels") or []
55
+ vid = r["video_id"]
56
+ tasks.append(dict(
57
+ source="audioset", source_file=f"{vid}.wav", loc=("bytes", raw),
58
+ sound_class=r["sound_class"], has_drone=False, labeled=True,
59
+ source_label=",".join(hl), group_id=f"audioset-{vid}", split="train",
60
+ text=f"AudioSet YouTube {vid} labels={','.join(hl)}",
61
+ meta=dict(environment="YouTube recording (AudioSet)",
62
+ non_drone_source=r.get("non_drone_source", ""))))
63
+ for t in tasks:
64
+ t["split"] = assign_split(t["source"], t["group_id"])
65
+ print("tasks:", len(tasks), flush=True)
66
+ B.run_tasks(tasks, "audioset")
67
+
68
+ # upload shards straight to the unified dataset repo
69
+ REPO = "Tonic/unified-drone-acoustic-cuas"
70
+ files = sorted(glob.glob(os.path.join(B.OUT, "*", "*.parquet")))
71
+ print("uploading", len(files), "audioset shards", flush=True)
72
+ t0 = time.time()
73
+ CH = 40
74
+ for c0 in range(0, len(files), CH):
75
+ chunk = files[c0:c0 + CH]
76
+ ops = [CommitOperationAdd(path_in_repo=os.path.relpath(f, B.OUT).replace(os.sep, "/"),
77
+ path_or_fileobj=f) for f in chunk]
78
+ for attempt in range(6):
79
+ try:
80
+ api.create_commit(repo_id=REPO, repo_type="dataset", operations=ops,
81
+ commit_message=f"audioset shards {c0}-{c0+len(chunk)-1}", num_threads=8)
82
+ break
83
+ except Exception as e:
84
+ print("retry", c0, str(e)[:120], flush=True)
85
+ time.sleep(min(2 ** attempt, 30))
86
+ print(f" {min(c0+CH,len(files))}/{len(files)} ({time.time()-t0:.0f}s)", flush=True)
87
+ print("AUDIOSET UPLOAD DONE", flush=True)