Tonic commited on
Commit
b9e4311
·
verified ·
1 Parent(s): 8f7bb95

audioset: struct audio + incremental shard push in as_extract_part.py

Browse files
Files changed (1) hide show
  1. as_extract_part.py +44 -13
as_extract_part.py CHANGED
@@ -15,7 +15,7 @@ import numpy as np
15
  import soundfile as sf
16
  import pyarrow.parquet as pq
17
  from huggingface_hub import HfApi, HfFileSystem
18
- from datasets import Dataset, Features, Audio, Value, Sequence
19
 
20
  EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"}
21
  VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
@@ -90,15 +90,46 @@ for fi, f in enumerate(files):
90
  print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
91
 
92
  print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
93
- feats = Features({
94
- "video_id": Value("string"),
95
- "labels": Sequence(Value("string")),
96
- "human_labels": Sequence(Value("string")),
97
- "sound_class": Value("string"),
98
- "non_drone_source": Value("string"),
99
- "audio": Audio(sampling_rate=TARGET_SR),
100
- })
101
- out = Dataset.from_list(rows, features=feats)
102
- out.push_to_hub("Tonic/raw-audioset-explosion", config_name=CONFIG, token=os.environ["HF_TOKEN"])
103
- api.update_repo_settings(repo_id="Tonic/raw-audioset-explosion", repo_type="dataset", gated="manual")
104
- print(f"PUSHED Tonic/raw-audioset-explosion config={CONFIG} (gated)", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  import soundfile as sf
16
  import pyarrow.parquet as pq
17
  from huggingface_hub import HfApi, HfFileSystem
18
+ from datasets import Dataset, Features, Value, Sequence
19
 
20
  EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"}
21
  VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
 
90
  print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
91
 
92
  print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
93
+
94
+ # incremental, resumable push: parquet shards straight into the staging repo
95
+ import pyarrow as pa, pyarrow.parquet as pq
96
+ from huggingface_hub import CommitOperationAdd
97
+ STAGE = "Tonic/raw-audioset-explosion"
98
+ api.update_repo_settings(repo_id=STAGE, repo_type="dataset", gated="manual")
99
+ existing = set(api.list_repo_files(STAGE, repo_type="dataset"))
100
+ COLS = ["video_id", "labels", "human_labels", "sound_class", "non_drone_source", "audio"]
101
+ shard = 0
102
+ batch = []
103
+ def flush():
104
+ global shard, batch
105
+ if not batch:
106
+ return
107
+ tbl = pa.table({
108
+ "video_id": [r["video_id"] for r in batch],
109
+ "labels": [r["labels"] for r in batch],
110
+ "human_labels": [r["human_labels"] for r in batch],
111
+ "sound_class": [r["sound_class"] for r in batch],
112
+ "non_drone_source": [r["non_drone_source"] for r in batch],
113
+ "audio": [r["audio"] for r in batch],
114
+ })
115
+ while True:
116
+ rp = f"{CONFIG}/data/part-{shard:05d}.parquet"
117
+ if rp not in existing:
118
+ break
119
+ shard += 1
120
+ buf = io.BytesIO()
121
+ pq.write_table(tbl, buf, compression="zstd")
122
+ api.create_commit(repo_id=STAGE, repo_type="dataset", token=os.environ["HF_TOKEN"],
123
+ operations=[CommitOperationAdd(path_in_repo=rp, path_or_fileobj=buf.getvalue())],
124
+ commit_message=f"{CONFIG} part {shard} ({len(batch)} rows)")
125
+ print(f"pushed {rp} rows={len(batch)} (expl={n_expl} veh={n_veh})", flush=True)
126
+ shard += 1
127
+ batch = []
128
+
129
+ # stream rows into batches during scan is complex; instead re-scan cheaply is not possible,
130
+ # so flush at end in chunks:
131
+ CH = 400
132
+ for i in range(0, len(rows), CH):
133
+ batch = rows[i:i + CH]
134
+ flush()
135
+ print(f"PUSHED {STAGE} config={CONFIG} shards={shard} rows={len(rows)} (gated)", flush=True)