audioset: struct audio + incremental shard push in as_extract_part.py
Browse files- as_extract_part.py +44 -13
as_extract_part.py
CHANGED
|
@@ -15,7 +15,7 @@ import numpy as np
|
|
| 15 |
import soundfile as sf
|
| 16 |
import pyarrow.parquet as pq
|
| 17 |
from huggingface_hub import HfApi, HfFileSystem
|
| 18 |
-
from datasets import Dataset, Features,
|
| 19 |
|
| 20 |
EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"}
|
| 21 |
VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
|
|
@@ -90,15 +90,46 @@ for fi, f in enumerate(files):
|
|
| 90 |
print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 91 |
|
| 92 |
print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
import soundfile as sf
|
| 16 |
import pyarrow.parquet as pq
|
| 17 |
from huggingface_hub import HfApi, HfFileSystem
|
| 18 |
+
from datasets import Dataset, Features, Value, Sequence
|
| 19 |
|
| 20 |
EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption", "Boom"}
|
| 21 |
VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
|
|
|
|
| 90 |
print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 91 |
|
| 92 |
print(f"DONE scan: expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 93 |
+
|
| 94 |
+
# incremental, resumable push: parquet shards straight into the staging repo
|
| 95 |
+
import pyarrow as pa, pyarrow.parquet as pq
|
| 96 |
+
from huggingface_hub import CommitOperationAdd
|
| 97 |
+
STAGE = "Tonic/raw-audioset-explosion"
|
| 98 |
+
api.update_repo_settings(repo_id=STAGE, repo_type="dataset", gated="manual")
|
| 99 |
+
existing = set(api.list_repo_files(STAGE, repo_type="dataset"))
|
| 100 |
+
COLS = ["video_id", "labels", "human_labels", "sound_class", "non_drone_source", "audio"]
|
| 101 |
+
shard = 0
|
| 102 |
+
batch = []
|
| 103 |
+
def flush():
|
| 104 |
+
global shard, batch
|
| 105 |
+
if not batch:
|
| 106 |
+
return
|
| 107 |
+
tbl = pa.table({
|
| 108 |
+
"video_id": [r["video_id"] for r in batch],
|
| 109 |
+
"labels": [r["labels"] for r in batch],
|
| 110 |
+
"human_labels": [r["human_labels"] for r in batch],
|
| 111 |
+
"sound_class": [r["sound_class"] for r in batch],
|
| 112 |
+
"non_drone_source": [r["non_drone_source"] for r in batch],
|
| 113 |
+
"audio": [r["audio"] for r in batch],
|
| 114 |
+
})
|
| 115 |
+
while True:
|
| 116 |
+
rp = f"{CONFIG}/data/part-{shard:05d}.parquet"
|
| 117 |
+
if rp not in existing:
|
| 118 |
+
break
|
| 119 |
+
shard += 1
|
| 120 |
+
buf = io.BytesIO()
|
| 121 |
+
pq.write_table(tbl, buf, compression="zstd")
|
| 122 |
+
api.create_commit(repo_id=STAGE, repo_type="dataset", token=os.environ["HF_TOKEN"],
|
| 123 |
+
operations=[CommitOperationAdd(path_in_repo=rp, path_or_fileobj=buf.getvalue())],
|
| 124 |
+
commit_message=f"{CONFIG} part {shard} ({len(batch)} rows)")
|
| 125 |
+
print(f"pushed {rp} rows={len(batch)} (expl={n_expl} veh={n_veh})", flush=True)
|
| 126 |
+
shard += 1
|
| 127 |
+
batch = []
|
| 128 |
+
|
| 129 |
+
# stream rows into batches during scan is complex; instead re-scan cheaply is not possible,
|
| 130 |
+
# so flush at end in chunks:
|
| 131 |
+
CH = 400
|
| 132 |
+
for i in range(0, len(rows), CH):
|
| 133 |
+
batch = rows[i:i + CH]
|
| 134 |
+
flush()
|
| 135 |
+
print(f"PUSHED {STAGE} config={CONFIG} shards={shard} rows={len(rows)} (gated)", flush=True)
|