Upload as_extract.py with huggingface_hub
Browse files- as_extract.py +32 -33
as_extract.py
CHANGED
|
@@ -50,39 +50,38 @@ for fi, f in enumerate(files):
|
|
| 50 |
break
|
| 51 |
with fs.open(f"datasets/{REPO}/{f}", "rb") as fh:
|
| 52 |
pf = pq.ParquetFile(fh)
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
n_veh += 1
|
| 86 |
if fi % 50 == 0:
|
| 87 |
print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 88 |
|
|
|
|
| 50 |
break
|
| 51 |
with fs.open(f"datasets/{REPO}/{f}", "rb") as fh:
|
| 52 |
pf = pq.ParquetFile(fh)
|
| 53 |
+
for rg in range(pf.num_row_groups):
|
| 54 |
+
lab = pf.read_row_group(rg, columns=["video_id", "human_labels", "labels"])
|
| 55 |
+
vids = lab.column("video_id").to_pylist()
|
| 56 |
+
hls = lab.column("human_labels").to_pylist()
|
| 57 |
+
lbs = lab.column("labels").to_pylist()
|
| 58 |
+
match = []
|
| 59 |
+
for i, hl in enumerate(hls):
|
| 60 |
+
hl = set(hl or [])
|
| 61 |
+
if n_expl < CAP_EXPL and hl & EXPL:
|
| 62 |
+
match.append((i, "explosion"))
|
| 63 |
+
elif n_veh < CAP_VEH and hl & VEH:
|
| 64 |
+
match.append((i, "vehicle_mechanical"))
|
| 65 |
+
if not match:
|
| 66 |
+
continue
|
| 67 |
+
full = pf.read_row_group(rg) # all columns; struct audio reads fine here
|
| 68 |
+
acol = full.column("audio").combine_chunks()
|
| 69 |
+
for i, cls in match:
|
| 70 |
+
raw = acol[i].as_py()["bytes"]
|
| 71 |
+
hl = hls[i] or []
|
| 72 |
+
rows.append({
|
| 73 |
+
"video_id": vids[i],
|
| 74 |
+
"labels": lbs[i] or [],
|
| 75 |
+
"human_labels": list(hl),
|
| 76 |
+
"sound_class": cls,
|
| 77 |
+
"non_drone_source": "audioset:" + ",".join(sorted(hl)),
|
| 78 |
+
"audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"},
|
| 79 |
+
})
|
| 80 |
+
if cls == "explosion":
|
| 81 |
+
n_expl += 1
|
| 82 |
+
else:
|
| 83 |
+
n_veh += 1
|
| 84 |
+
print(f"[{fi}/{len(files)}] {f} rg{rg}: {len(match)} matches (expl={n_expl} veh={n_veh})", flush=True)
|
|
|
|
| 85 |
if fi % 50 == 0:
|
| 86 |
print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
|
| 87 |
|