Tonic commited on
Commit
2390cf5
·
verified ·
1 Parent(s): 4c21ee6

Upload as_extract.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. as_extract.py +32 -33
as_extract.py CHANGED
@@ -50,39 +50,38 @@ for fi, f in enumerate(files):
50
  break
51
  with fs.open(f"datasets/{REPO}/{f}", "rb") as fh:
52
  pf = pq.ParquetFile(fh)
53
- lab = pf.read(columns=["video_id", "labels", "human_labels"])
54
- vids = lab.column("video_id").to_pylist()
55
- hls = lab.column("human_labels").to_pylist()
56
- lbs = lab.column("labels").to_pylist()
57
- match = []
58
- for i, hl in enumerate(hls):
59
- hl = set(hl or [])
60
- if n_expl < CAP_EXPL and hl & EXPL:
61
- match.append((i, "explosion"))
62
- elif n_veh < CAP_VEH and hl & VEH:
63
- match.append((i, "vehicle_mechanical"))
64
- if not match:
65
- continue
66
- print(f"[{fi}/{len(files)}] {f}: {len(match)} matches", flush=True)
67
- aud = pf.read(columns=["audio"])
68
- acol = aud.column("audio")
69
- if hasattr(acol, "combine_chunks"):
70
- acol = acol.combine_chunks()
71
- for i, cls in match:
72
- raw = acol[i].as_py()["bytes"]
73
- hl = hls[i] or []
74
- rows.append({
75
- "video_id": vids[i],
76
- "labels": lbs[i] or [],
77
- "human_labels": list(hl),
78
- "sound_class": cls,
79
- "non_drone_source": "audioset:" + ",".join(sorted(hl)),
80
- "audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"},
81
- })
82
- if cls == "explosion":
83
- n_expl += 1
84
- else:
85
- n_veh += 1
86
  if fi % 50 == 0:
87
  print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
88
 
 
50
  break
51
  with fs.open(f"datasets/{REPO}/{f}", "rb") as fh:
52
  pf = pq.ParquetFile(fh)
53
+ for rg in range(pf.num_row_groups):
54
+ lab = pf.read_row_group(rg, columns=["video_id", "human_labels", "labels"])
55
+ vids = lab.column("video_id").to_pylist()
56
+ hls = lab.column("human_labels").to_pylist()
57
+ lbs = lab.column("labels").to_pylist()
58
+ match = []
59
+ for i, hl in enumerate(hls):
60
+ hl = set(hl or [])
61
+ if n_expl < CAP_EXPL and hl & EXPL:
62
+ match.append((i, "explosion"))
63
+ elif n_veh < CAP_VEH and hl & VEH:
64
+ match.append((i, "vehicle_mechanical"))
65
+ if not match:
66
+ continue
67
+ full = pf.read_row_group(rg) # all columns; struct audio reads fine here
68
+ acol = full.column("audio").combine_chunks()
69
+ for i, cls in match:
70
+ raw = acol[i].as_py()["bytes"]
71
+ hl = hls[i] or []
72
+ rows.append({
73
+ "video_id": vids[i],
74
+ "labels": lbs[i] or [],
75
+ "human_labels": list(hl),
76
+ "sound_class": cls,
77
+ "non_drone_source": "audioset:" + ",".join(sorted(hl)),
78
+ "audio": {"bytes": encode_wav(raw), "path": f"{vids[i]}.wav"},
79
+ })
80
+ if cls == "explosion":
81
+ n_expl += 1
82
+ else:
83
+ n_veh += 1
84
+ print(f"[{fi}/{len(files)}] {f} rg{rg}: {len(match)} matches (expl={n_expl} veh={n_veh})", flush=True)
 
85
  if fi % 50 == 0:
86
  print(f"progress: shards={fi} expl={n_expl} veh={n_veh} rows={len(rows)}", flush=True)
87