Tonic commited on
Commit
53a638d
·
verified ·
1 Parent(s): c031767

as_extract_part: env-based args

Browse files
Files changed (1) hide show
  1. as_extract_part.py +7 -11
as_extract_part.py CHANGED
@@ -4,15 +4,12 @@
4
  # ///
5
  """Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i
6
  and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel."""
7
- import argparse
8
- ap = argparse.ArgumentParser()
9
- ap.add_argument("--part", type=int, required=True)
10
- ap.add_argument("--nparts", type=int, default=4)
11
- ap.add_argument("--cap-expl", type=int, default=1500)
12
- ap.add_argument("--cap-veh", type=int, default=1500)
13
- ap.add_argument("--config", default=None)
14
- args = ap.parse_args()
15
- CONFIG = args.config or f"clips_p{args.part}"
16
  import io, os
17
  import numpy as np
18
  import soundfile as sf
@@ -24,7 +21,6 @@ EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption",
24
  VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
25
  "Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle",
26
  "Car passing by", "Race car, auto racing", "Airplane, aeroplane"}
27
- CAP_EXPL, CAP_VEH = args.cap_expl, args.cap_veh
28
  TARGET_SR = 16000
29
 
30
  api = HfApi(token=os.environ["HF_TOKEN"])
@@ -35,7 +31,7 @@ SPLITS = ("bal_train", "eval", "unbal_train")
35
  files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset")
36
  if f.startswith("data/") and f.endswith(".parquet")
37
  and f.split("/")[1] in SPLITS]
38
- files = [f for i, f in enumerate(files_all) if i % args.nparts == args.part]
39
  print(f"{len(files)} parquet shards", flush=True)
40
 
41
  rows = []
 
4
  # ///
5
  """Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i
6
  and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel."""
7
+ import os as _os
8
+ PART = int(_os.environ.get("PART", "0"))
9
+ NPARTS = int(_os.environ.get("NPARTS", "4"))
10
+ CAP_EXPL = int(_os.environ.get("CAP_EXPL", "1500"))
11
+ CAP_VEH = int(_os.environ.get("CAP_VEH", "1500"))
12
+ CONFIG = _os.environ.get("CONFIG") or f"clips_p{PART}"
 
 
 
13
  import io, os
14
  import numpy as np
15
  import soundfile as sf
 
21
  VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
22
  "Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle",
23
  "Car passing by", "Race car, auto racing", "Airplane, aeroplane"}
 
24
  TARGET_SR = 16000
25
 
26
  api = HfApi(token=os.environ["HF_TOKEN"])
 
31
  files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset")
32
  if f.startswith("data/") and f.endswith(".parquet")
33
  and f.split("/")[1] in SPLITS]
34
+ files = [f for i, f in enumerate(files_all) if i % NPARTS == PART]
35
  print(f"{len(files)} parquet shards", flush=True)
36
 
37
  rows = []