as_extract_part: env-based args
Browse files- as_extract_part.py +7 -11
as_extract_part.py
CHANGED
|
@@ -4,15 +4,12 @@
|
|
| 4 |
# ///
|
| 5 |
"""Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i
|
| 6 |
and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel."""
|
| 7 |
-
import
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
ap.add_argument("--config", default=None)
|
| 14 |
-
args = ap.parse_args()
|
| 15 |
-
CONFIG = args.config or f"clips_p{args.part}"
|
| 16 |
import io, os
|
| 17 |
import numpy as np
|
| 18 |
import soundfile as sf
|
|
@@ -24,7 +21,6 @@ EXPL = {"Explosion", "Gunshot, gunfire", "Fireworks", "Burst, pop", "Eruption",
|
|
| 24 |
VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
|
| 25 |
"Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle",
|
| 26 |
"Car passing by", "Race car, auto racing", "Airplane, aeroplane"}
|
| 27 |
-
CAP_EXPL, CAP_VEH = args.cap_expl, args.cap_veh
|
| 28 |
TARGET_SR = 16000
|
| 29 |
|
| 30 |
api = HfApi(token=os.environ["HF_TOKEN"])
|
|
@@ -35,7 +31,7 @@ SPLITS = ("bal_train", "eval", "unbal_train")
|
|
| 35 |
files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset")
|
| 36 |
if f.startswith("data/") and f.endswith(".parquet")
|
| 37 |
and f.split("/")[1] in SPLITS]
|
| 38 |
-
files = [f for i, f in enumerate(files_all) if i %
|
| 39 |
print(f"{len(files)} parquet shards", flush=True)
|
| 40 |
|
| 41 |
rows = []
|
|
|
|
| 4 |
# ///
|
| 5 |
"""Part-split version of as_extract.py: --part i --nparts N scans files where index % N == i
|
| 6 |
and pushes to config clips_p{i} of Tonic/raw-audioset-explosion. Run N jobs in parallel."""
|
| 7 |
+
import os as _os
|
| 8 |
+
PART = int(_os.environ.get("PART", "0"))
|
| 9 |
+
NPARTS = int(_os.environ.get("NPARTS", "4"))
|
| 10 |
+
CAP_EXPL = int(_os.environ.get("CAP_EXPL", "1500"))
|
| 11 |
+
CAP_VEH = int(_os.environ.get("CAP_VEH", "1500"))
|
| 12 |
+
CONFIG = _os.environ.get("CONFIG") or f"clips_p{PART}"
|
|
|
|
|
|
|
|
|
|
| 13 |
import io, os
|
| 14 |
import numpy as np
|
| 15 |
import soundfile as sf
|
|
|
|
| 21 |
VEH = {"Vehicle", "Engine", "Motor vehicle (on road)", "Car", "Truck", "Motorcycle",
|
| 22 |
"Aircraft", "Helicopter", "Train", "Motorboat", "Ship", "Emergency vehicle",
|
| 23 |
"Car passing by", "Race car, auto racing", "Airplane, aeroplane"}
|
|
|
|
| 24 |
TARGET_SR = 16000
|
| 25 |
|
| 26 |
api = HfApi(token=os.environ["HF_TOKEN"])
|
|
|
|
| 31 |
files_all = [f for f in api.list_repo_files(REPO, repo_type="dataset")
|
| 32 |
if f.startswith("data/") and f.endswith(".parquet")
|
| 33 |
and f.split("/")[1] in SPLITS]
|
| 34 |
+
files = [f for i, f in enumerate(files_all) if i % NPARTS == PART]
|
| 35 |
print(f"{len(files)} parquet shards", flush=True)
|
| 36 |
|
| 37 |
rows = []
|