audioset: struct audio + incremental shard push in build_audioset_job.py
Browse files- build_audioset_job.py +5 -1
build_audioset_job.py
CHANGED
|
@@ -38,7 +38,11 @@ def load_locator(loc):
|
|
| 38 |
B.load_locator = load_locator
|
| 39 |
|
| 40 |
from datasets import load_dataset
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
print("staging rows:", len(ds), flush=True)
|
| 43 |
|
| 44 |
tasks = []
|
|
|
|
| 38 |
B.load_locator = load_locator
|
| 39 |
|
| 40 |
from datasets import load_dataset
|
| 41 |
+
_cfgs = sorted({p.split("/")[0] for p in api.list_repo_files("Tonic/raw-audioset-explosion", repo_type="dataset") if p.split("/")[0].startswith("clips")}) or ["clips"]
|
| 42 |
+
from datasets import concatenate_datasets as _cd, load_dataset as _ld
|
| 43 |
+
_dss = [_ld("Tonic/raw-audioset-explosion", c, token=os.environ["HF_TOKEN"])["train"] for c in _cfgs]
|
| 44 |
+
ds = _dss[0] if len(_dss) == 1 else _cd(_dss)
|
| 45 |
+
print("staging configs:", _cfgs, flush=True)
|
| 46 |
print("staging rows:", len(ds), flush=True)
|
| 47 |
|
| 48 |
tasks = []
|