Tonic commited on
Commit
059c8b8
·
verified ·
1 Parent(s): b9e4311

audioset: struct audio + incremental shard push in build_audioset_job.py

Browse files
Files changed (1) hide show
  1. build_audioset_job.py +5 -1
build_audioset_job.py CHANGED
@@ -38,7 +38,11 @@ def load_locator(loc):
38
  B.load_locator = load_locator
39
 
40
  from datasets import load_dataset
41
- ds = load_dataset("Tonic/raw-audioset-explosion", "clips", token=os.environ["HF_TOKEN"])["train"]
 
 
 
 
42
  print("staging rows:", len(ds), flush=True)
43
 
44
  tasks = []
 
38
  B.load_locator = load_locator
39
 
40
  from datasets import load_dataset
41
+ _cfgs = sorted({p.split("/")[0] for p in api.list_repo_files("Tonic/raw-audioset-explosion", repo_type="dataset") if p.split("/")[0].startswith("clips")}) or ["clips"]
42
+ from datasets import concatenate_datasets as _cd, load_dataset as _ld
43
+ _dss = [_ld("Tonic/raw-audioset-explosion", c, token=os.environ["HF_TOKEN"])["train"] for c in _cfgs]
44
+ ds = _dss[0] if len(_dss) == 1 else _cd(_dss)
45
+ print("staging configs:", _cfgs, flush=True)
46
  print("staging rows:", len(ds), flush=True)
47
 
48
  tasks = []