Tonic commited on
Commit
cbe2712
·
verified ·
1 Parent(s): 23c5952

Upload subset_build.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. subset_build.py +33 -0
subset_build.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = ["datasets>=4.0", "huggingface_hub>=0.30"]
4
+ # ///
5
+ """Build Tonic/challenge5-train-subset: stratified 60k-row windows_5s subset of
6
+ Tonic/unified-drone-acoustic-cuas, always keeping every challenge5 row.
7
+
8
+ Run as an HF Job (cpu-basic): uv run subset_build.py
9
+ """
10
+ import os
11
+ from datasets import load_dataset, DatasetDict
12
+
13
+ SRC = "Tonic/unified-drone-acoustic-cuas"
14
+ DST = "Tonic/challenge5-train-subset"
15
+ TRAIN_N, VALID_N = 60_000, 4_000
16
+
17
+ ds = load_dataset(SRC, "windows_5s", token=os.environ["HF_TOKEN"])
18
+ out = {}
19
+ for split, n in (("train", TRAIN_N), ("validation", VALID_N)):
20
+ d = ds[split]
21
+ c5 = d.filter(lambda r: r["source"] == "challenge5", num_proc=8)
22
+ rest = d.filter(lambda r: r["source"] != "challenge5", num_proc=8)
23
+ rest = rest.shuffle(seed=42).select(range(min(n - len(c5), len(rest))))
24
+ from datasets import concatenate_datasets
25
+ out[split] = concatenate_datasets([c5, rest])
26
+ print(f"{split}: {len(out[split])} rows (challenge5: {len(c5)})", flush=True)
27
+ # test: keep only challenge5 (held-out labels) -> tiny
28
+ c5t = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=8)
29
+ out["test"] = c5t
30
+ print(f"test: {len(c5t)} rows (challenge5 only)", flush=True)
31
+
32
+ DatasetDict(out).push_to_hub(DST, config_name="windows_5s", token=os.environ["HF_TOKEN"])
33
+ print("PUSHED", DST, flush=True)