Upload subset_build.py with huggingface_hub
Browse files- subset_build.py +33 -0
subset_build.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.11"
|
| 3 |
+
# dependencies = ["datasets>=4.0", "huggingface_hub>=0.30"]
|
| 4 |
+
# ///
|
| 5 |
+
"""Build Tonic/challenge5-train-subset: stratified 60k-row windows_5s subset of
|
| 6 |
+
Tonic/unified-drone-acoustic-cuas, always keeping every challenge5 row.
|
| 7 |
+
|
| 8 |
+
Run as an HF Job (cpu-basic): uv run subset_build.py
|
| 9 |
+
"""
|
| 10 |
+
import os
|
| 11 |
+
from datasets import load_dataset, DatasetDict
|
| 12 |
+
|
| 13 |
+
SRC = "Tonic/unified-drone-acoustic-cuas"
|
| 14 |
+
DST = "Tonic/challenge5-train-subset"
|
| 15 |
+
TRAIN_N, VALID_N = 60_000, 4_000
|
| 16 |
+
|
| 17 |
+
ds = load_dataset(SRC, "windows_5s", token=os.environ["HF_TOKEN"])
|
| 18 |
+
out = {}
|
| 19 |
+
for split, n in (("train", TRAIN_N), ("validation", VALID_N)):
|
| 20 |
+
d = ds[split]
|
| 21 |
+
c5 = d.filter(lambda r: r["source"] == "challenge5", num_proc=8)
|
| 22 |
+
rest = d.filter(lambda r: r["source"] != "challenge5", num_proc=8)
|
| 23 |
+
rest = rest.shuffle(seed=42).select(range(min(n - len(c5), len(rest))))
|
| 24 |
+
from datasets import concatenate_datasets
|
| 25 |
+
out[split] = concatenate_datasets([c5, rest])
|
| 26 |
+
print(f"{split}: {len(out[split])} rows (challenge5: {len(c5)})", flush=True)
|
| 27 |
+
# test: keep only challenge5 (held-out labels) -> tiny
|
| 28 |
+
c5t = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=8)
|
| 29 |
+
out["test"] = c5t
|
| 30 |
+
print(f"test: {len(c5t)} rows (challenge5 only)", flush=True)
|
| 31 |
+
|
| 32 |
+
DatasetDict(out).push_to_hub(DST, config_name="windows_5s", token=os.environ["HF_TOKEN"])
|
| 33 |
+
print("PUSHED", DST, flush=True)
|