File size: 1,386 Bytes
cbe2712
 
ad59923
cbe2712
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# /// script
# requires-python = ">=3.11"
# dependencies = ["datasets>=4.0", "huggingface_hub>=0.30", "pillow"]
# ///
"""Build Tonic/challenge5-train-subset: stratified 60k-row windows_5s subset of
Tonic/unified-drone-acoustic-cuas, always keeping every challenge5 row.

Run as an HF Job (cpu-basic):  uv run subset_build.py
"""
import os
from datasets import load_dataset, DatasetDict

SRC = "Tonic/unified-drone-acoustic-cuas"
DST = "Tonic/challenge5-train-subset"
TRAIN_N, VALID_N = 60_000, 4_000

ds = load_dataset(SRC, "windows_5s", token=os.environ["HF_TOKEN"])
out = {}
for split, n in (("train", TRAIN_N), ("validation", VALID_N)):
    d = ds[split]
    c5 = d.filter(lambda r: r["source"] == "challenge5", num_proc=8)
    rest = d.filter(lambda r: r["source"] != "challenge5", num_proc=8)
    rest = rest.shuffle(seed=42).select(range(min(n - len(c5), len(rest))))
    from datasets import concatenate_datasets
    out[split] = concatenate_datasets([c5, rest])
    print(f"{split}: {len(out[split])} rows (challenge5: {len(c5)})", flush=True)
# test: keep only challenge5 (held-out labels) -> tiny
c5t = ds["test"].filter(lambda r: r["source"] == "challenge5", num_proc=8)
out["test"] = c5t
print(f"test: {len(c5t)} rows (challenge5 only)", flush=True)

DatasetDict(out).push_to_hub(DST, config_name="windows_5s", token=os.environ["HF_TOKEN"])
print("PUSHED", DST, flush=True)