Download scripts/build_devhard.py from Pricile/waxal2026-backup: direct link, hf CLI and curl.
- Browser
- Download file 1.47 kB
-
https://huggingface.co/Pricile/waxal2026-backup/resolve/main/scripts/build_devhard.py
- Command line
-
hf download hf://Pricile/waxal2026-backup/scripts/build_devhard.py
-
curl -L -o build_devhard.py https://huggingface.co/Pricile/waxal2026-backup/resolve/main/scripts/build_devhard.py
1.47 kB
| import json, random, collections, os | |
| random.seed(1234) | |
| MANI="/scratch/prep/manifests" | |
| OUT="/root/devhard"; os.makedirs(OUT, exist_ok=True) | |
| def load(p): return [json.loads(l) for l in open(p,encoding="utf-8")] | |
| dev_all=[]; trainmin={} | |
| for L in ["lin","lug","sna"]: | |
| rows=load(f"{MANI}/waxal_{L}_train.jsonl") | |
| by=collections.defaultdict(list) | |
| for r in rows: by[r.get("speaker")].append(r) | |
| # candidate holdout speakers: middle band (avoid gutting training), 8..70 utts | |
| cand=[s for s,v in by.items() if 8<=len(v)<=70] | |
| random.shuffle(cand) | |
| target=min(400, int(0.05*len(rows))) | |
| hold=set(); n=0 | |
| for s in cand: | |
| if n>=target: break | |
| hold.add(s); n+=len(by[s]) | |
| dev=[r for r in rows if r.get("speaker") in hold] | |
| tmin=[r for r in rows if r.get("speaker") not in hold] | |
| for r in dev: r["lang"]=L | |
| dev_all+=dev; trainmin[L]=tmin | |
| json.dump if False else None | |
| with open(f"{OUT}/devhard_{L}.jsonl","w",encoding="utf-8") as f: | |
| for r in dev: f.write(json.dumps(r,ensure_ascii=False)+"\n") | |
| with open(f"{OUT}/train_{L}_min.jsonl","w",encoding="utf-8") as f: | |
| for r in tmin: f.write(json.dumps(r,ensure_ascii=False)+"\n") | |
| print(f"{L}: holdout_spk={len(hold)} devhard_utts={len(dev)} train_min_utts={len(tmin)} (was {len(rows)})") | |
| with open(f"{OUT}/devhard_all.jsonl","w",encoding="utf-8") as f: | |
| for r in dev_all: f.write(json.dumps(r,ensure_ascii=False)+"\n") | |
| print("TOTAL devhard:",len(dev_all)) | |