Download code/check_leak100.py from Pricile/waxal2026-backup: direct link, hf CLI and curl.
- Browser
- Download file 3.57 kB
-
https://huggingface.co/Pricile/waxal2026-backup/resolve/main/code/check_leak100.py
- Command line
-
hf download hf://Pricile/waxal2026-backup/code/check_leak100.py
-
curl -L -o check_leak100.py https://huggingface.co/Pricile/waxal2026-backup/resolve/main/code/check_leak100.py
3.57 kB
| #!/usr/bin/env python3 | |
| """`KasuleTrevor/Lingala_100hrs` contient-il des transcriptions du SPLIT TEST de | |
| WAXAL ? Si oui, l'utiliser serait s'entrainer sur du test = disqualification. | |
| On a mesure qu'il recoupe WAXAL a 49 %. La question qui reste est : QUELLE | |
| partie de WAXAL ? train/validation (acceptable, c'est deja nos donnees) ou test | |
| (interdit) ? | |
| Methode : telecharger les transcriptions du split TEST lin de google/WaxalNLP | |
| (colonne texte seulement, pas l'audio) et mesurer le recouvrement avec un | |
| echantillon large de Lingala_100hrs (plusieurs parquets, pas un seul). | |
| """ | |
| import glob, json, os, re | |
| os.environ.setdefault("HF_HUB_DISABLE_XET", "1") | |
| os.environ.setdefault("HF_HOME", "/scratch/hf_home") | |
| import pyarrow.parquet as pq | |
| from huggingface_hub import HfApi, hf_hub_download, snapshot_download | |
| tok = open(os.path.expanduser("~/.cache/huggingface/token")).read().strip() | |
| api = HfApi(token=tok) | |
| def norm(s): | |
| return " ".join(re.sub(r"[^\w ]", " ", str(s).lower()).split()) | |
| def texts_from_parquets(paths, cands=("text", "sentence", "transcription", "transcript")): | |
| out = [] | |
| for p in paths: | |
| names = pq.ParquetFile(p).schema_arrow.names | |
| c = next((x for x in cands if x in names), None) | |
| if not c: | |
| continue | |
| out += [norm(v) for v in pq.read_table(p, columns=[c]).column(c).to_pylist() if v] | |
| return out | |
| print("=== 1) transcriptions du split TEST lin de WAXAL ===", flush=True) | |
| snapshot_download("google/WaxalNLP", repo_type="dataset", | |
| allow_patterns=["data/ASR/lin/lin-test-*.parquet"], | |
| local_dir="/scratch/waxtest", token=tok, max_workers=8) | |
| tf = sorted(glob.glob("/scratch/waxtest/data/ASR/lin/lin-test-*.parquet")) | |
| test_txt = set(texts_from_parquets(tf)) | |
| print(" %d parquets | %d phrases uniques dans le TEST lin" % (len(tf), len(test_txt)), flush=True) | |
| print("\n=== 2) transcriptions du TRAIN+VAL lin de WAXAL (reference de comparaison) ===", flush=True) | |
| tv = set() | |
| for p in ("waxal_lin_train", "waxal_lin_validation"): | |
| f = "/scratch/prep/manifests/%s.jsonl" % p | |
| if os.path.exists(f): | |
| for l in open(f, encoding="utf-8"): | |
| t = norm(json.loads(l).get("text", "")) | |
| if t: | |
| tv.add(t) | |
| print(" %d phrases uniques" % len(tv), flush=True) | |
| print(" (controle : test inter train+val = %d, doit etre ~0)" % len(test_txt & tv), flush=True) | |
| print("\n=== 3) echantillon LARGE de Lingala_100hrs ===", flush=True) | |
| fs = [s.rfilename for s in api.dataset_info("KasuleTrevor/Lingala_100hrs").siblings | |
| if s.rfilename.endswith(".parquet")] | |
| sel = fs[:8] | |
| paths = [hf_hub_download("KasuleTrevor/Lingala_100hrs", f, repo_type="dataset", | |
| token=tok, local_dir="/scratch/dl100") for f in sel] | |
| k100 = texts_from_parquets(paths) | |
| print(" %d parquets lus | %d lignes" % (len(paths), len(k100)), flush=True) | |
| inter_test = sum(1 for t in k100 if t in test_txt) | |
| inter_tv = sum(1 for t in k100 if t in tv) | |
| print("\n=== VERDICT ===", flush=True) | |
| print(" recouvrement avec TRAIN+VAL WAXAL : %d/%d (%.1f %%)" % (inter_tv, len(k100), 100 * inter_tv / max(len(k100), 1)), flush=True) | |
| print(" recouvrement avec TEST WAXAL : %d/%d (%.1f %%) %s" | |
| % (inter_test, len(k100), 100 * inter_test / max(len(k100), 1), | |
| "<<< CONTIENT DU TEST -- INTERDIT" if inter_test > 0 else "<<< aucune trace de test"), flush=True) | |
| if inter_test: | |
| ex = [t for t in k100 if t in test_txt][:3] | |
| for e in ex: | |
| print(" ex de fuite : %s" % e[:110], flush=True) | |
| print("LEAK_CHECK_DONE", flush=True) | |