Download code/apply_fill_empty.py from Pricile/waxal2026-backup: direct link, hf CLI and curl.
- Browser
- Download file 1.44 kB
-
https://huggingface.co/Pricile/waxal2026-backup/resolve/main/code/apply_fill_empty.py
- Command line
-
hf download hf://Pricile/waxal2026-backup/code/apply_fill_empty.py
-
curl -L -o apply_fill_empty.py https://huggingface.co/Pricile/waxal2026-backup/resolve/main/code/apply_fill_empty.py
1.44 kB
| #!/usr/bin/env python3 | |
| """Applique le fill-empty (leophill) a un CSV: les clips a Target <=2 mots deviennent | |
| le 4-gram le plus frequent du train de leur langue. Gain valide +0.0025 sur la val.""" | |
| import csv | |
| import json | |
| import sys | |
| from collections import Counter | |
| def norm(t): | |
| return " ".join(str(t).replace("|", " ").split()) | |
| def top_gram(lang): | |
| rows = [json.loads(l) for l in open(f"/scratch/prep/manifests/waxal_{lang}_train.jsonl", encoding="utf-8")] | |
| texts = [norm(r["text"]).lower() for r in rows if r["text"].strip()] | |
| grams = Counter() | |
| for t in texts: | |
| w = t.split() | |
| for i in range(len(w) - 3): | |
| grams[" ".join(w[i:i+4])] += 1 | |
| return grams.most_common(1)[0][0] if grams else "a" | |
| def main(): | |
| inp, out = sys.argv[1], sys.argv[2] | |
| priors = {l: top_gram(l) for l in ("lin", "lug", "sna")} | |
| print("priors:", priors) | |
| rows = list(csv.DictReader(open(inp, encoding="utf-8"))) | |
| n = 0 | |
| for r in rows: | |
| lang = r["ID"].split("_")[0] | |
| if len(norm(r["Target"]).split()) <= 2 and lang in priors: | |
| r["Target"] = priors[lang] | |
| n += 1 | |
| with open(out, "w", newline="", encoding="utf-8") as f: | |
| w = csv.writer(f) | |
| w.writerow(["ID", "Target"]) | |
| for r in rows: | |
| w.writerow([r["ID"], r["Target"] or "a"]) | |
| print(f"FILL_APPLIED {out}: {n} clips remplaces sur {len(rows)}") | |
| if __name__ == "__main__": | |
| main() | |