Download eval/run_baselines.py from sapinsapin/wikalid: direct link, hf CLI and curl.
- Browser
- Download file 2.46 kB
-
https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/run_baselines.py
- Command line
-
hf download hf://sapinsapin/wikalid/eval/run_baselines.py
-
curl -L -o run_baselines.py https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/run_baselines.py
2.46 kB
| """Run fastText baselines (GlotLID v3, OpenLID-v3, fastText lid.176) live on every available eval set. | |
| usage: <python with fasttext> run_baselines.py [outdir=eval/out/preds] (models from $LID_MODELS_* or the box defaults) | |
| Writes <outdir>/<tool>__<set>.json = {"tool", "meta", "pred": {id: [label, prob, top3]}} (same format as the research repo).""" | |
| import json, os, sys, time | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from sets_v1 import available, rows | |
| import fasttext | |
| fasttext.FastText.eprint = lambda *a, **k: None | |
| OUT = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "out", "preds") | |
| MODELS = {"glotlid": os.environ.get("LID_MODELS_GLOTLID", "/workspace/pretraining-box/runtime/lm_resource/glotlid_v3.bin"), | |
| "openlid3": os.environ.get("LID_MODELS_OPENLID3", "/workspace/lid-work/models/openlid3/openlid-v3.bin"), | |
| "lid176": os.environ.get("LID_MODELS_LID176", "/workspace/pretraining-box/runtime/lm_resource/lid.176.bin")} | |
| ISO = {"tl": "tgl", "en": "eng", "es": "spa", "id": "ind", "ms": "msa", "fil": "tgl", "zsm": "msa", "zlm": "msa", "may": "msa"} | |
| KEEP = {"tgl", "ceb", "hil", "war", "ilo", "bcl", "pam", "pag", "cbk", "eng", "spa", "ind", "msa", "krj", "tsg"} | |
| def canon(l): # fastText label -> ISO 639-3 (probabilities of equal codes are summed) | |
| b = l.replace("__label__", "").split("_", 1)[0]; return ISO.get(b, b) | |
| def to_cls(c): # ISO code -> OurLID class (after picking the top code, as in the research repo) | |
| return c if c in KEEP else "nol" if c in ("zxx", "und") else "oth" | |
| os.makedirs(OUT, exist_ok=True) | |
| for tool, path in MODELS.items(): | |
| m = fasttext.load_model(path) | |
| for n in available(): | |
| R = rows(n, "all"); t0 = time.time(); pred = {} | |
| labs, probs = m.predict([" ".join(r["text"].split())[:2000] or " " for r in R], k=3) | |
| for r, ls, ps in zip(R, labs, probs): | |
| d = {} | |
| for l, p in zip(ls, ps): c = canon(l); d[c] = d.get(c, 0.0) + float(p) | |
| top = sorted(d.items(), key=lambda x: -x[1]) | |
| pred[r["id"]] = [to_cls(top[0][0]), top[0][1], {k: round(v, 4) for k, v in top}] | |
| json.dump({"tool": tool, "meta": {"items_per_s": len(R) / max(1e-9, time.time() - t0), "model": os.path.basename(path)}, "pred": pred}, | |
| open(f"{OUT}/{tool}__{n}.json", "w")) | |
| print(tool, n, len(R), flush=True) | |
| del m | |