"""Run fastText baselines (GlotLID v3, OpenLID-v3, fastText lid.176) live on every available eval set. usage: run_baselines.py [outdir=eval/out/preds] (models from $LID_MODELS_* or the box defaults) Writes /__.json = {"tool", "meta", "pred": {id: [label, prob, top3]}} (same format as the research repo).""" import json, os, sys, time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from sets_v1 import available, rows import fasttext fasttext.FastText.eprint = lambda *a, **k: None OUT = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "out", "preds") MODELS = {"glotlid": os.environ.get("LID_MODELS_GLOTLID", "/workspace/pretraining-box/runtime/lm_resource/glotlid_v3.bin"), "openlid3": os.environ.get("LID_MODELS_OPENLID3", "/workspace/lid-work/models/openlid3/openlid-v3.bin"), "lid176": os.environ.get("LID_MODELS_LID176", "/workspace/pretraining-box/runtime/lm_resource/lid.176.bin")} ISO = {"tl": "tgl", "en": "eng", "es": "spa", "id": "ind", "ms": "msa", "fil": "tgl", "zsm": "msa", "zlm": "msa", "may": "msa"} KEEP = {"tgl", "ceb", "hil", "war", "ilo", "bcl", "pam", "pag", "cbk", "eng", "spa", "ind", "msa", "krj", "tsg"} def canon(l): # fastText label -> ISO 639-3 (probabilities of equal codes are summed) b = l.replace("__label__", "").split("_", 1)[0]; return ISO.get(b, b) def to_cls(c): # ISO code -> OurLID class (after picking the top code, as in the research repo) return c if c in KEEP else "nol" if c in ("zxx", "und") else "oth" os.makedirs(OUT, exist_ok=True) for tool, path in MODELS.items(): m = fasttext.load_model(path) for n in available(): R = rows(n, "all"); t0 = time.time(); pred = {} labs, probs = m.predict([" ".join(r["text"].split())[:2000] or " " for r in R], k=3) for r, ls, ps in zip(R, labs, probs): d = {} for l, p in zip(ls, ps): c = canon(l); d[c] = d.get(c, 0.0) + float(p) top = sorted(d.items(), key=lambda x: -x[1]) pred[r["id"]] = [to_cls(top[0][0]), top[0][1], {k: round(v, 4) for k, v in top}] json.dump({"tool": tool, "meta": {"items_per_s": len(R) / max(1e-9, time.time() - t0), "model": os.path.basename(path)}, "pred": pred}, open(f"{OUT}/{tool}__{n}.json", "w")) print(tool, n, len(R), flush=True) del m