File size: 2,461 Bytes
863c520
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
"""Run fastText baselines (GlotLID v3, OpenLID-v3, fastText lid.176) live on every available eval set.
usage: <python with fasttext> run_baselines.py [outdir=eval/out/preds]      (models from $LID_MODELS_* or the box defaults)
Writes <outdir>/<tool>__<set>.json = {"tool", "meta", "pred": {id: [label, prob, top3]}} (same format as the research repo)."""
import json, os, sys, time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from sets_v1 import available, rows
import fasttext
fasttext.FastText.eprint = lambda *a, **k: None
OUT = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "out", "preds")
MODELS = {"glotlid": os.environ.get("LID_MODELS_GLOTLID", "/workspace/pretraining-box/runtime/lm_resource/glotlid_v3.bin"),
          "openlid3": os.environ.get("LID_MODELS_OPENLID3", "/workspace/lid-work/models/openlid3/openlid-v3.bin"),
          "lid176": os.environ.get("LID_MODELS_LID176", "/workspace/pretraining-box/runtime/lm_resource/lid.176.bin")}
ISO = {"tl": "tgl", "en": "eng", "es": "spa", "id": "ind", "ms": "msa", "fil": "tgl", "zsm": "msa", "zlm": "msa", "may": "msa"}
KEEP = {"tgl", "ceb", "hil", "war", "ilo", "bcl", "pam", "pag", "cbk", "eng", "spa", "ind", "msa", "krj", "tsg"}
def canon(l):                         # fastText label -> ISO 639-3 (probabilities of equal codes are summed)
    b = l.replace("__label__", "").split("_", 1)[0]; return ISO.get(b, b)
def to_cls(c):                        # ISO code -> OurLID class (after picking the top code, as in the research repo)
    return c if c in KEEP else "nol" if c in ("zxx", "und") else "oth"
os.makedirs(OUT, exist_ok=True)
for tool, path in MODELS.items():
    m = fasttext.load_model(path)
    for n in available():
        R = rows(n, "all"); t0 = time.time(); pred = {}
        labs, probs = m.predict([" ".join(r["text"].split())[:2000] or " " for r in R], k=3)
        for r, ls, ps in zip(R, labs, probs):
            d = {}
            for l, p in zip(ls, ps): c = canon(l); d[c] = d.get(c, 0.0) + float(p)
            top = sorted(d.items(), key=lambda x: -x[1])
            pred[r["id"]] = [to_cls(top[0][0]), top[0][1], {k: round(v, 4) for k, v in top}]
        json.dump({"tool": tool, "meta": {"items_per_s": len(R) / max(1e-9, time.time() - t0), "model": os.path.basename(path)}, "pred": pred},
                  open(f"{OUT}/{tool}__{n}.json", "w"))
        print(tool, n, len(R), flush=True)
    del m