File size: 9,830 Bytes
8f78c5a
863c520
 
 
8f78c5a
863c520
 
 
 
 
 
 
 
 
 
 
 
8f78c5a
863c520
 
 
 
 
 
8f78c5a
 
863c520
 
 
 
 
 
 
 
 
 
 
 
8f78c5a
863c520
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f78c5a
863c520
 
 
 
8f78c5a
863c520
 
 
 
 
 
 
 
 
 
 
8f78c5a
863c520
 
 
 
 
 
 
 
 
 
 
 
 
 
8f78c5a
863c520
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""Reproducible evaluation of the installed `wikalid` package (1.0.1) against GlotLID v3, OpenLID-v3 and fastText lid.176.

usage:  python eval/evaluate.py [--workers 4] [--skip-predict]
needs:  eval/out/preds/{glotlid,openlid3,lid176}__<set>.json from eval/run_baselines.py (live fastText runs)
writes: eval/out/preds/wikalid-1.0.1__<set>.json, eval/RESULTS_v1.0.1.json, eval/RESULTS_v1.0.1.md

Protocol (unchanged from the research loop): every set is split 50/50 by md5 of a group key into dev (all tuning
decisions were taken there) and sealed (never used for a decision). Headline = sealed halves. "corrected" gold fixes known
label noise in wiki/hard/pld/cbk by consensus of GlotLID/OpenLID/lid176 (same rule as before; caveat: it shares detectors
with the baselines it is compared against, which favours them, not us). Private sets are evaluated locally and never
published; only the public sets (wiki, hard, tato) can be re-run by a third party."""
import argparse, json, os, re, sys, time
for _k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"): os.environ.setdefault(_k, "1")   # 1 BLAS thread per worker
from multiprocessing import get_context
import numpy as np
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE)
from sets_v1 import CORE, TRUST, PUBLIC, available, rows, ok, part_of
PRED = os.path.join(HERE, "out", "preds"); TAG = "wikalid-1.0.1"; BASE = ["glotlid", "openlid3", "lid176"]
CBKM = set(json.load(open(os.path.join(HERE, "data", "cbk_markers.json"))))
_P = _G = None


def _init():
    global _P, _G
    import wikalid as ourlid
    from wikalid.gate import Gate
    _P = ourlid.load(rules=True); _G = Gate(ourlid.model_dir())


def _work(batch):
    out = {}
    for i, t in batch:
        a = _P.analyze(t); g = _G.decide_one(t, a)
        out[i] = [a["label"], a["conf"], a["mixture"], a["cs_level"], a["cs_share"], a["abstain"], a["label_set"], g["gate"], g["gate_p_correct"]]
    return out


def predict(workers, resume=False):
    import wikalid as ourlid
    os.makedirs(PRED, exist_ok=True); speed = {}
    for n in available():
        if resume and os.path.exists(f"{PRED}/{TAG}__{n}.json"): continue
        R = rows(n, "all"); items = [(r["id"], r["text"]) for r in R]; B = [items[i:i + 200] for i in range(0, len(items), 200)]
        t0 = time.time(); pred = {}
        with get_context("fork").Pool(workers, initializer=_init) as pool:
            for d in pool.imap_unordered(_work, B): pred.update(d)
        el = time.time() - t0; speed[n] = len(R) / el
        json.dump({"tool": TAG, "meta": {"version": ourlid.__version__, "items_per_s_total": speed[n], "workers": workers}, "pred": pred},
                  open(f"{PRED}/{TAG}__{n}.json", "w"))
        print(TAG, n, len(R), f"{speed[n]:.0f} it/s ({workers} workers)", flush=True)
    return speed


def load(t, n):
    p = f"{PRED}/{t}__{n}.json"
    return json.load(open(p))["pred"] if os.path.exists(p) else None


def corrected(n, R):
    ex = [load(t, n) for t in BASE]
    if any(e is None for e in ex): return [r["label"] for r in R]
    out = []
    for r in R:
        l, i = r["label"], r["id"]
        if l != "eng" and n in ("wiki", "hard", "pld") and sum(1 for d in ex if d[i][0] == "eng" and d[i][1] >= 0.6) >= 2: l = "eng"
        elif l == "cbk" and sum(1 for d in ex if d[i][0] == "spa" and d[i][1] >= 0.6) >= 2 and not (set(re.findall(r"[a-záéíóúñ']+", r["text"].lower())) & CBKM): l = "spa"
        out.append(l)
    return out


def score(n, R, P, cor):
    raw = np.array([ok(n, r, P[r["id"]][0]) for r in R], float)
    corr = np.array([(P[r["id"]][0] == g) or (g == r["label"] and ok(n, r, P[r["id"]][0])) for r, g in zip(R, cor)], float)
    y = np.array(cor); p = np.array([P[r["id"]][0] for r in R]); f = []
    for c in sorted(set(y)):
        if (y == c).sum() < 15: continue
        tp = ((y == c) & (p == c)).sum(); fp = ((y != c) & (p == c)).sum(); fn = ((y == c) & (p != c)).sum(); f.append(2 * tp / max(1, 2 * tp + fp + fn))
    return raw, corr, float(np.mean(f)) if f else float("nan")


def wilson(k, n, z=1.96):
    if n == 0: return (0.0, 0.0)
    p = k / n; d = 1 + z * z / n; c = (p + z * z / 2 / n) / d; h = z * np.sqrt(p * (1 - p) / n + z * z / 4 / n / n) / d; return (c - h, c + h)


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--workers", type=int, default=4); ap.add_argument("--skip-predict", action="store_true"); ap.add_argument("--resume", action="store_true"); a = ap.parse_args()
    speed = predict(a.workers, a.resume) if not a.skip_predict else {}
    tools = [TAG] + BASE; res = {"version": TAG, "speed_items_per_s": speed, "core": {}, "trust": {}, "paired_vs_glotlid": {}, "gate_sealed": {}, "consistency_vs_research_c7r": {}}
    rng = np.random.default_rng(0)
    for part in ("sealed", "dev", "all"):
        res["core"][part] = {}
        for n in CORE:
            if n not in available(): continue
            R = rows(n, part); cor = corrected(n, R); res["core"][part][n] = {"n": len(R), "public": n in PUBLIC}
            for t in tools:
                P = load(t, n)
                if P is None: continue
                raw, corr, f1 = score(n, R, P, cor); res["core"][part][n][t] = {"acc_raw": raw.mean(), "acc_corrected": corr.mean(), "macro_f1_corrected": f1}
                if part == "sealed" and t == TAG:
                    G = load("glotlid", n); _, cg, _ = score(n, R, G, cor); d = corr - cg; idx = rng.integers(0, len(R), (2000, len(R))); bs = d[idx].mean(1)
                    res["paired_vs_glotlid"][n] = {"delta": float(d.mean()), "ci95": [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]}
                    acc = [P[r["id"]][7] in ("accept", "accept_eng", "accept_ph_other") for r in R]
                    if any(acc): res["gate_sealed"][n] = {"coverage": float(np.mean(acc)), "precision_accepted_corrected": float(corr[np.array(acc)].mean()), "n": len(R)}
    for n in TRUST:
        if n not in available(): continue
        R = rows(n, "all"); res["trust"][n] = {"n": len(R)}
        for t in tools:
            P = load(t, n)
            if P is None: continue
            hit = np.array([ok(n, r, P[r["id"]][0]) for r in R]); lo, hi = wilson(hit.sum(), len(R))
            per = {}
            for key in sorted({r["label"] for r in R} | {r.get("cat") for r in R if r.get("cat")}):
                ix = [i for i, r in enumerate(R) if r["label"] == key or r.get("cat") == key]
                k = int(hit[ix].sum()); l2, h2 = wilson(k, len(ix)); per[key] = {"acc": k / len(ix), "n": len(ix), "ci95": [l2, h2]}
            res["trust"][n][t] = {"acc": float(hit.mean()), "ci95": [lo, hi], "by": per}
    for n in available():
        p = f"/workspace/ourlid/eval/preds/c7r__{n}.json"
        A, B = load(TAG, n), (json.load(open(p))["pred"] if os.path.exists(p) else None)
        if A and B: res["consistency_vs_research_c7r"][n] = {"n": len(A), "same_label": sum(A[i][0] == B[i][0] for i in A if i in B)}
    json.dump(res, open(os.path.join(HERE, "RESULTS_v1.0.1.json"), "w"), indent=1, default=float)
    write_md(res, tools)


def write_md(res, tools):
    L = ["# WikaLID 1.0.1 evaluation (generated by eval/evaluate.py; do not edit by hand)", ""]
    S = res["core"]["sealed"]; sets = list(S)
    L += ["## Sealed halves (never used for any decision): accuracy raw / corrected gold (macro-F1 corrected)", "",
          "| system | " + " | ".join(f"{n}{'*' if S[n]['public'] else ''} (n={S[n]['n']})" for n in sets) + " | mean corrected |", "|---|" + "---|" * (len(sets) + 1)]
    for t in tools:
        cells, ms = [], []
        for n in sets:
            c = S[n].get(t)
            if not c: cells.append("n/a"); continue
            cells.append(f"{c['acc_raw']:.3f} / {c['acc_corrected']:.3f} ({c['macro_f1_corrected']:.2f})"); ms.append(c["acc_corrected"])
        L.append(f"| {t} | " + " | ".join(cells) + f" | **{np.mean(ms):.3f}** |")
    L += ["", "\\* public set (ships in eval/data, anyone can re-run). Others are private (news gold, crawl realism, PLD read speech, BasahaCorpus stories).", "",
          "## WikaLID 1.0.1 minus GlotLID v3, sealed, corrected gold (paired bootstrap, 2,000 resamples)", "", "| set | delta | 95% CI |", "|---|---|---|"]
    for n, d in res["paired_vs_glotlid"].items(): L.append(f"| {n} | {d['delta']:+.3f} | [{d['ci95'][0]:+.3f}, {d['ci95'][1]:+.3f}] |")
    L += ["", "## Gate v2 on sealed halves (accept = accept / accept_eng / accept_ph_other)", "", "| set | coverage | precision of accepted (corrected gold) |", "|---|---|---|"]
    for n, d in res["gate_sealed"].items(): L.append(f"| {n} | {d['coverage']:.3f} | {d['precision_accepted_corrected']:.3f} |")
    L += ["", "## Human-labelled external trust sets (all rows; accuracy with 95% Wilson interval)", ""]
    for n, d in res["trust"].items():
        L += [f"### {n} (n={d['n']})", "", "| system | overall | " + " | ".join(sorted(d[TAG]["by"])) + " |", "|---|---|" + "---|" * len(d[TAG]["by"])]
        for t in tools:
            if t not in d: continue
            L.append(f"| {t} | {d[t]['acc']:.3f} [{d[t]['ci95'][0]:.2f}-{d[t]['ci95'][1]:.2f}] | " + " | ".join(f"{d[t]['by'][k]['acc']:.3f}" for k in sorted(d[TAG]["by"])) + " |")
        L.append("")
    L += ["## Consistency with the research model that labels Himay (c7 + rules, cached predictions)", "", "| set | n | same label |", "|---|---|---|"]
    for n, d in res["consistency_vs_research_c7r"].items(): L.append(f"| {n} | {d['n']} | {d['same_label']} |")
    if res["speed_items_per_s"]:
        L += ["", "## Throughput during this run (items/s, all workers, shared box)", "", " | ".join(f"{n}: {v:.0f}" for n, v in res["speed_items_per_s"].items())]
    open(os.path.join(HERE, "RESULTS_v1.0.1.md"), "w").write("\n".join(L) + "\n")


if __name__ == "__main__":
    main()