File size: 9,830 Bytes
8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 8f78c5a 863c520 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | """Reproducible evaluation of the installed `wikalid` package (1.0.1) against GlotLID v3, OpenLID-v3 and fastText lid.176.
usage: python eval/evaluate.py [--workers 4] [--skip-predict]
needs: eval/out/preds/{glotlid,openlid3,lid176}__<set>.json from eval/run_baselines.py (live fastText runs)
writes: eval/out/preds/wikalid-1.0.1__<set>.json, eval/RESULTS_v1.0.1.json, eval/RESULTS_v1.0.1.md
Protocol (unchanged from the research loop): every set is split 50/50 by md5 of a group key into dev (all tuning
decisions were taken there) and sealed (never used for a decision). Headline = sealed halves. "corrected" gold fixes known
label noise in wiki/hard/pld/cbk by consensus of GlotLID/OpenLID/lid176 (same rule as before; caveat: it shares detectors
with the baselines it is compared against, which favours them, not us). Private sets are evaluated locally and never
published; only the public sets (wiki, hard, tato) can be re-run by a third party."""
import argparse, json, os, re, sys, time
for _k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"): os.environ.setdefault(_k, "1") # 1 BLAS thread per worker
from multiprocessing import get_context
import numpy as np
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE)
from sets_v1 import CORE, TRUST, PUBLIC, available, rows, ok, part_of
PRED = os.path.join(HERE, "out", "preds"); TAG = "wikalid-1.0.1"; BASE = ["glotlid", "openlid3", "lid176"]
CBKM = set(json.load(open(os.path.join(HERE, "data", "cbk_markers.json"))))
_P = _G = None
def _init():
global _P, _G
import wikalid as ourlid
from wikalid.gate import Gate
_P = ourlid.load(rules=True); _G = Gate(ourlid.model_dir())
def _work(batch):
out = {}
for i, t in batch:
a = _P.analyze(t); g = _G.decide_one(t, a)
out[i] = [a["label"], a["conf"], a["mixture"], a["cs_level"], a["cs_share"], a["abstain"], a["label_set"], g["gate"], g["gate_p_correct"]]
return out
def predict(workers, resume=False):
import wikalid as ourlid
os.makedirs(PRED, exist_ok=True); speed = {}
for n in available():
if resume and os.path.exists(f"{PRED}/{TAG}__{n}.json"): continue
R = rows(n, "all"); items = [(r["id"], r["text"]) for r in R]; B = [items[i:i + 200] for i in range(0, len(items), 200)]
t0 = time.time(); pred = {}
with get_context("fork").Pool(workers, initializer=_init) as pool:
for d in pool.imap_unordered(_work, B): pred.update(d)
el = time.time() - t0; speed[n] = len(R) / el
json.dump({"tool": TAG, "meta": {"version": ourlid.__version__, "items_per_s_total": speed[n], "workers": workers}, "pred": pred},
open(f"{PRED}/{TAG}__{n}.json", "w"))
print(TAG, n, len(R), f"{speed[n]:.0f} it/s ({workers} workers)", flush=True)
return speed
def load(t, n):
p = f"{PRED}/{t}__{n}.json"
return json.load(open(p))["pred"] if os.path.exists(p) else None
def corrected(n, R):
ex = [load(t, n) for t in BASE]
if any(e is None for e in ex): return [r["label"] for r in R]
out = []
for r in R:
l, i = r["label"], r["id"]
if l != "eng" and n in ("wiki", "hard", "pld") and sum(1 for d in ex if d[i][0] == "eng" and d[i][1] >= 0.6) >= 2: l = "eng"
elif l == "cbk" and sum(1 for d in ex if d[i][0] == "spa" and d[i][1] >= 0.6) >= 2 and not (set(re.findall(r"[a-záéíóúñ']+", r["text"].lower())) & CBKM): l = "spa"
out.append(l)
return out
def score(n, R, P, cor):
raw = np.array([ok(n, r, P[r["id"]][0]) for r in R], float)
corr = np.array([(P[r["id"]][0] == g) or (g == r["label"] and ok(n, r, P[r["id"]][0])) for r, g in zip(R, cor)], float)
y = np.array(cor); p = np.array([P[r["id"]][0] for r in R]); f = []
for c in sorted(set(y)):
if (y == c).sum() < 15: continue
tp = ((y == c) & (p == c)).sum(); fp = ((y != c) & (p == c)).sum(); fn = ((y == c) & (p != c)).sum(); f.append(2 * tp / max(1, 2 * tp + fp + fn))
return raw, corr, float(np.mean(f)) if f else float("nan")
def wilson(k, n, z=1.96):
if n == 0: return (0.0, 0.0)
p = k / n; d = 1 + z * z / n; c = (p + z * z / 2 / n) / d; h = z * np.sqrt(p * (1 - p) / n + z * z / 4 / n / n) / d; return (c - h, c + h)
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--workers", type=int, default=4); ap.add_argument("--skip-predict", action="store_true"); ap.add_argument("--resume", action="store_true"); a = ap.parse_args()
speed = predict(a.workers, a.resume) if not a.skip_predict else {}
tools = [TAG] + BASE; res = {"version": TAG, "speed_items_per_s": speed, "core": {}, "trust": {}, "paired_vs_glotlid": {}, "gate_sealed": {}, "consistency_vs_research_c7r": {}}
rng = np.random.default_rng(0)
for part in ("sealed", "dev", "all"):
res["core"][part] = {}
for n in CORE:
if n not in available(): continue
R = rows(n, part); cor = corrected(n, R); res["core"][part][n] = {"n": len(R), "public": n in PUBLIC}
for t in tools:
P = load(t, n)
if P is None: continue
raw, corr, f1 = score(n, R, P, cor); res["core"][part][n][t] = {"acc_raw": raw.mean(), "acc_corrected": corr.mean(), "macro_f1_corrected": f1}
if part == "sealed" and t == TAG:
G = load("glotlid", n); _, cg, _ = score(n, R, G, cor); d = corr - cg; idx = rng.integers(0, len(R), (2000, len(R))); bs = d[idx].mean(1)
res["paired_vs_glotlid"][n] = {"delta": float(d.mean()), "ci95": [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]}
acc = [P[r["id"]][7] in ("accept", "accept_eng", "accept_ph_other") for r in R]
if any(acc): res["gate_sealed"][n] = {"coverage": float(np.mean(acc)), "precision_accepted_corrected": float(corr[np.array(acc)].mean()), "n": len(R)}
for n in TRUST:
if n not in available(): continue
R = rows(n, "all"); res["trust"][n] = {"n": len(R)}
for t in tools:
P = load(t, n)
if P is None: continue
hit = np.array([ok(n, r, P[r["id"]][0]) for r in R]); lo, hi = wilson(hit.sum(), len(R))
per = {}
for key in sorted({r["label"] for r in R} | {r.get("cat") for r in R if r.get("cat")}):
ix = [i for i, r in enumerate(R) if r["label"] == key or r.get("cat") == key]
k = int(hit[ix].sum()); l2, h2 = wilson(k, len(ix)); per[key] = {"acc": k / len(ix), "n": len(ix), "ci95": [l2, h2]}
res["trust"][n][t] = {"acc": float(hit.mean()), "ci95": [lo, hi], "by": per}
for n in available():
p = f"/workspace/ourlid/eval/preds/c7r__{n}.json"
A, B = load(TAG, n), (json.load(open(p))["pred"] if os.path.exists(p) else None)
if A and B: res["consistency_vs_research_c7r"][n] = {"n": len(A), "same_label": sum(A[i][0] == B[i][0] for i in A if i in B)}
json.dump(res, open(os.path.join(HERE, "RESULTS_v1.0.1.json"), "w"), indent=1, default=float)
write_md(res, tools)
def write_md(res, tools):
L = ["# WikaLID 1.0.1 evaluation (generated by eval/evaluate.py; do not edit by hand)", ""]
S = res["core"]["sealed"]; sets = list(S)
L += ["## Sealed halves (never used for any decision): accuracy raw / corrected gold (macro-F1 corrected)", "",
"| system | " + " | ".join(f"{n}{'*' if S[n]['public'] else ''} (n={S[n]['n']})" for n in sets) + " | mean corrected |", "|---|" + "---|" * (len(sets) + 1)]
for t in tools:
cells, ms = [], []
for n in sets:
c = S[n].get(t)
if not c: cells.append("n/a"); continue
cells.append(f"{c['acc_raw']:.3f} / {c['acc_corrected']:.3f} ({c['macro_f1_corrected']:.2f})"); ms.append(c["acc_corrected"])
L.append(f"| {t} | " + " | ".join(cells) + f" | **{np.mean(ms):.3f}** |")
L += ["", "\\* public set (ships in eval/data, anyone can re-run). Others are private (news gold, crawl realism, PLD read speech, BasahaCorpus stories).", "",
"## WikaLID 1.0.1 minus GlotLID v3, sealed, corrected gold (paired bootstrap, 2,000 resamples)", "", "| set | delta | 95% CI |", "|---|---|---|"]
for n, d in res["paired_vs_glotlid"].items(): L.append(f"| {n} | {d['delta']:+.3f} | [{d['ci95'][0]:+.3f}, {d['ci95'][1]:+.3f}] |")
L += ["", "## Gate v2 on sealed halves (accept = accept / accept_eng / accept_ph_other)", "", "| set | coverage | precision of accepted (corrected gold) |", "|---|---|---|"]
for n, d in res["gate_sealed"].items(): L.append(f"| {n} | {d['coverage']:.3f} | {d['precision_accepted_corrected']:.3f} |")
L += ["", "## Human-labelled external trust sets (all rows; accuracy with 95% Wilson interval)", ""]
for n, d in res["trust"].items():
L += [f"### {n} (n={d['n']})", "", "| system | overall | " + " | ".join(sorted(d[TAG]["by"])) + " |", "|---|---|" + "---|" * len(d[TAG]["by"])]
for t in tools:
if t not in d: continue
L.append(f"| {t} | {d[t]['acc']:.3f} [{d[t]['ci95'][0]:.2f}-{d[t]['ci95'][1]:.2f}] | " + " | ".join(f"{d[t]['by'][k]['acc']:.3f}" for k in sorted(d[TAG]["by"])) + " |")
L.append("")
L += ["## Consistency with the research model that labels Himay (c7 + rules, cached predictions)", "", "| set | n | same label |", "|---|---|---|"]
for n, d in res["consistency_vs_research_c7r"].items(): L.append(f"| {n} | {d['n']} | {d['same_label']} |")
if res["speed_items_per_s"]:
L += ["", "## Throughput during this run (items/s, all workers, shared box)", "", " | ".join(f"{n}: {v:.0f}" for n, v in res["speed_items_per_s"].items())]
open(os.path.join(HERE, "RESULTS_v1.0.1.md"), "w").write("\n".join(L) + "\n")
if __name__ == "__main__":
main()
|