wikalid / eval /evaluate.py
jpaulpoliquit's picture
WikaLID 1.0.1: rename from OurLID, public model card, ourlid alias kept
8f78c5a verified
Raw History Blame Contribute Delete
9.83 kB
"""Reproducible evaluation of the installed `wikalid` package (1.0.1) against GlotLID v3, OpenLID-v3 and fastText lid.176.
usage: python eval/evaluate.py [--workers 4] [--skip-predict]
needs: eval/out/preds/{glotlid,openlid3,lid176}__<set>.json from eval/run_baselines.py (live fastText runs)
writes: eval/out/preds/wikalid-1.0.1__<set>.json, eval/RESULTS_v1.0.1.json, eval/RESULTS_v1.0.1.md
Protocol (unchanged from the research loop): every set is split 50/50 by md5 of a group key into dev (all tuning
decisions were taken there) and sealed (never used for a decision). Headline = sealed halves. "corrected" gold fixes known
label noise in wiki/hard/pld/cbk by consensus of GlotLID/OpenLID/lid176 (same rule as before; caveat: it shares detectors
with the baselines it is compared against, which favours them, not us). Private sets are evaluated locally and never
published; only the public sets (wiki, hard, tato) can be re-run by a third party."""
import argparse, json, os, re, sys, time
for _k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"): os.environ.setdefault(_k, "1") # 1 BLAS thread per worker
from multiprocessing import get_context
import numpy as np
HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE)
from sets_v1 import CORE, TRUST, PUBLIC, available, rows, ok, part_of
PRED = os.path.join(HERE, "out", "preds"); TAG = "wikalid-1.0.1"; BASE = ["glotlid", "openlid3", "lid176"]
CBKM = set(json.load(open(os.path.join(HERE, "data", "cbk_markers.json"))))
_P = _G = None
def _init():
global _P, _G
import wikalid as ourlid
from wikalid.gate import Gate
_P = ourlid.load(rules=True); _G = Gate(ourlid.model_dir())
def _work(batch):
out = {}
for i, t in batch:
a = _P.analyze(t); g = _G.decide_one(t, a)
out[i] = [a["label"], a["conf"], a["mixture"], a["cs_level"], a["cs_share"], a["abstain"], a["label_set"], g["gate"], g["gate_p_correct"]]
return out
def predict(workers, resume=False):
import wikalid as ourlid
os.makedirs(PRED, exist_ok=True); speed = {}
for n in available():
if resume and os.path.exists(f"{PRED}/{TAG}__{n}.json"): continue
R = rows(n, "all"); items = [(r["id"], r["text"]) for r in R]; B = [items[i:i + 200] for i in range(0, len(items), 200)]
t0 = time.time(); pred = {}
with get_context("fork").Pool(workers, initializer=_init) as pool:
for d in pool.imap_unordered(_work, B): pred.update(d)
el = time.time() - t0; speed[n] = len(R) / el
json.dump({"tool": TAG, "meta": {"version": ourlid.__version__, "items_per_s_total": speed[n], "workers": workers}, "pred": pred},
open(f"{PRED}/{TAG}__{n}.json", "w"))
print(TAG, n, len(R), f"{speed[n]:.0f} it/s ({workers} workers)", flush=True)
return speed
def load(t, n):
p = f"{PRED}/{t}__{n}.json"
return json.load(open(p))["pred"] if os.path.exists(p) else None
def corrected(n, R):
ex = [load(t, n) for t in BASE]
if any(e is None for e in ex): return [r["label"] for r in R]
out = []
for r in R:
l, i = r["label"], r["id"]
if l != "eng" and n in ("wiki", "hard", "pld") and sum(1 for d in ex if d[i][0] == "eng" and d[i][1] >= 0.6) >= 2: l = "eng"
elif l == "cbk" and sum(1 for d in ex if d[i][0] == "spa" and d[i][1] >= 0.6) >= 2 and not (set(re.findall(r"[a-záéíóúñ']+", r["text"].lower())) & CBKM): l = "spa"
out.append(l)
return out
def score(n, R, P, cor):
raw = np.array([ok(n, r, P[r["id"]][0]) for r in R], float)
corr = np.array([(P[r["id"]][0] == g) or (g == r["label"] and ok(n, r, P[r["id"]][0])) for r, g in zip(R, cor)], float)
y = np.array(cor); p = np.array([P[r["id"]][0] for r in R]); f = []
for c in sorted(set(y)):
if (y == c).sum() < 15: continue
tp = ((y == c) & (p == c)).sum(); fp = ((y != c) & (p == c)).sum(); fn = ((y == c) & (p != c)).sum(); f.append(2 * tp / max(1, 2 * tp + fp + fn))
return raw, corr, float(np.mean(f)) if f else float("nan")
def wilson(k, n, z=1.96):
if n == 0: return (0.0, 0.0)
p = k / n; d = 1 + z * z / n; c = (p + z * z / 2 / n) / d; h = z * np.sqrt(p * (1 - p) / n + z * z / 4 / n / n) / d; return (c - h, c + h)
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--workers", type=int, default=4); ap.add_argument("--skip-predict", action="store_true"); ap.add_argument("--resume", action="store_true"); a = ap.parse_args()
speed = predict(a.workers, a.resume) if not a.skip_predict else {}
tools = [TAG] + BASE; res = {"version": TAG, "speed_items_per_s": speed, "core": {}, "trust": {}, "paired_vs_glotlid": {}, "gate_sealed": {}, "consistency_vs_research_c7r": {}}
rng = np.random.default_rng(0)
for part in ("sealed", "dev", "all"):
res["core"][part] = {}
for n in CORE:
if n not in available(): continue
R = rows(n, part); cor = corrected(n, R); res["core"][part][n] = {"n": len(R), "public": n in PUBLIC}
for t in tools:
P = load(t, n)
if P is None: continue
raw, corr, f1 = score(n, R, P, cor); res["core"][part][n][t] = {"acc_raw": raw.mean(), "acc_corrected": corr.mean(), "macro_f1_corrected": f1}
if part == "sealed" and t == TAG:
G = load("glotlid", n); _, cg, _ = score(n, R, G, cor); d = corr - cg; idx = rng.integers(0, len(R), (2000, len(R))); bs = d[idx].mean(1)
res["paired_vs_glotlid"][n] = {"delta": float(d.mean()), "ci95": [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]}
acc = [P[r["id"]][7] in ("accept", "accept_eng", "accept_ph_other") for r in R]
if any(acc): res["gate_sealed"][n] = {"coverage": float(np.mean(acc)), "precision_accepted_corrected": float(corr[np.array(acc)].mean()), "n": len(R)}
for n in TRUST:
if n not in available(): continue
R = rows(n, "all"); res["trust"][n] = {"n": len(R)}
for t in tools:
P = load(t, n)
if P is None: continue
hit = np.array([ok(n, r, P[r["id"]][0]) for r in R]); lo, hi = wilson(hit.sum(), len(R))
per = {}
for key in sorted({r["label"] for r in R} | {r.get("cat") for r in R if r.get("cat")}):
ix = [i for i, r in enumerate(R) if r["label"] == key or r.get("cat") == key]
k = int(hit[ix].sum()); l2, h2 = wilson(k, len(ix)); per[key] = {"acc": k / len(ix), "n": len(ix), "ci95": [l2, h2]}
res["trust"][n][t] = {"acc": float(hit.mean()), "ci95": [lo, hi], "by": per}
for n in available():
p = f"/workspace/ourlid/eval/preds/c7r__{n}.json"
A, B = load(TAG, n), (json.load(open(p))["pred"] if os.path.exists(p) else None)
if A and B: res["consistency_vs_research_c7r"][n] = {"n": len(A), "same_label": sum(A[i][0] == B[i][0] for i in A if i in B)}
json.dump(res, open(os.path.join(HERE, "RESULTS_v1.0.1.json"), "w"), indent=1, default=float)
write_md(res, tools)
def write_md(res, tools):
L = ["# WikaLID 1.0.1 evaluation (generated by eval/evaluate.py; do not edit by hand)", ""]
S = res["core"]["sealed"]; sets = list(S)
L += ["## Sealed halves (never used for any decision): accuracy raw / corrected gold (macro-F1 corrected)", "",
"| system | " + " | ".join(f"{n}{'*' if S[n]['public'] else ''} (n={S[n]['n']})" for n in sets) + " | mean corrected |", "|---|" + "---|" * (len(sets) + 1)]
for t in tools:
cells, ms = [], []
for n in sets:
c = S[n].get(t)
if not c: cells.append("n/a"); continue
cells.append(f"{c['acc_raw']:.3f} / {c['acc_corrected']:.3f} ({c['macro_f1_corrected']:.2f})"); ms.append(c["acc_corrected"])
L.append(f"| {t} | " + " | ".join(cells) + f" | **{np.mean(ms):.3f}** |")
L += ["", "\\* public set (ships in eval/data, anyone can re-run). Others are private (news gold, crawl realism, PLD read speech, BasahaCorpus stories).", "",
"## WikaLID 1.0.1 minus GlotLID v3, sealed, corrected gold (paired bootstrap, 2,000 resamples)", "", "| set | delta | 95% CI |", "|---|---|---|"]
for n, d in res["paired_vs_glotlid"].items(): L.append(f"| {n} | {d['delta']:+.3f} | [{d['ci95'][0]:+.3f}, {d['ci95'][1]:+.3f}] |")
L += ["", "## Gate v2 on sealed halves (accept = accept / accept_eng / accept_ph_other)", "", "| set | coverage | precision of accepted (corrected gold) |", "|---|---|---|"]
for n, d in res["gate_sealed"].items(): L.append(f"| {n} | {d['coverage']:.3f} | {d['precision_accepted_corrected']:.3f} |")
L += ["", "## Human-labelled external trust sets (all rows; accuracy with 95% Wilson interval)", ""]
for n, d in res["trust"].items():
L += [f"### {n} (n={d['n']})", "", "| system | overall | " + " | ".join(sorted(d[TAG]["by"])) + " |", "|---|---|" + "---|" * len(d[TAG]["by"])]
for t in tools:
if t not in d: continue
L.append(f"| {t} | {d[t]['acc']:.3f} [{d[t]['ci95'][0]:.2f}-{d[t]['ci95'][1]:.2f}] | " + " | ".join(f"{d[t]['by'][k]['acc']:.3f}" for k in sorted(d[TAG]["by"])) + " |")
L.append("")
L += ["## Consistency with the research model that labels Himay (c7 + rules, cached predictions)", "", "| set | n | same label |", "|---|---|---|"]
for n, d in res["consistency_vs_research_c7r"].items(): L.append(f"| {n} | {d['n']} | {d['same_label']} |")
if res["speed_items_per_s"]:
L += ["", "## Throughput during this run (items/s, all workers, shared box)", "", " | ".join(f"{n}: {v:.0f}" for n, v in res["speed_items_per_s"].items())]
open(os.path.join(HERE, "RESULTS_v1.0.1.md"), "w").write("\n".join(L) + "\n")
if __name__ == "__main__":
main()