Download eval/evaluate.py from sapinsapin/wikalid: direct link, hf CLI and curl.
- Browser
- Download file 9.83 kB
-
https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/evaluate.py
- Command line
-
hf download hf://sapinsapin/wikalid/eval/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/sapinsapin/wikalid/resolve/main/eval/evaluate.py
9.83 kB
| """Reproducible evaluation of the installed `wikalid` package (1.0.1) against GlotLID v3, OpenLID-v3 and fastText lid.176. | |
| usage: python eval/evaluate.py [--workers 4] [--skip-predict] | |
| needs: eval/out/preds/{glotlid,openlid3,lid176}__<set>.json from eval/run_baselines.py (live fastText runs) | |
| writes: eval/out/preds/wikalid-1.0.1__<set>.json, eval/RESULTS_v1.0.1.json, eval/RESULTS_v1.0.1.md | |
| Protocol (unchanged from the research loop): every set is split 50/50 by md5 of a group key into dev (all tuning | |
| decisions were taken there) and sealed (never used for a decision). Headline = sealed halves. "corrected" gold fixes known | |
| label noise in wiki/hard/pld/cbk by consensus of GlotLID/OpenLID/lid176 (same rule as before; caveat: it shares detectors | |
| with the baselines it is compared against, which favours them, not us). Private sets are evaluated locally and never | |
| published; only the public sets (wiki, hard, tato) can be re-run by a third party.""" | |
| import argparse, json, os, re, sys, time | |
| for _k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS"): os.environ.setdefault(_k, "1") # 1 BLAS thread per worker | |
| from multiprocessing import get_context | |
| import numpy as np | |
| HERE = os.path.dirname(os.path.abspath(__file__)); sys.path.insert(0, HERE) | |
| from sets_v1 import CORE, TRUST, PUBLIC, available, rows, ok, part_of | |
| PRED = os.path.join(HERE, "out", "preds"); TAG = "wikalid-1.0.1"; BASE = ["glotlid", "openlid3", "lid176"] | |
| CBKM = set(json.load(open(os.path.join(HERE, "data", "cbk_markers.json")))) | |
| _P = _G = None | |
| def _init(): | |
| global _P, _G | |
| import wikalid as ourlid | |
| from wikalid.gate import Gate | |
| _P = ourlid.load(rules=True); _G = Gate(ourlid.model_dir()) | |
| def _work(batch): | |
| out = {} | |
| for i, t in batch: | |
| a = _P.analyze(t); g = _G.decide_one(t, a) | |
| out[i] = [a["label"], a["conf"], a["mixture"], a["cs_level"], a["cs_share"], a["abstain"], a["label_set"], g["gate"], g["gate_p_correct"]] | |
| return out | |
| def predict(workers, resume=False): | |
| import wikalid as ourlid | |
| os.makedirs(PRED, exist_ok=True); speed = {} | |
| for n in available(): | |
| if resume and os.path.exists(f"{PRED}/{TAG}__{n}.json"): continue | |
| R = rows(n, "all"); items = [(r["id"], r["text"]) for r in R]; B = [items[i:i + 200] for i in range(0, len(items), 200)] | |
| t0 = time.time(); pred = {} | |
| with get_context("fork").Pool(workers, initializer=_init) as pool: | |
| for d in pool.imap_unordered(_work, B): pred.update(d) | |
| el = time.time() - t0; speed[n] = len(R) / el | |
| json.dump({"tool": TAG, "meta": {"version": ourlid.__version__, "items_per_s_total": speed[n], "workers": workers}, "pred": pred}, | |
| open(f"{PRED}/{TAG}__{n}.json", "w")) | |
| print(TAG, n, len(R), f"{speed[n]:.0f} it/s ({workers} workers)", flush=True) | |
| return speed | |
| def load(t, n): | |
| p = f"{PRED}/{t}__{n}.json" | |
| return json.load(open(p))["pred"] if os.path.exists(p) else None | |
| def corrected(n, R): | |
| ex = [load(t, n) for t in BASE] | |
| if any(e is None for e in ex): return [r["label"] for r in R] | |
| out = [] | |
| for r in R: | |
| l, i = r["label"], r["id"] | |
| if l != "eng" and n in ("wiki", "hard", "pld") and sum(1 for d in ex if d[i][0] == "eng" and d[i][1] >= 0.6) >= 2: l = "eng" | |
| elif l == "cbk" and sum(1 for d in ex if d[i][0] == "spa" and d[i][1] >= 0.6) >= 2 and not (set(re.findall(r"[a-záéíóúñ']+", r["text"].lower())) & CBKM): l = "spa" | |
| out.append(l) | |
| return out | |
| def score(n, R, P, cor): | |
| raw = np.array([ok(n, r, P[r["id"]][0]) for r in R], float) | |
| corr = np.array([(P[r["id"]][0] == g) or (g == r["label"] and ok(n, r, P[r["id"]][0])) for r, g in zip(R, cor)], float) | |
| y = np.array(cor); p = np.array([P[r["id"]][0] for r in R]); f = [] | |
| for c in sorted(set(y)): | |
| if (y == c).sum() < 15: continue | |
| tp = ((y == c) & (p == c)).sum(); fp = ((y != c) & (p == c)).sum(); fn = ((y == c) & (p != c)).sum(); f.append(2 * tp / max(1, 2 * tp + fp + fn)) | |
| return raw, corr, float(np.mean(f)) if f else float("nan") | |
| def wilson(k, n, z=1.96): | |
| if n == 0: return (0.0, 0.0) | |
| p = k / n; d = 1 + z * z / n; c = (p + z * z / 2 / n) / d; h = z * np.sqrt(p * (1 - p) / n + z * z / 4 / n / n) / d; return (c - h, c + h) | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("--workers", type=int, default=4); ap.add_argument("--skip-predict", action="store_true"); ap.add_argument("--resume", action="store_true"); a = ap.parse_args() | |
| speed = predict(a.workers, a.resume) if not a.skip_predict else {} | |
| tools = [TAG] + BASE; res = {"version": TAG, "speed_items_per_s": speed, "core": {}, "trust": {}, "paired_vs_glotlid": {}, "gate_sealed": {}, "consistency_vs_research_c7r": {}} | |
| rng = np.random.default_rng(0) | |
| for part in ("sealed", "dev", "all"): | |
| res["core"][part] = {} | |
| for n in CORE: | |
| if n not in available(): continue | |
| R = rows(n, part); cor = corrected(n, R); res["core"][part][n] = {"n": len(R), "public": n in PUBLIC} | |
| for t in tools: | |
| P = load(t, n) | |
| if P is None: continue | |
| raw, corr, f1 = score(n, R, P, cor); res["core"][part][n][t] = {"acc_raw": raw.mean(), "acc_corrected": corr.mean(), "macro_f1_corrected": f1} | |
| if part == "sealed" and t == TAG: | |
| G = load("glotlid", n); _, cg, _ = score(n, R, G, cor); d = corr - cg; idx = rng.integers(0, len(R), (2000, len(R))); bs = d[idx].mean(1) | |
| res["paired_vs_glotlid"][n] = {"delta": float(d.mean()), "ci95": [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]} | |
| acc = [P[r["id"]][7] in ("accept", "accept_eng", "accept_ph_other") for r in R] | |
| if any(acc): res["gate_sealed"][n] = {"coverage": float(np.mean(acc)), "precision_accepted_corrected": float(corr[np.array(acc)].mean()), "n": len(R)} | |
| for n in TRUST: | |
| if n not in available(): continue | |
| R = rows(n, "all"); res["trust"][n] = {"n": len(R)} | |
| for t in tools: | |
| P = load(t, n) | |
| if P is None: continue | |
| hit = np.array([ok(n, r, P[r["id"]][0]) for r in R]); lo, hi = wilson(hit.sum(), len(R)) | |
| per = {} | |
| for key in sorted({r["label"] for r in R} | {r.get("cat") for r in R if r.get("cat")}): | |
| ix = [i for i, r in enumerate(R) if r["label"] == key or r.get("cat") == key] | |
| k = int(hit[ix].sum()); l2, h2 = wilson(k, len(ix)); per[key] = {"acc": k / len(ix), "n": len(ix), "ci95": [l2, h2]} | |
| res["trust"][n][t] = {"acc": float(hit.mean()), "ci95": [lo, hi], "by": per} | |
| for n in available(): | |
| p = f"/workspace/ourlid/eval/preds/c7r__{n}.json" | |
| A, B = load(TAG, n), (json.load(open(p))["pred"] if os.path.exists(p) else None) | |
| if A and B: res["consistency_vs_research_c7r"][n] = {"n": len(A), "same_label": sum(A[i][0] == B[i][0] for i in A if i in B)} | |
| json.dump(res, open(os.path.join(HERE, "RESULTS_v1.0.1.json"), "w"), indent=1, default=float) | |
| write_md(res, tools) | |
| def write_md(res, tools): | |
| L = ["# WikaLID 1.0.1 evaluation (generated by eval/evaluate.py; do not edit by hand)", ""] | |
| S = res["core"]["sealed"]; sets = list(S) | |
| L += ["## Sealed halves (never used for any decision): accuracy raw / corrected gold (macro-F1 corrected)", "", | |
| "| system | " + " | ".join(f"{n}{'*' if S[n]['public'] else ''} (n={S[n]['n']})" for n in sets) + " | mean corrected |", "|---|" + "---|" * (len(sets) + 1)] | |
| for t in tools: | |
| cells, ms = [], [] | |
| for n in sets: | |
| c = S[n].get(t) | |
| if not c: cells.append("n/a"); continue | |
| cells.append(f"{c['acc_raw']:.3f} / {c['acc_corrected']:.3f} ({c['macro_f1_corrected']:.2f})"); ms.append(c["acc_corrected"]) | |
| L.append(f"| {t} | " + " | ".join(cells) + f" | **{np.mean(ms):.3f}** |") | |
| L += ["", "\\* public set (ships in eval/data, anyone can re-run). Others are private (news gold, crawl realism, PLD read speech, BasahaCorpus stories).", "", | |
| "## WikaLID 1.0.1 minus GlotLID v3, sealed, corrected gold (paired bootstrap, 2,000 resamples)", "", "| set | delta | 95% CI |", "|---|---|---|"] | |
| for n, d in res["paired_vs_glotlid"].items(): L.append(f"| {n} | {d['delta']:+.3f} | [{d['ci95'][0]:+.3f}, {d['ci95'][1]:+.3f}] |") | |
| L += ["", "## Gate v2 on sealed halves (accept = accept / accept_eng / accept_ph_other)", "", "| set | coverage | precision of accepted (corrected gold) |", "|---|---|---|"] | |
| for n, d in res["gate_sealed"].items(): L.append(f"| {n} | {d['coverage']:.3f} | {d['precision_accepted_corrected']:.3f} |") | |
| L += ["", "## Human-labelled external trust sets (all rows; accuracy with 95% Wilson interval)", ""] | |
| for n, d in res["trust"].items(): | |
| L += [f"### {n} (n={d['n']})", "", "| system | overall | " + " | ".join(sorted(d[TAG]["by"])) + " |", "|---|---|" + "---|" * len(d[TAG]["by"])] | |
| for t in tools: | |
| if t not in d: continue | |
| L.append(f"| {t} | {d[t]['acc']:.3f} [{d[t]['ci95'][0]:.2f}-{d[t]['ci95'][1]:.2f}] | " + " | ".join(f"{d[t]['by'][k]['acc']:.3f}" for k in sorted(d[TAG]["by"])) + " |") | |
| L.append("") | |
| L += ["## Consistency with the research model that labels Himay (c7 + rules, cached predictions)", "", "| set | n | same label |", "|---|---|---|"] | |
| for n, d in res["consistency_vs_research_c7r"].items(): L.append(f"| {n} | {d['n']} | {d['same_label']} |") | |
| if res["speed_items_per_s"]: | |
| L += ["", "## Throughput during this run (items/s, all workers, shared box)", "", " | ".join(f"{n}: {v:.0f}" for n, v in res["speed_items_per_s"].items())] | |
| open(os.path.join(HERE, "RESULTS_v1.0.1.md"), "w").write("\n".join(L) + "\n") | |
| if __name__ == "__main__": | |
| main() | |