wikalid / eval /sets_v1.py
jpaulpoliquit's picture
OurLID 1.0.0 (private preview): package, weights, model card, reproducible eval
863c520 verified
Raw History Blame Contribute Delete
2.07 kB
"""Eval-set loader for OurLID 1.0 (same split and scoring rules as the research repo's eval/sets.py, which produced every
earlier number). Public sets ship in eval/data; private sets (news gold, crawl realism, PLD, Basaha, trust sets) are read
from $OURLID_PRIVATE_EVAL and are never published."""
import hashlib, json, os
HERE = os.path.dirname(os.path.abspath(__file__))
PRIV = os.environ.get("OURLID_PRIVATE_EVAL", "/workspace/ourlid/eval/private")
CORE = {"crawl": f"{PRIV}/crawl_realism_eval.jsonl", "goldspan": f"{PRIV}/gold_spans.jsonl", "golddoc": f"{PRIV}/gold_docs.jsonl",
"pld": f"{PRIV}/pld.jsonl", "wiki": f"{HERE}/data/wiki_test_v0.jsonl", "hard": f"{HERE}/data/phlid_hard_v0.jsonl",
"basaha": f"{PRIV}/basaha2_eval.jsonl", "tato": f"{HERE}/data/tato_test_v0.jsonl"}
TRUST = {"t_sib": f"{PRIV}/trust_sib.jsonl", "t_frag": f"{PRIV}/trust_frag.jsonl", "t_caps": f"{PRIV}/trust_caps.jsonl",
"t_bikol": f"{PRIV}/trust_bikol.jsonl", "t_aklsgd": f"{PRIV}/trust_aklsgd.jsonl", "t_tshort": f"{PRIV}/trust_tshort.jsonl"}
PUBLIC = {"wiki", "hard", "tato"}
SETS = {**CORE, **TRUST}
def available():
return {n: p for n, p in SETS.items() if os.path.exists(p)}
def group(n, r):
i = r["id"]
if r.get("grp"): return r["grp"]
if n == "goldspan": return i.rsplit("-", 1)[0]
if n == "basaha": return "_".join(i.split("_")[:3])
return i
def part_of(n, r):
h = int(hashlib.md5(("sealed-v1|" + n + "|" + group(n, r)).encode()).hexdigest(), 16)
return "sealed" if h % 2 == 0 else "dev"
def rows(n, part="all"):
R = [json.loads(l) for l in open(SETS[n], encoding="utf8")]
if n == "crawl": R = [r for r in R if not r["uncertain"]]
if n == "hard": R = [r for r in R if r["cat"] != "quoted_english"]
return R if part == "all" else [r for r in R if part_of(n, r) == part]
def ok(n, r, p):
if n == "crawl" and r.get("ambig_frame") and p in ("tgl", "eng") and r["label"] in ("tgl", "eng"): return True
if n == "basaha" and p in r.get("lenient", []): return True
return p == r["label"]