"""Eval-set loader for OurLID 1.0 (same split and scoring rules as the research repo's eval/sets.py, which produced every earlier number). Public sets ship in eval/data; private sets (news gold, crawl realism, PLD, Basaha, trust sets) are read from $OURLID_PRIVATE_EVAL and are never published.""" import hashlib, json, os HERE = os.path.dirname(os.path.abspath(__file__)) PRIV = os.environ.get("OURLID_PRIVATE_EVAL", "/workspace/ourlid/eval/private") CORE = {"crawl": f"{PRIV}/crawl_realism_eval.jsonl", "goldspan": f"{PRIV}/gold_spans.jsonl", "golddoc": f"{PRIV}/gold_docs.jsonl", "pld": f"{PRIV}/pld.jsonl", "wiki": f"{HERE}/data/wiki_test_v0.jsonl", "hard": f"{HERE}/data/phlid_hard_v0.jsonl", "basaha": f"{PRIV}/basaha2_eval.jsonl", "tato": f"{HERE}/data/tato_test_v0.jsonl"} TRUST = {"t_sib": f"{PRIV}/trust_sib.jsonl", "t_frag": f"{PRIV}/trust_frag.jsonl", "t_caps": f"{PRIV}/trust_caps.jsonl", "t_bikol": f"{PRIV}/trust_bikol.jsonl", "t_aklsgd": f"{PRIV}/trust_aklsgd.jsonl", "t_tshort": f"{PRIV}/trust_tshort.jsonl"} PUBLIC = {"wiki", "hard", "tato"} SETS = {**CORE, **TRUST} def available(): return {n: p for n, p in SETS.items() if os.path.exists(p)} def group(n, r): i = r["id"] if r.get("grp"): return r["grp"] if n == "goldspan": return i.rsplit("-", 1)[0] if n == "basaha": return "_".join(i.split("_")[:3]) return i def part_of(n, r): h = int(hashlib.md5(("sealed-v1|" + n + "|" + group(n, r)).encode()).hexdigest(), 16) return "sealed" if h % 2 == 0 else "dev" def rows(n, part="all"): R = [json.loads(l) for l in open(SETS[n], encoding="utf8")] if n == "crawl": R = [r for r in R if not r["uncertain"]] if n == "hard": R = [r for r in R if r["cat"] != "quoted_english"] return R if part == "all" else [r for r in R if part_of(n, r) == part] def ok(n, r, p): if n == "crawl" and r.get("ambig_frame") and p in ("tgl", "eng") and r["label"] in ("tgl", "eng"): return True if n == "basaha" and p in r.get("lenient", []): return True return p == r["label"]