File size: 2,065 Bytes
863c520
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
"""Eval-set loader for OurLID 1.0 (same split and scoring rules as the research repo's eval/sets.py, which produced every
earlier number). Public sets ship in eval/data; private sets (news gold, crawl realism, PLD, Basaha, trust sets) are read
from $OURLID_PRIVATE_EVAL and are never published."""
import hashlib, json, os
HERE = os.path.dirname(os.path.abspath(__file__))
PRIV = os.environ.get("OURLID_PRIVATE_EVAL", "/workspace/ourlid/eval/private")
CORE = {"crawl": f"{PRIV}/crawl_realism_eval.jsonl", "goldspan": f"{PRIV}/gold_spans.jsonl", "golddoc": f"{PRIV}/gold_docs.jsonl",
        "pld": f"{PRIV}/pld.jsonl", "wiki": f"{HERE}/data/wiki_test_v0.jsonl", "hard": f"{HERE}/data/phlid_hard_v0.jsonl",
        "basaha": f"{PRIV}/basaha2_eval.jsonl", "tato": f"{HERE}/data/tato_test_v0.jsonl"}
TRUST = {"t_sib": f"{PRIV}/trust_sib.jsonl", "t_frag": f"{PRIV}/trust_frag.jsonl", "t_caps": f"{PRIV}/trust_caps.jsonl",
         "t_bikol": f"{PRIV}/trust_bikol.jsonl", "t_aklsgd": f"{PRIV}/trust_aklsgd.jsonl", "t_tshort": f"{PRIV}/trust_tshort.jsonl"}
PUBLIC = {"wiki", "hard", "tato"}
SETS = {**CORE, **TRUST}


def available():
    return {n: p for n, p in SETS.items() if os.path.exists(p)}


def group(n, r):
    i = r["id"]
    if r.get("grp"): return r["grp"]
    if n == "goldspan": return i.rsplit("-", 1)[0]
    if n == "basaha": return "_".join(i.split("_")[:3])
    return i


def part_of(n, r):
    h = int(hashlib.md5(("sealed-v1|" + n + "|" + group(n, r)).encode()).hexdigest(), 16)
    return "sealed" if h % 2 == 0 else "dev"


def rows(n, part="all"):
    R = [json.loads(l) for l in open(SETS[n], encoding="utf8")]
    if n == "crawl": R = [r for r in R if not r["uncertain"]]
    if n == "hard": R = [r for r in R if r["cat"] != "quoted_english"]
    return R if part == "all" else [r for r in R if part_of(n, r) == part]


def ok(n, r, p):
    if n == "crawl" and r.get("ambig_frame") and p in ("tgl", "eng") and r["label"] in ("tgl", "eng"): return True
    if n == "basaha" and p in r.get("lenient", []): return True
    return p == r["label"]